test_find_replace.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974
  1. from datetime import datetime
  2. import re
  3. import numpy as np
  4. import pytest
  5. from pandas.errors import PerformanceWarning
  6. import pandas as pd
  7. from pandas import (
  8. Series,
  9. _testing as tm,
  10. )
  11. # --------------------------------------------------------------------------------------
  12. # str.contains
  13. # --------------------------------------------------------------------------------------
  14. def test_contains(any_string_dtype):
  15. values = np.array(
  16. ["foo", np.nan, "fooommm__foo", "mmm_", "foommm[_]+bar"], dtype=np.object_
  17. )
  18. values = Series(values, dtype=any_string_dtype)
  19. pat = "mmm[_]+"
  20. result = values.str.contains(pat)
  21. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  22. expected = Series(
  23. np.array([False, np.nan, True, True, False], dtype=np.object_),
  24. dtype=expected_dtype,
  25. )
  26. tm.assert_series_equal(result, expected)
  27. result = values.str.contains(pat, regex=False)
  28. expected = Series(
  29. np.array([False, np.nan, False, False, True], dtype=np.object_),
  30. dtype=expected_dtype,
  31. )
  32. tm.assert_series_equal(result, expected)
  33. values = Series(
  34. np.array(["foo", "xyz", "fooommm__foo", "mmm_"], dtype=object),
  35. dtype=any_string_dtype,
  36. )
  37. result = values.str.contains(pat)
  38. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  39. expected = Series(np.array([False, False, True, True]), dtype=expected_dtype)
  40. tm.assert_series_equal(result, expected)
  41. # case insensitive using regex
  42. values = Series(
  43. np.array(["Foo", "xYz", "fOOomMm__fOo", "MMM_"], dtype=object),
  44. dtype=any_string_dtype,
  45. )
  46. with tm.maybe_produces_warning(
  47. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  48. ):
  49. result = values.str.contains("FOO|mmm", case=False)
  50. expected = Series(np.array([True, False, True, True]), dtype=expected_dtype)
  51. tm.assert_series_equal(result, expected)
  52. # case insensitive without regex
  53. result = values.str.contains("foo", regex=False, case=False)
  54. expected = Series(np.array([True, False, True, False]), dtype=expected_dtype)
  55. tm.assert_series_equal(result, expected)
  56. # unicode
  57. values = Series(
  58. np.array(["foo", np.nan, "fooommm__foo", "mmm_"], dtype=np.object_),
  59. dtype=any_string_dtype,
  60. )
  61. pat = "mmm[_]+"
  62. result = values.str.contains(pat)
  63. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  64. expected = Series(
  65. np.array([False, np.nan, True, True], dtype=np.object_), dtype=expected_dtype
  66. )
  67. tm.assert_series_equal(result, expected)
  68. result = values.str.contains(pat, na=False)
  69. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  70. expected = Series(np.array([False, False, True, True]), dtype=expected_dtype)
  71. tm.assert_series_equal(result, expected)
  72. values = Series(
  73. np.array(["foo", "xyz", "fooommm__foo", "mmm_"], dtype=np.object_),
  74. dtype=any_string_dtype,
  75. )
  76. result = values.str.contains(pat)
  77. expected = Series(np.array([False, False, True, True]), dtype=expected_dtype)
  78. tm.assert_series_equal(result, expected)
  79. def test_contains_object_mixed():
  80. mixed = Series(
  81. np.array(
  82. ["a", np.nan, "b", True, datetime.today(), "foo", None, 1, 2.0],
  83. dtype=object,
  84. )
  85. )
  86. result = mixed.str.contains("o")
  87. expected = Series(
  88. np.array(
  89. [False, np.nan, False, np.nan, np.nan, True, np.nan, np.nan, np.nan],
  90. dtype=np.object_,
  91. )
  92. )
  93. tm.assert_series_equal(result, expected)
  94. def test_contains_na_kwarg_for_object_category():
  95. # gh 22158
  96. # na for category
  97. values = Series(["a", "b", "c", "a", np.nan], dtype="category")
  98. result = values.str.contains("a", na=True)
  99. expected = Series([True, False, False, True, True])
  100. tm.assert_series_equal(result, expected)
  101. result = values.str.contains("a", na=False)
  102. expected = Series([True, False, False, True, False])
  103. tm.assert_series_equal(result, expected)
  104. # na for objects
  105. values = Series(["a", "b", "c", "a", np.nan])
  106. result = values.str.contains("a", na=True)
  107. expected = Series([True, False, False, True, True])
  108. tm.assert_series_equal(result, expected)
  109. result = values.str.contains("a", na=False)
  110. expected = Series([True, False, False, True, False])
  111. tm.assert_series_equal(result, expected)
  112. @pytest.mark.parametrize(
  113. "na, expected",
  114. [
  115. (None, pd.NA),
  116. (True, True),
  117. (False, False),
  118. (0, False),
  119. (3, True),
  120. (np.nan, pd.NA),
  121. ],
  122. )
  123. @pytest.mark.parametrize("regex", [True, False])
  124. def test_contains_na_kwarg_for_nullable_string_dtype(
  125. nullable_string_dtype, na, expected, regex
  126. ):
  127. # https://github.com/pandas-dev/pandas/pull/41025#issuecomment-824062416
  128. values = Series(["a", "b", "c", "a", np.nan], dtype=nullable_string_dtype)
  129. result = values.str.contains("a", na=na, regex=regex)
  130. expected = Series([True, False, False, True, expected], dtype="boolean")
  131. tm.assert_series_equal(result, expected)
  132. def test_contains_moar(any_string_dtype):
  133. # PR #1179
  134. s = Series(
  135. ["A", "B", "C", "Aaba", "Baca", "", np.nan, "CABA", "dog", "cat"],
  136. dtype=any_string_dtype,
  137. )
  138. result = s.str.contains("a")
  139. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  140. expected = Series(
  141. [False, False, False, True, True, False, np.nan, False, False, True],
  142. dtype=expected_dtype,
  143. )
  144. tm.assert_series_equal(result, expected)
  145. with tm.maybe_produces_warning(
  146. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  147. ):
  148. result = s.str.contains("a", case=False)
  149. expected = Series(
  150. [True, False, False, True, True, False, np.nan, True, False, True],
  151. dtype=expected_dtype,
  152. )
  153. tm.assert_series_equal(result, expected)
  154. result = s.str.contains("Aa")
  155. expected = Series(
  156. [False, False, False, True, False, False, np.nan, False, False, False],
  157. dtype=expected_dtype,
  158. )
  159. tm.assert_series_equal(result, expected)
  160. result = s.str.contains("ba")
  161. expected = Series(
  162. [False, False, False, True, False, False, np.nan, False, False, False],
  163. dtype=expected_dtype,
  164. )
  165. tm.assert_series_equal(result, expected)
  166. with tm.maybe_produces_warning(
  167. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  168. ):
  169. result = s.str.contains("ba", case=False)
  170. expected = Series(
  171. [False, False, False, True, True, False, np.nan, True, False, False],
  172. dtype=expected_dtype,
  173. )
  174. tm.assert_series_equal(result, expected)
  175. def test_contains_nan(any_string_dtype):
  176. # PR #14171
  177. s = Series([np.nan, np.nan, np.nan], dtype=any_string_dtype)
  178. result = s.str.contains("foo", na=False)
  179. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  180. expected = Series([False, False, False], dtype=expected_dtype)
  181. tm.assert_series_equal(result, expected)
  182. result = s.str.contains("foo", na=True)
  183. expected = Series([True, True, True], dtype=expected_dtype)
  184. tm.assert_series_equal(result, expected)
  185. result = s.str.contains("foo", na="foo")
  186. if any_string_dtype == "object":
  187. expected = Series(["foo", "foo", "foo"], dtype=np.object_)
  188. else:
  189. expected = Series([True, True, True], dtype="boolean")
  190. tm.assert_series_equal(result, expected)
  191. result = s.str.contains("foo")
  192. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  193. expected = Series([np.nan, np.nan, np.nan], dtype=expected_dtype)
  194. tm.assert_series_equal(result, expected)
  195. # --------------------------------------------------------------------------------------
  196. # str.startswith
  197. # --------------------------------------------------------------------------------------
  198. @pytest.mark.parametrize("pat", ["foo", ("foo", "baz")])
  199. @pytest.mark.parametrize("dtype", [None, "category"])
  200. @pytest.mark.parametrize("null_value", [None, np.nan, pd.NA])
  201. @pytest.mark.parametrize("na", [True, False])
  202. def test_startswith(pat, dtype, null_value, na):
  203. # add category dtype parametrizations for GH-36241
  204. values = Series(
  205. ["om", null_value, "foo_nom", "nom", "bar_foo", null_value, "foo"],
  206. dtype=dtype,
  207. )
  208. result = values.str.startswith(pat)
  209. exp = Series([False, np.nan, True, False, False, np.nan, True])
  210. tm.assert_series_equal(result, exp)
  211. result = values.str.startswith(pat, na=na)
  212. exp = Series([False, na, True, False, False, na, True])
  213. tm.assert_series_equal(result, exp)
  214. # mixed
  215. mixed = np.array(
  216. ["a", np.nan, "b", True, datetime.today(), "foo", None, 1, 2.0],
  217. dtype=np.object_,
  218. )
  219. rs = Series(mixed).str.startswith("f")
  220. xp = Series([False, np.nan, False, np.nan, np.nan, True, np.nan, np.nan, np.nan])
  221. tm.assert_series_equal(rs, xp)
  222. @pytest.mark.parametrize("na", [None, True, False])
  223. def test_startswith_nullable_string_dtype(nullable_string_dtype, na):
  224. values = Series(
  225. ["om", None, "foo_nom", "nom", "bar_foo", None, "foo", "regex", "rege."],
  226. dtype=nullable_string_dtype,
  227. )
  228. result = values.str.startswith("foo", na=na)
  229. exp = Series(
  230. [False, na, True, False, False, na, True, False, False], dtype="boolean"
  231. )
  232. tm.assert_series_equal(result, exp)
  233. result = values.str.startswith("rege.", na=na)
  234. exp = Series(
  235. [False, na, False, False, False, na, False, False, True], dtype="boolean"
  236. )
  237. tm.assert_series_equal(result, exp)
  238. # --------------------------------------------------------------------------------------
  239. # str.endswith
  240. # --------------------------------------------------------------------------------------
  241. @pytest.mark.parametrize("pat", ["foo", ("foo", "baz")])
  242. @pytest.mark.parametrize("dtype", [None, "category"])
  243. @pytest.mark.parametrize("null_value", [None, np.nan, pd.NA])
  244. @pytest.mark.parametrize("na", [True, False])
  245. def test_endswith(pat, dtype, null_value, na):
  246. # add category dtype parametrizations for GH-36241
  247. values = Series(
  248. ["om", null_value, "foo_nom", "nom", "bar_foo", null_value, "foo"],
  249. dtype=dtype,
  250. )
  251. result = values.str.endswith(pat)
  252. exp = Series([False, np.nan, False, False, True, np.nan, True])
  253. tm.assert_series_equal(result, exp)
  254. result = values.str.endswith(pat, na=na)
  255. exp = Series([False, na, False, False, True, na, True])
  256. tm.assert_series_equal(result, exp)
  257. # mixed
  258. mixed = np.array(
  259. ["a", np.nan, "b", True, datetime.today(), "foo", None, 1, 2.0],
  260. dtype=object,
  261. )
  262. rs = Series(mixed).str.endswith("f")
  263. xp = Series([False, np.nan, False, np.nan, np.nan, False, np.nan, np.nan, np.nan])
  264. tm.assert_series_equal(rs, xp)
  265. @pytest.mark.parametrize("na", [None, True, False])
  266. def test_endswith_nullable_string_dtype(nullable_string_dtype, na):
  267. values = Series(
  268. ["om", None, "foo_nom", "nom", "bar_foo", None, "foo", "regex", "rege."],
  269. dtype=nullable_string_dtype,
  270. )
  271. result = values.str.endswith("foo", na=na)
  272. exp = Series(
  273. [False, na, False, False, True, na, True, False, False], dtype="boolean"
  274. )
  275. tm.assert_series_equal(result, exp)
  276. result = values.str.endswith("rege.", na=na)
  277. exp = Series(
  278. [False, na, False, False, False, na, False, False, True], dtype="boolean"
  279. )
  280. tm.assert_series_equal(result, exp)
  281. # --------------------------------------------------------------------------------------
  282. # str.replace
  283. # --------------------------------------------------------------------------------------
  284. def test_replace(any_string_dtype):
  285. ser = Series(["fooBAD__barBAD", np.nan], dtype=any_string_dtype)
  286. result = ser.str.replace("BAD[_]*", "", regex=True)
  287. expected = Series(["foobar", np.nan], dtype=any_string_dtype)
  288. tm.assert_series_equal(result, expected)
  289. def test_replace_max_replacements(any_string_dtype):
  290. ser = Series(["fooBAD__barBAD", np.nan], dtype=any_string_dtype)
  291. expected = Series(["foobarBAD", np.nan], dtype=any_string_dtype)
  292. result = ser.str.replace("BAD[_]*", "", n=1, regex=True)
  293. tm.assert_series_equal(result, expected)
  294. expected = Series(["foo__barBAD", np.nan], dtype=any_string_dtype)
  295. result = ser.str.replace("BAD", "", n=1, regex=False)
  296. tm.assert_series_equal(result, expected)
  297. def test_replace_mixed_object():
  298. ser = Series(
  299. ["aBAD", np.nan, "bBAD", True, datetime.today(), "fooBAD", None, 1, 2.0]
  300. )
  301. result = Series(ser).str.replace("BAD[_]*", "", regex=True)
  302. expected = Series(["a", np.nan, "b", np.nan, np.nan, "foo", np.nan, np.nan, np.nan])
  303. tm.assert_series_equal(result, expected)
  304. def test_replace_unicode(any_string_dtype):
  305. ser = Series([b"abcd,\xc3\xa0".decode("utf-8")], dtype=any_string_dtype)
  306. expected = Series([b"abcd, \xc3\xa0".decode("utf-8")], dtype=any_string_dtype)
  307. with tm.maybe_produces_warning(
  308. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  309. ):
  310. result = ser.str.replace(r"(?<=\w),(?=\w)", ", ", flags=re.UNICODE, regex=True)
  311. tm.assert_series_equal(result, expected)
  312. @pytest.mark.parametrize("repl", [None, 3, {"a": "b"}])
  313. @pytest.mark.parametrize("data", [["a", "b", None], ["a", "b", "c", "ad"]])
  314. def test_replace_wrong_repl_type_raises(any_string_dtype, index_or_series, repl, data):
  315. # https://github.com/pandas-dev/pandas/issues/13438
  316. msg = "repl must be a string or callable"
  317. obj = index_or_series(data, dtype=any_string_dtype)
  318. with pytest.raises(TypeError, match=msg):
  319. obj.str.replace("a", repl)
  320. def test_replace_callable(any_string_dtype):
  321. # GH 15055
  322. ser = Series(["fooBAD__barBAD", np.nan], dtype=any_string_dtype)
  323. # test with callable
  324. repl = lambda m: m.group(0).swapcase()
  325. with tm.maybe_produces_warning(
  326. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  327. ):
  328. result = ser.str.replace("[a-z][A-Z]{2}", repl, n=2, regex=True)
  329. expected = Series(["foObaD__baRbaD", np.nan], dtype=any_string_dtype)
  330. tm.assert_series_equal(result, expected)
  331. @pytest.mark.parametrize(
  332. "repl", [lambda: None, lambda m, x: None, lambda m, x, y=None: None]
  333. )
  334. def test_replace_callable_raises(any_string_dtype, repl):
  335. # GH 15055
  336. values = Series(["fooBAD__barBAD", np.nan], dtype=any_string_dtype)
  337. # test with wrong number of arguments, raising an error
  338. msg = (
  339. r"((takes)|(missing)) (?(2)from \d+ to )?\d+ "
  340. r"(?(3)required )positional arguments?"
  341. )
  342. with pytest.raises(TypeError, match=msg):
  343. with tm.maybe_produces_warning(
  344. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  345. ):
  346. values.str.replace("a", repl, regex=True)
  347. def test_replace_callable_named_groups(any_string_dtype):
  348. # test regex named groups
  349. ser = Series(["Foo Bar Baz", np.nan], dtype=any_string_dtype)
  350. pat = r"(?P<first>\w+) (?P<middle>\w+) (?P<last>\w+)"
  351. repl = lambda m: m.group("middle").swapcase()
  352. with tm.maybe_produces_warning(
  353. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  354. ):
  355. result = ser.str.replace(pat, repl, regex=True)
  356. expected = Series(["bAR", np.nan], dtype=any_string_dtype)
  357. tm.assert_series_equal(result, expected)
  358. def test_replace_compiled_regex(any_string_dtype):
  359. # GH 15446
  360. ser = Series(["fooBAD__barBAD", np.nan], dtype=any_string_dtype)
  361. # test with compiled regex
  362. pat = re.compile(r"BAD_*")
  363. with tm.maybe_produces_warning(
  364. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  365. ):
  366. result = ser.str.replace(pat, "", regex=True)
  367. expected = Series(["foobar", np.nan], dtype=any_string_dtype)
  368. tm.assert_series_equal(result, expected)
  369. with tm.maybe_produces_warning(
  370. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  371. ):
  372. result = ser.str.replace(pat, "", n=1, regex=True)
  373. expected = Series(["foobarBAD", np.nan], dtype=any_string_dtype)
  374. tm.assert_series_equal(result, expected)
  375. def test_replace_compiled_regex_mixed_object():
  376. pat = re.compile(r"BAD_*")
  377. ser = Series(
  378. ["aBAD", np.nan, "bBAD", True, datetime.today(), "fooBAD", None, 1, 2.0]
  379. )
  380. result = Series(ser).str.replace(pat, "", regex=True)
  381. expected = Series(["a", np.nan, "b", np.nan, np.nan, "foo", np.nan, np.nan, np.nan])
  382. tm.assert_series_equal(result, expected)
  383. def test_replace_compiled_regex_unicode(any_string_dtype):
  384. ser = Series([b"abcd,\xc3\xa0".decode("utf-8")], dtype=any_string_dtype)
  385. expected = Series([b"abcd, \xc3\xa0".decode("utf-8")], dtype=any_string_dtype)
  386. pat = re.compile(r"(?<=\w),(?=\w)", flags=re.UNICODE)
  387. with tm.maybe_produces_warning(
  388. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  389. ):
  390. result = ser.str.replace(pat, ", ", regex=True)
  391. tm.assert_series_equal(result, expected)
  392. def test_replace_compiled_regex_raises(any_string_dtype):
  393. # case and flags provided to str.replace will have no effect
  394. # and will produce warnings
  395. ser = Series(["fooBAD__barBAD__bad", np.nan], dtype=any_string_dtype)
  396. pat = re.compile(r"BAD_*")
  397. msg = "case and flags cannot be set when pat is a compiled regex"
  398. with pytest.raises(ValueError, match=msg):
  399. ser.str.replace(pat, "", flags=re.IGNORECASE, regex=True)
  400. with pytest.raises(ValueError, match=msg):
  401. ser.str.replace(pat, "", case=False, regex=True)
  402. with pytest.raises(ValueError, match=msg):
  403. ser.str.replace(pat, "", case=True, regex=True)
  404. def test_replace_compiled_regex_callable(any_string_dtype):
  405. # test with callable
  406. ser = Series(["fooBAD__barBAD", np.nan], dtype=any_string_dtype)
  407. repl = lambda m: m.group(0).swapcase()
  408. pat = re.compile("[a-z][A-Z]{2}")
  409. with tm.maybe_produces_warning(
  410. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  411. ):
  412. result = ser.str.replace(pat, repl, n=2, regex=True)
  413. expected = Series(["foObaD__baRbaD", np.nan], dtype=any_string_dtype)
  414. tm.assert_series_equal(result, expected)
  415. @pytest.mark.parametrize(
  416. "regex,expected", [(True, ["bao", "bao", np.nan]), (False, ["bao", "foo", np.nan])]
  417. )
  418. def test_replace_literal(regex, expected, any_string_dtype):
  419. # GH16808 literal replace (regex=False vs regex=True)
  420. ser = Series(["f.o", "foo", np.nan], dtype=any_string_dtype)
  421. expected = Series(expected, dtype=any_string_dtype)
  422. result = ser.str.replace("f.", "ba", regex=regex)
  423. tm.assert_series_equal(result, expected)
  424. def test_replace_literal_callable_raises(any_string_dtype):
  425. ser = Series([], dtype=any_string_dtype)
  426. repl = lambda m: m.group(0).swapcase()
  427. msg = "Cannot use a callable replacement when regex=False"
  428. with pytest.raises(ValueError, match=msg):
  429. ser.str.replace("abc", repl, regex=False)
  430. def test_replace_literal_compiled_raises(any_string_dtype):
  431. ser = Series([], dtype=any_string_dtype)
  432. pat = re.compile("[a-z][A-Z]{2}")
  433. msg = "Cannot use a compiled regex as replacement pattern with regex=False"
  434. with pytest.raises(ValueError, match=msg):
  435. ser.str.replace(pat, "", regex=False)
  436. def test_replace_moar(any_string_dtype):
  437. # PR #1179
  438. ser = Series(
  439. ["A", "B", "C", "Aaba", "Baca", "", np.nan, "CABA", "dog", "cat"],
  440. dtype=any_string_dtype,
  441. )
  442. result = ser.str.replace("A", "YYY")
  443. expected = Series(
  444. ["YYY", "B", "C", "YYYaba", "Baca", "", np.nan, "CYYYBYYY", "dog", "cat"],
  445. dtype=any_string_dtype,
  446. )
  447. tm.assert_series_equal(result, expected)
  448. with tm.maybe_produces_warning(
  449. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  450. ):
  451. result = ser.str.replace("A", "YYY", case=False)
  452. expected = Series(
  453. [
  454. "YYY",
  455. "B",
  456. "C",
  457. "YYYYYYbYYY",
  458. "BYYYcYYY",
  459. "",
  460. np.nan,
  461. "CYYYBYYY",
  462. "dog",
  463. "cYYYt",
  464. ],
  465. dtype=any_string_dtype,
  466. )
  467. tm.assert_series_equal(result, expected)
  468. with tm.maybe_produces_warning(
  469. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  470. ):
  471. result = ser.str.replace("^.a|dog", "XX-XX ", case=False, regex=True)
  472. expected = Series(
  473. [
  474. "A",
  475. "B",
  476. "C",
  477. "XX-XX ba",
  478. "XX-XX ca",
  479. "",
  480. np.nan,
  481. "XX-XX BA",
  482. "XX-XX ",
  483. "XX-XX t",
  484. ],
  485. dtype=any_string_dtype,
  486. )
  487. tm.assert_series_equal(result, expected)
  488. def test_replace_not_case_sensitive_not_regex(any_string_dtype):
  489. # https://github.com/pandas-dev/pandas/issues/41602
  490. ser = Series(["A.", "a.", "Ab", "ab", np.nan], dtype=any_string_dtype)
  491. with tm.maybe_produces_warning(
  492. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  493. ):
  494. result = ser.str.replace("a", "c", case=False, regex=False)
  495. expected = Series(["c.", "c.", "cb", "cb", np.nan], dtype=any_string_dtype)
  496. tm.assert_series_equal(result, expected)
  497. with tm.maybe_produces_warning(
  498. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  499. ):
  500. result = ser.str.replace("a.", "c.", case=False, regex=False)
  501. expected = Series(["c.", "c.", "Ab", "ab", np.nan], dtype=any_string_dtype)
  502. tm.assert_series_equal(result, expected)
  503. def test_replace_regex(any_string_dtype):
  504. # https://github.com/pandas-dev/pandas/pull/24809
  505. s = Series(["a", "b", "ac", np.nan, ""], dtype=any_string_dtype)
  506. result = s.str.replace("^.$", "a", regex=True)
  507. expected = Series(["a", "a", "ac", np.nan, ""], dtype=any_string_dtype)
  508. tm.assert_series_equal(result, expected)
  509. @pytest.mark.parametrize("regex", [True, False])
  510. def test_replace_regex_single_character(regex, any_string_dtype):
  511. # https://github.com/pandas-dev/pandas/pull/24809, enforced in 2.0
  512. # GH 24804
  513. s = Series(["a.b", ".", "b", np.nan, ""], dtype=any_string_dtype)
  514. result = s.str.replace(".", "a", regex=regex)
  515. if regex:
  516. expected = Series(["aaa", "a", "a", np.nan, ""], dtype=any_string_dtype)
  517. else:
  518. expected = Series(["aab", "a", "b", np.nan, ""], dtype=any_string_dtype)
  519. tm.assert_series_equal(result, expected)
  520. # --------------------------------------------------------------------------------------
  521. # str.match
  522. # --------------------------------------------------------------------------------------
  523. def test_match(any_string_dtype):
  524. # New match behavior introduced in 0.13
  525. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  526. values = Series(["fooBAD__barBAD", np.nan, "foo"], dtype=any_string_dtype)
  527. result = values.str.match(".*(BAD[_]+).*(BAD)")
  528. expected = Series([True, np.nan, False], dtype=expected_dtype)
  529. tm.assert_series_equal(result, expected)
  530. values = Series(
  531. ["fooBAD__barBAD", "BAD_BADleroybrown", np.nan, "foo"], dtype=any_string_dtype
  532. )
  533. result = values.str.match(".*BAD[_]+.*BAD")
  534. expected = Series([True, True, np.nan, False], dtype=expected_dtype)
  535. tm.assert_series_equal(result, expected)
  536. result = values.str.match("BAD[_]+.*BAD")
  537. expected = Series([False, True, np.nan, False], dtype=expected_dtype)
  538. tm.assert_series_equal(result, expected)
  539. values = Series(
  540. ["fooBAD__barBAD", "^BAD_BADleroybrown", np.nan, "foo"], dtype=any_string_dtype
  541. )
  542. result = values.str.match("^BAD[_]+.*BAD")
  543. expected = Series([False, False, np.nan, False], dtype=expected_dtype)
  544. tm.assert_series_equal(result, expected)
  545. result = values.str.match("\\^BAD[_]+.*BAD")
  546. expected = Series([False, True, np.nan, False], dtype=expected_dtype)
  547. tm.assert_series_equal(result, expected)
  548. def test_match_mixed_object():
  549. mixed = Series(
  550. [
  551. "aBAD_BAD",
  552. np.nan,
  553. "BAD_b_BAD",
  554. True,
  555. datetime.today(),
  556. "foo",
  557. None,
  558. 1,
  559. 2.0,
  560. ]
  561. )
  562. result = Series(mixed).str.match(".*(BAD[_]+).*(BAD)")
  563. expected = Series(
  564. [True, np.nan, True, np.nan, np.nan, False, np.nan, np.nan, np.nan]
  565. )
  566. assert isinstance(result, Series)
  567. tm.assert_series_equal(result, expected)
  568. def test_match_na_kwarg(any_string_dtype):
  569. # GH #6609
  570. s = Series(["a", "b", np.nan], dtype=any_string_dtype)
  571. result = s.str.match("a", na=False)
  572. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  573. expected = Series([True, False, False], dtype=expected_dtype)
  574. tm.assert_series_equal(result, expected)
  575. result = s.str.match("a")
  576. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  577. expected = Series([True, False, np.nan], dtype=expected_dtype)
  578. tm.assert_series_equal(result, expected)
  579. def test_match_case_kwarg(any_string_dtype):
  580. values = Series(["ab", "AB", "abc", "ABC"], dtype=any_string_dtype)
  581. with tm.maybe_produces_warning(
  582. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  583. ):
  584. result = values.str.match("ab", case=False)
  585. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  586. expected = Series([True, True, True, True], dtype=expected_dtype)
  587. tm.assert_series_equal(result, expected)
  588. # --------------------------------------------------------------------------------------
  589. # str.fullmatch
  590. # --------------------------------------------------------------------------------------
  591. def test_fullmatch(any_string_dtype):
  592. # GH 32806
  593. ser = Series(
  594. ["fooBAD__barBAD", "BAD_BADleroybrown", np.nan, "foo"], dtype=any_string_dtype
  595. )
  596. result = ser.str.fullmatch(".*BAD[_]+.*BAD")
  597. expected_dtype = "object" if any_string_dtype == "object" else "boolean"
  598. expected = Series([True, False, np.nan, False], dtype=expected_dtype)
  599. tm.assert_series_equal(result, expected)
  600. def test_fullmatch_na_kwarg(any_string_dtype):
  601. ser = Series(
  602. ["fooBAD__barBAD", "BAD_BADleroybrown", np.nan, "foo"], dtype=any_string_dtype
  603. )
  604. result = ser.str.fullmatch(".*BAD[_]+.*BAD", na=False)
  605. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  606. expected = Series([True, False, False, False], dtype=expected_dtype)
  607. tm.assert_series_equal(result, expected)
  608. def test_fullmatch_case_kwarg(any_string_dtype):
  609. ser = Series(["ab", "AB", "abc", "ABC"], dtype=any_string_dtype)
  610. expected_dtype = np.bool_ if any_string_dtype == "object" else "boolean"
  611. expected = Series([True, False, False, False], dtype=expected_dtype)
  612. result = ser.str.fullmatch("ab", case=True)
  613. tm.assert_series_equal(result, expected)
  614. expected = Series([True, True, False, False], dtype=expected_dtype)
  615. with tm.maybe_produces_warning(
  616. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  617. ):
  618. result = ser.str.fullmatch("ab", case=False)
  619. tm.assert_series_equal(result, expected)
  620. with tm.maybe_produces_warning(
  621. PerformanceWarning, any_string_dtype == "string[pyarrow]"
  622. ):
  623. result = ser.str.fullmatch("ab", flags=re.IGNORECASE)
  624. tm.assert_series_equal(result, expected)
  625. # --------------------------------------------------------------------------------------
  626. # str.findall
  627. # --------------------------------------------------------------------------------------
  628. def test_findall(any_string_dtype):
  629. ser = Series(["fooBAD__barBAD", np.nan, "foo", "BAD"], dtype=any_string_dtype)
  630. result = ser.str.findall("BAD[_]*")
  631. expected = Series([["BAD__", "BAD"], np.nan, [], ["BAD"]])
  632. tm.assert_series_equal(result, expected)
  633. def test_findall_mixed_object():
  634. ser = Series(
  635. [
  636. "fooBAD__barBAD",
  637. np.nan,
  638. "foo",
  639. True,
  640. datetime.today(),
  641. "BAD",
  642. None,
  643. 1,
  644. 2.0,
  645. ]
  646. )
  647. result = ser.str.findall("BAD[_]*")
  648. expected = Series(
  649. [
  650. ["BAD__", "BAD"],
  651. np.nan,
  652. [],
  653. np.nan,
  654. np.nan,
  655. ["BAD"],
  656. np.nan,
  657. np.nan,
  658. np.nan,
  659. ]
  660. )
  661. tm.assert_series_equal(result, expected)
  662. # --------------------------------------------------------------------------------------
  663. # str.find
  664. # --------------------------------------------------------------------------------------
  665. def test_find(any_string_dtype):
  666. ser = Series(
  667. ["ABCDEFG", "BCDEFEF", "DEFGHIJEF", "EFGHEF", "XXXX"], dtype=any_string_dtype
  668. )
  669. expected_dtype = np.int64 if any_string_dtype == "object" else "Int64"
  670. result = ser.str.find("EF")
  671. expected = Series([4, 3, 1, 0, -1], dtype=expected_dtype)
  672. tm.assert_series_equal(result, expected)
  673. expected = np.array([v.find("EF") for v in np.array(ser)], dtype=np.int64)
  674. tm.assert_numpy_array_equal(np.array(result, dtype=np.int64), expected)
  675. result = ser.str.rfind("EF")
  676. expected = Series([4, 5, 7, 4, -1], dtype=expected_dtype)
  677. tm.assert_series_equal(result, expected)
  678. expected = np.array([v.rfind("EF") for v in np.array(ser)], dtype=np.int64)
  679. tm.assert_numpy_array_equal(np.array(result, dtype=np.int64), expected)
  680. result = ser.str.find("EF", 3)
  681. expected = Series([4, 3, 7, 4, -1], dtype=expected_dtype)
  682. tm.assert_series_equal(result, expected)
  683. expected = np.array([v.find("EF", 3) for v in np.array(ser)], dtype=np.int64)
  684. tm.assert_numpy_array_equal(np.array(result, dtype=np.int64), expected)
  685. result = ser.str.rfind("EF", 3)
  686. expected = Series([4, 5, 7, 4, -1], dtype=expected_dtype)
  687. tm.assert_series_equal(result, expected)
  688. expected = np.array([v.rfind("EF", 3) for v in np.array(ser)], dtype=np.int64)
  689. tm.assert_numpy_array_equal(np.array(result, dtype=np.int64), expected)
  690. result = ser.str.find("EF", 3, 6)
  691. expected = Series([4, 3, -1, 4, -1], dtype=expected_dtype)
  692. tm.assert_series_equal(result, expected)
  693. expected = np.array([v.find("EF", 3, 6) for v in np.array(ser)], dtype=np.int64)
  694. tm.assert_numpy_array_equal(np.array(result, dtype=np.int64), expected)
  695. result = ser.str.rfind("EF", 3, 6)
  696. expected = Series([4, 3, -1, 4, -1], dtype=expected_dtype)
  697. tm.assert_series_equal(result, expected)
  698. expected = np.array([v.rfind("EF", 3, 6) for v in np.array(ser)], dtype=np.int64)
  699. tm.assert_numpy_array_equal(np.array(result, dtype=np.int64), expected)
  700. def test_find_bad_arg_raises(any_string_dtype):
  701. ser = Series([], dtype=any_string_dtype)
  702. with pytest.raises(TypeError, match="expected a string object, not int"):
  703. ser.str.find(0)
  704. with pytest.raises(TypeError, match="expected a string object, not int"):
  705. ser.str.rfind(0)
  706. def test_find_nan(any_string_dtype):
  707. ser = Series(
  708. ["ABCDEFG", np.nan, "DEFGHIJEF", np.nan, "XXXX"], dtype=any_string_dtype
  709. )
  710. expected_dtype = np.float64 if any_string_dtype == "object" else "Int64"
  711. result = ser.str.find("EF")
  712. expected = Series([4, np.nan, 1, np.nan, -1], dtype=expected_dtype)
  713. tm.assert_series_equal(result, expected)
  714. result = ser.str.rfind("EF")
  715. expected = Series([4, np.nan, 7, np.nan, -1], dtype=expected_dtype)
  716. tm.assert_series_equal(result, expected)
  717. result = ser.str.find("EF", 3)
  718. expected = Series([4, np.nan, 7, np.nan, -1], dtype=expected_dtype)
  719. tm.assert_series_equal(result, expected)
  720. result = ser.str.rfind("EF", 3)
  721. expected = Series([4, np.nan, 7, np.nan, -1], dtype=expected_dtype)
  722. tm.assert_series_equal(result, expected)
  723. result = ser.str.find("EF", 3, 6)
  724. expected = Series([4, np.nan, -1, np.nan, -1], dtype=expected_dtype)
  725. tm.assert_series_equal(result, expected)
  726. result = ser.str.rfind("EF", 3, 6)
  727. expected = Series([4, np.nan, -1, np.nan, -1], dtype=expected_dtype)
  728. tm.assert_series_equal(result, expected)
  729. # --------------------------------------------------------------------------------------
  730. # str.translate
  731. # --------------------------------------------------------------------------------------
  732. def test_translate(index_or_series, any_string_dtype):
  733. obj = index_or_series(
  734. ["abcdefg", "abcc", "cdddfg", "cdefggg"], dtype=any_string_dtype
  735. )
  736. table = str.maketrans("abc", "cde")
  737. result = obj.str.translate(table)
  738. expected = index_or_series(
  739. ["cdedefg", "cdee", "edddfg", "edefggg"], dtype=any_string_dtype
  740. )
  741. tm.assert_equal(result, expected)
  742. def test_translate_mixed_object():
  743. # Series with non-string values
  744. s = Series(["a", "b", "c", 1.2])
  745. table = str.maketrans("abc", "cde")
  746. expected = Series(["c", "d", "e", np.nan])
  747. result = s.str.translate(table)
  748. tm.assert_series_equal(result, expected)
  749. # --------------------------------------------------------------------------------------
  750. def test_flags_kwarg(any_string_dtype):
  751. data = {
  752. "Dave": "dave@google.com",
  753. "Steve": "steve@gmail.com",
  754. "Rob": "rob@gmail.com",
  755. "Wes": np.nan,
  756. }
  757. data = Series(data, dtype=any_string_dtype)
  758. pat = r"([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\.([A-Z]{2,4})"
  759. using_pyarrow = any_string_dtype == "string[pyarrow]"
  760. result = data.str.extract(pat, flags=re.IGNORECASE, expand=True)
  761. assert result.iloc[0].tolist() == ["dave", "google", "com"]
  762. with tm.maybe_produces_warning(PerformanceWarning, using_pyarrow):
  763. result = data.str.match(pat, flags=re.IGNORECASE)
  764. assert result[0]
  765. with tm.maybe_produces_warning(PerformanceWarning, using_pyarrow):
  766. result = data.str.fullmatch(pat, flags=re.IGNORECASE)
  767. assert result[0]
  768. result = data.str.findall(pat, flags=re.IGNORECASE)
  769. assert result[0][0] == ("dave", "google", "com")
  770. result = data.str.count(pat, flags=re.IGNORECASE)
  771. assert result[0] == 1
  772. msg = "has match groups"
  773. with tm.assert_produces_warning(
  774. UserWarning, match=msg, raise_on_extra_warnings=not using_pyarrow
  775. ):
  776. result = data.str.contains(pat, flags=re.IGNORECASE)
  777. assert result[0]