test_quantile.py 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989
  1. import numpy as np
  2. import pytest
  3. from pandas.compat.numpy import (
  4. np_percentile_argname,
  5. np_version_under1p21,
  6. )
  7. import pandas as pd
  8. from pandas import (
  9. DataFrame,
  10. Index,
  11. Series,
  12. Timestamp,
  13. )
  14. import pandas._testing as tm
  15. @pytest.fixture(
  16. params=[["linear", "single"], ["nearest", "table"]], ids=lambda x: "-".join(x)
  17. )
  18. def interp_method(request):
  19. """(interpolation, method) arguments for quantile"""
  20. return request.param
  21. class TestDataFrameQuantile:
  22. @pytest.mark.parametrize(
  23. "df,expected",
  24. [
  25. [
  26. DataFrame(
  27. {
  28. 0: Series(pd.arrays.SparseArray([1, 2])),
  29. 1: Series(pd.arrays.SparseArray([3, 4])),
  30. }
  31. ),
  32. Series([1.5, 3.5], name=0.5),
  33. ],
  34. [
  35. DataFrame(Series([0.0, None, 1.0, 2.0], dtype="Sparse[float]")),
  36. Series([1.0], name=0.5),
  37. ],
  38. ],
  39. )
  40. def test_quantile_sparse(self, df, expected):
  41. # GH#17198
  42. # GH#24600
  43. result = df.quantile()
  44. expected = expected.astype("Sparse[float]")
  45. tm.assert_series_equal(result, expected)
  46. def test_quantile(
  47. self, datetime_frame, interp_method, using_array_manager, request
  48. ):
  49. interpolation, method = interp_method
  50. df = datetime_frame
  51. result = df.quantile(
  52. 0.1, axis=0, numeric_only=True, interpolation=interpolation, method=method
  53. )
  54. expected = Series(
  55. [np.percentile(df[col], 10) for col in df.columns],
  56. index=df.columns,
  57. name=0.1,
  58. )
  59. if interpolation == "linear":
  60. # np.percentile values only comparable to linear interpolation
  61. tm.assert_series_equal(result, expected)
  62. else:
  63. tm.assert_index_equal(result.index, expected.index)
  64. request.node.add_marker(
  65. pytest.mark.xfail(
  66. using_array_manager, reason="Name set incorrectly for arraymanager"
  67. )
  68. )
  69. assert result.name == expected.name
  70. result = df.quantile(
  71. 0.9, axis=1, numeric_only=True, interpolation=interpolation, method=method
  72. )
  73. expected = Series(
  74. [np.percentile(df.loc[date], 90) for date in df.index],
  75. index=df.index,
  76. name=0.9,
  77. )
  78. if interpolation == "linear":
  79. # np.percentile values only comparable to linear interpolation
  80. tm.assert_series_equal(result, expected)
  81. else:
  82. tm.assert_index_equal(result.index, expected.index)
  83. request.node.add_marker(
  84. pytest.mark.xfail(
  85. using_array_manager, reason="Name set incorrectly for arraymanager"
  86. )
  87. )
  88. assert result.name == expected.name
  89. def test_empty(self, interp_method):
  90. interpolation, method = interp_method
  91. q = DataFrame({"x": [], "y": []}).quantile(
  92. 0.1, axis=0, numeric_only=True, interpolation=interpolation, method=method
  93. )
  94. assert np.isnan(q["x"]) and np.isnan(q["y"])
  95. def test_non_numeric_exclusion(self, interp_method, request, using_array_manager):
  96. interpolation, method = interp_method
  97. df = DataFrame({"col1": ["A", "A", "B", "B"], "col2": [1, 2, 3, 4]})
  98. rs = df.quantile(
  99. 0.5, numeric_only=True, interpolation=interpolation, method=method
  100. )
  101. xp = df.median(numeric_only=True).rename(0.5)
  102. if interpolation == "nearest":
  103. xp = (xp + 0.5).astype(np.int64)
  104. if method == "table" and using_array_manager:
  105. request.node.add_marker(
  106. pytest.mark.xfail(reason="Axis name incorrectly set.")
  107. )
  108. tm.assert_series_equal(rs, xp)
  109. def test_axis(self, interp_method, request, using_array_manager):
  110. # axis
  111. interpolation, method = interp_method
  112. df = DataFrame({"A": [1, 2, 3], "B": [2, 3, 4]}, index=[1, 2, 3])
  113. result = df.quantile(0.5, axis=1, interpolation=interpolation, method=method)
  114. expected = Series([1.5, 2.5, 3.5], index=[1, 2, 3], name=0.5)
  115. if interpolation == "nearest":
  116. expected = expected.astype(np.int64)
  117. if method == "table" and using_array_manager:
  118. request.node.add_marker(
  119. pytest.mark.xfail(reason="Axis name incorrectly set.")
  120. )
  121. tm.assert_series_equal(result, expected)
  122. result = df.quantile(
  123. [0.5, 0.75], axis=1, interpolation=interpolation, method=method
  124. )
  125. expected = DataFrame(
  126. {1: [1.5, 1.75], 2: [2.5, 2.75], 3: [3.5, 3.75]}, index=[0.5, 0.75]
  127. )
  128. if interpolation == "nearest":
  129. expected.iloc[0, :] -= 0.5
  130. expected.iloc[1, :] += 0.25
  131. expected = expected.astype(np.int64)
  132. tm.assert_frame_equal(result, expected, check_index_type=True)
  133. def test_axis_numeric_only_true(self, interp_method, request, using_array_manager):
  134. # We may want to break API in the future to change this
  135. # so that we exclude non-numeric along the same axis
  136. # See GH #7312
  137. interpolation, method = interp_method
  138. df = DataFrame([[1, 2, 3], ["a", "b", 4]])
  139. result = df.quantile(
  140. 0.5, axis=1, numeric_only=True, interpolation=interpolation, method=method
  141. )
  142. expected = Series([3.0, 4.0], index=[0, 1], name=0.5)
  143. if interpolation == "nearest":
  144. expected = expected.astype(np.int64)
  145. if method == "table" and using_array_manager:
  146. request.node.add_marker(
  147. pytest.mark.xfail(reason="Axis name incorrectly set.")
  148. )
  149. tm.assert_series_equal(result, expected)
  150. def test_quantile_date_range(self, interp_method, request, using_array_manager):
  151. # GH 2460
  152. interpolation, method = interp_method
  153. dti = pd.date_range("2016-01-01", periods=3, tz="US/Pacific")
  154. ser = Series(dti)
  155. df = DataFrame(ser)
  156. result = df.quantile(
  157. numeric_only=False, interpolation=interpolation, method=method
  158. )
  159. expected = Series(
  160. ["2016-01-02 00:00:00"], name=0.5, dtype="datetime64[ns, US/Pacific]"
  161. )
  162. if method == "table" and using_array_manager:
  163. request.node.add_marker(
  164. pytest.mark.xfail(reason="Axis name incorrectly set.")
  165. )
  166. tm.assert_series_equal(result, expected)
  167. def test_quantile_axis_mixed(self, interp_method, request, using_array_manager):
  168. # mixed on axis=1
  169. interpolation, method = interp_method
  170. df = DataFrame(
  171. {
  172. "A": [1, 2, 3],
  173. "B": [2.0, 3.0, 4.0],
  174. "C": pd.date_range("20130101", periods=3),
  175. "D": ["foo", "bar", "baz"],
  176. }
  177. )
  178. result = df.quantile(
  179. 0.5, axis=1, numeric_only=True, interpolation=interpolation, method=method
  180. )
  181. expected = Series([1.5, 2.5, 3.5], name=0.5)
  182. if interpolation == "nearest":
  183. expected -= 0.5
  184. if method == "table" and using_array_manager:
  185. request.node.add_marker(
  186. pytest.mark.xfail(reason="Axis name incorrectly set.")
  187. )
  188. tm.assert_series_equal(result, expected)
  189. # must raise
  190. msg = "'<' not supported between instances of 'Timestamp' and 'float'"
  191. with pytest.raises(TypeError, match=msg):
  192. df.quantile(0.5, axis=1, numeric_only=False)
  193. def test_quantile_axis_parameter(self, interp_method, request, using_array_manager):
  194. # GH 9543/9544
  195. interpolation, method = interp_method
  196. if method == "table" and using_array_manager:
  197. request.node.add_marker(
  198. pytest.mark.xfail(reason="Axis name incorrectly set.")
  199. )
  200. df = DataFrame({"A": [1, 2, 3], "B": [2, 3, 4]}, index=[1, 2, 3])
  201. result = df.quantile(0.5, axis=0, interpolation=interpolation, method=method)
  202. expected = Series([2.0, 3.0], index=["A", "B"], name=0.5)
  203. if interpolation == "nearest":
  204. expected = expected.astype(np.int64)
  205. tm.assert_series_equal(result, expected)
  206. expected = df.quantile(
  207. 0.5, axis="index", interpolation=interpolation, method=method
  208. )
  209. if interpolation == "nearest":
  210. expected = expected.astype(np.int64)
  211. tm.assert_series_equal(result, expected)
  212. result = df.quantile(0.5, axis=1, interpolation=interpolation, method=method)
  213. expected = Series([1.5, 2.5, 3.5], index=[1, 2, 3], name=0.5)
  214. if interpolation == "nearest":
  215. expected = expected.astype(np.int64)
  216. tm.assert_series_equal(result, expected)
  217. result = df.quantile(
  218. 0.5, axis="columns", interpolation=interpolation, method=method
  219. )
  220. tm.assert_series_equal(result, expected)
  221. msg = "No axis named -1 for object type DataFrame"
  222. with pytest.raises(ValueError, match=msg):
  223. df.quantile(0.1, axis=-1, interpolation=interpolation, method=method)
  224. msg = "No axis named column for object type DataFrame"
  225. with pytest.raises(ValueError, match=msg):
  226. df.quantile(0.1, axis="column")
  227. def test_quantile_interpolation(self):
  228. # see gh-10174
  229. # interpolation method other than default linear
  230. df = DataFrame({"A": [1, 2, 3], "B": [2, 3, 4]}, index=[1, 2, 3])
  231. result = df.quantile(0.5, axis=1, interpolation="nearest")
  232. expected = Series([1, 2, 3], index=[1, 2, 3], name=0.5)
  233. tm.assert_series_equal(result, expected)
  234. # cross-check interpolation=nearest results in original dtype
  235. exp = np.percentile(
  236. np.array([[1, 2, 3], [2, 3, 4]]),
  237. 0.5,
  238. axis=0,
  239. **{np_percentile_argname: "nearest"},
  240. )
  241. expected = Series(exp, index=[1, 2, 3], name=0.5, dtype="int64")
  242. tm.assert_series_equal(result, expected)
  243. # float
  244. df = DataFrame({"A": [1.0, 2.0, 3.0], "B": [2.0, 3.0, 4.0]}, index=[1, 2, 3])
  245. result = df.quantile(0.5, axis=1, interpolation="nearest")
  246. expected = Series([1.0, 2.0, 3.0], index=[1, 2, 3], name=0.5)
  247. tm.assert_series_equal(result, expected)
  248. exp = np.percentile(
  249. np.array([[1.0, 2.0, 3.0], [2.0, 3.0, 4.0]]),
  250. 0.5,
  251. axis=0,
  252. **{np_percentile_argname: "nearest"},
  253. )
  254. expected = Series(exp, index=[1, 2, 3], name=0.5, dtype="float64")
  255. tm.assert_series_equal(result, expected)
  256. # axis
  257. result = df.quantile([0.5, 0.75], axis=1, interpolation="lower")
  258. expected = DataFrame(
  259. {1: [1.0, 1.0], 2: [2.0, 2.0], 3: [3.0, 3.0]}, index=[0.5, 0.75]
  260. )
  261. tm.assert_frame_equal(result, expected)
  262. # test degenerate case
  263. df = DataFrame({"x": [], "y": []})
  264. q = df.quantile(0.1, axis=0, interpolation="higher")
  265. assert np.isnan(q["x"]) and np.isnan(q["y"])
  266. # multi
  267. df = DataFrame([[1, 1, 1], [2, 2, 2], [3, 3, 3]], columns=["a", "b", "c"])
  268. result = df.quantile([0.25, 0.5], interpolation="midpoint")
  269. # https://github.com/numpy/numpy/issues/7163
  270. expected = DataFrame(
  271. [[1.5, 1.5, 1.5], [2.0, 2.0, 2.0]],
  272. index=[0.25, 0.5],
  273. columns=["a", "b", "c"],
  274. )
  275. tm.assert_frame_equal(result, expected)
  276. def test_quantile_interpolation_datetime(self, datetime_frame):
  277. # see gh-10174
  278. # interpolation = linear (default case)
  279. df = datetime_frame
  280. q = df.quantile(0.1, axis=0, numeric_only=True, interpolation="linear")
  281. assert q["A"] == np.percentile(df["A"], 10)
  282. def test_quantile_interpolation_int(self, int_frame):
  283. # see gh-10174
  284. df = int_frame
  285. # interpolation = linear (default case)
  286. q = df.quantile(0.1)
  287. assert q["A"] == np.percentile(df["A"], 10)
  288. # test with and without interpolation keyword
  289. q1 = df.quantile(0.1, axis=0, interpolation="linear")
  290. assert q1["A"] == np.percentile(df["A"], 10)
  291. tm.assert_series_equal(q, q1)
  292. def test_quantile_multi(self, interp_method, request, using_array_manager):
  293. interpolation, method = interp_method
  294. df = DataFrame([[1, 1, 1], [2, 2, 2], [3, 3, 3]], columns=["a", "b", "c"])
  295. result = df.quantile([0.25, 0.5], interpolation=interpolation, method=method)
  296. expected = DataFrame(
  297. [[1.5, 1.5, 1.5], [2.0, 2.0, 2.0]],
  298. index=[0.25, 0.5],
  299. columns=["a", "b", "c"],
  300. )
  301. if interpolation == "nearest":
  302. expected = expected.astype(np.int64)
  303. if method == "table" and using_array_manager:
  304. request.node.add_marker(
  305. pytest.mark.xfail(reason="Axis name incorrectly set.")
  306. )
  307. tm.assert_frame_equal(result, expected)
  308. def test_quantile_multi_axis_1(self, interp_method, request, using_array_manager):
  309. interpolation, method = interp_method
  310. df = DataFrame([[1, 1, 1], [2, 2, 2], [3, 3, 3]], columns=["a", "b", "c"])
  311. result = df.quantile(
  312. [0.25, 0.5], axis=1, interpolation=interpolation, method=method
  313. )
  314. expected = DataFrame(
  315. [[1.0, 2.0, 3.0]] * 2, index=[0.25, 0.5], columns=[0, 1, 2]
  316. )
  317. if interpolation == "nearest":
  318. expected = expected.astype(np.int64)
  319. if method == "table" and using_array_manager:
  320. request.node.add_marker(
  321. pytest.mark.xfail(reason="Axis name incorrectly set.")
  322. )
  323. tm.assert_frame_equal(result, expected)
  324. def test_quantile_multi_empty(self, interp_method):
  325. interpolation, method = interp_method
  326. result = DataFrame({"x": [], "y": []}).quantile(
  327. [0.1, 0.9], axis=0, interpolation=interpolation, method=method
  328. )
  329. expected = DataFrame(
  330. {"x": [np.nan, np.nan], "y": [np.nan, np.nan]}, index=[0.1, 0.9]
  331. )
  332. tm.assert_frame_equal(result, expected)
  333. def test_quantile_datetime(self):
  334. df = DataFrame({"a": pd.to_datetime(["2010", "2011"]), "b": [0, 5]})
  335. # exclude datetime
  336. result = df.quantile(0.5, numeric_only=True)
  337. expected = Series([2.5], index=["b"], name=0.5)
  338. tm.assert_series_equal(result, expected)
  339. # datetime
  340. result = df.quantile(0.5, numeric_only=False)
  341. expected = Series(
  342. [Timestamp("2010-07-02 12:00:00"), 2.5], index=["a", "b"], name=0.5
  343. )
  344. tm.assert_series_equal(result, expected)
  345. # datetime w/ multi
  346. result = df.quantile([0.5], numeric_only=False)
  347. expected = DataFrame(
  348. [[Timestamp("2010-07-02 12:00:00"), 2.5]], index=[0.5], columns=["a", "b"]
  349. )
  350. tm.assert_frame_equal(result, expected)
  351. # axis = 1
  352. df["c"] = pd.to_datetime(["2011", "2012"])
  353. result = df[["a", "c"]].quantile(0.5, axis=1, numeric_only=False)
  354. expected = Series(
  355. [Timestamp("2010-07-02 12:00:00"), Timestamp("2011-07-02 12:00:00")],
  356. index=[0, 1],
  357. name=0.5,
  358. )
  359. tm.assert_series_equal(result, expected)
  360. result = df[["a", "c"]].quantile([0.5], axis=1, numeric_only=False)
  361. expected = DataFrame(
  362. [[Timestamp("2010-07-02 12:00:00"), Timestamp("2011-07-02 12:00:00")]],
  363. index=[0.5],
  364. columns=[0, 1],
  365. )
  366. tm.assert_frame_equal(result, expected)
  367. # empty when numeric_only=True
  368. result = df[["a", "c"]].quantile(0.5, numeric_only=True)
  369. expected = Series([], index=[], dtype=np.float64, name=0.5)
  370. tm.assert_series_equal(result, expected)
  371. result = df[["a", "c"]].quantile([0.5], numeric_only=True)
  372. expected = DataFrame(index=[0.5], columns=[])
  373. tm.assert_frame_equal(result, expected)
  374. @pytest.mark.parametrize(
  375. "dtype",
  376. [
  377. "datetime64[ns]",
  378. "datetime64[ns, US/Pacific]",
  379. "timedelta64[ns]",
  380. "Period[D]",
  381. ],
  382. )
  383. def test_quantile_dt64_empty(self, dtype, interp_method):
  384. # GH#41544
  385. interpolation, method = interp_method
  386. df = DataFrame(columns=["a", "b"], dtype=dtype)
  387. res = df.quantile(
  388. 0.5, axis=1, numeric_only=False, interpolation=interpolation, method=method
  389. )
  390. expected = Series([], index=[], name=0.5, dtype=dtype)
  391. tm.assert_series_equal(res, expected)
  392. # no columns in result, so no dtype preservation
  393. res = df.quantile(
  394. [0.5],
  395. axis=1,
  396. numeric_only=False,
  397. interpolation=interpolation,
  398. method=method,
  399. )
  400. expected = DataFrame(index=[0.5], columns=[])
  401. tm.assert_frame_equal(res, expected)
  402. @pytest.mark.parametrize("invalid", [-1, 2, [0.5, -1], [0.5, 2]])
  403. def test_quantile_invalid(self, invalid, datetime_frame, interp_method):
  404. msg = "percentiles should all be in the interval \\[0, 1\\]"
  405. interpolation, method = interp_method
  406. with pytest.raises(ValueError, match=msg):
  407. datetime_frame.quantile(invalid, interpolation=interpolation, method=method)
  408. def test_quantile_box(self, interp_method, request, using_array_manager):
  409. interpolation, method = interp_method
  410. if method == "table" and using_array_manager:
  411. request.node.add_marker(
  412. pytest.mark.xfail(reason="Axis name incorrectly set.")
  413. )
  414. df = DataFrame(
  415. {
  416. "A": [
  417. Timestamp("2011-01-01"),
  418. Timestamp("2011-01-02"),
  419. Timestamp("2011-01-03"),
  420. ],
  421. "B": [
  422. Timestamp("2011-01-01", tz="US/Eastern"),
  423. Timestamp("2011-01-02", tz="US/Eastern"),
  424. Timestamp("2011-01-03", tz="US/Eastern"),
  425. ],
  426. "C": [
  427. pd.Timedelta("1 days"),
  428. pd.Timedelta("2 days"),
  429. pd.Timedelta("3 days"),
  430. ],
  431. }
  432. )
  433. res = df.quantile(
  434. 0.5, numeric_only=False, interpolation=interpolation, method=method
  435. )
  436. exp = Series(
  437. [
  438. Timestamp("2011-01-02"),
  439. Timestamp("2011-01-02", tz="US/Eastern"),
  440. pd.Timedelta("2 days"),
  441. ],
  442. name=0.5,
  443. index=["A", "B", "C"],
  444. )
  445. tm.assert_series_equal(res, exp)
  446. res = df.quantile(
  447. [0.5], numeric_only=False, interpolation=interpolation, method=method
  448. )
  449. exp = DataFrame(
  450. [
  451. [
  452. Timestamp("2011-01-02"),
  453. Timestamp("2011-01-02", tz="US/Eastern"),
  454. pd.Timedelta("2 days"),
  455. ]
  456. ],
  457. index=[0.5],
  458. columns=["A", "B", "C"],
  459. )
  460. tm.assert_frame_equal(res, exp)
  461. def test_quantile_box_nat(self):
  462. # DatetimeLikeBlock may be consolidated and contain NaT in different loc
  463. df = DataFrame(
  464. {
  465. "A": [
  466. Timestamp("2011-01-01"),
  467. pd.NaT,
  468. Timestamp("2011-01-02"),
  469. Timestamp("2011-01-03"),
  470. ],
  471. "a": [
  472. Timestamp("2011-01-01"),
  473. Timestamp("2011-01-02"),
  474. pd.NaT,
  475. Timestamp("2011-01-03"),
  476. ],
  477. "B": [
  478. Timestamp("2011-01-01", tz="US/Eastern"),
  479. pd.NaT,
  480. Timestamp("2011-01-02", tz="US/Eastern"),
  481. Timestamp("2011-01-03", tz="US/Eastern"),
  482. ],
  483. "b": [
  484. Timestamp("2011-01-01", tz="US/Eastern"),
  485. Timestamp("2011-01-02", tz="US/Eastern"),
  486. pd.NaT,
  487. Timestamp("2011-01-03", tz="US/Eastern"),
  488. ],
  489. "C": [
  490. pd.Timedelta("1 days"),
  491. pd.Timedelta("2 days"),
  492. pd.Timedelta("3 days"),
  493. pd.NaT,
  494. ],
  495. "c": [
  496. pd.NaT,
  497. pd.Timedelta("1 days"),
  498. pd.Timedelta("2 days"),
  499. pd.Timedelta("3 days"),
  500. ],
  501. },
  502. columns=list("AaBbCc"),
  503. )
  504. res = df.quantile(0.5, numeric_only=False)
  505. exp = Series(
  506. [
  507. Timestamp("2011-01-02"),
  508. Timestamp("2011-01-02"),
  509. Timestamp("2011-01-02", tz="US/Eastern"),
  510. Timestamp("2011-01-02", tz="US/Eastern"),
  511. pd.Timedelta("2 days"),
  512. pd.Timedelta("2 days"),
  513. ],
  514. name=0.5,
  515. index=list("AaBbCc"),
  516. )
  517. tm.assert_series_equal(res, exp)
  518. res = df.quantile([0.5], numeric_only=False)
  519. exp = DataFrame(
  520. [
  521. [
  522. Timestamp("2011-01-02"),
  523. Timestamp("2011-01-02"),
  524. Timestamp("2011-01-02", tz="US/Eastern"),
  525. Timestamp("2011-01-02", tz="US/Eastern"),
  526. pd.Timedelta("2 days"),
  527. pd.Timedelta("2 days"),
  528. ]
  529. ],
  530. index=[0.5],
  531. columns=list("AaBbCc"),
  532. )
  533. tm.assert_frame_equal(res, exp)
  534. def test_quantile_nan(self, interp_method, request, using_array_manager):
  535. interpolation, method = interp_method
  536. if method == "table" and using_array_manager:
  537. request.node.add_marker(
  538. pytest.mark.xfail(reason="Axis name incorrectly set.")
  539. )
  540. # GH 14357 - float block where some cols have missing values
  541. df = DataFrame({"a": np.arange(1, 6.0), "b": np.arange(1, 6.0)})
  542. df.iloc[-1, 1] = np.nan
  543. res = df.quantile(0.5, interpolation=interpolation, method=method)
  544. exp = Series(
  545. [3.0, 2.5 if interpolation == "linear" else 3.0], index=["a", "b"], name=0.5
  546. )
  547. tm.assert_series_equal(res, exp)
  548. res = df.quantile([0.5, 0.75], interpolation=interpolation, method=method)
  549. exp = DataFrame(
  550. {
  551. "a": [3.0, 4.0],
  552. "b": [2.5, 3.25] if interpolation == "linear" else [3.0, 4.0],
  553. },
  554. index=[0.5, 0.75],
  555. )
  556. tm.assert_frame_equal(res, exp)
  557. res = df.quantile(0.5, axis=1, interpolation=interpolation, method=method)
  558. exp = Series(np.arange(1.0, 6.0), name=0.5)
  559. tm.assert_series_equal(res, exp)
  560. res = df.quantile(
  561. [0.5, 0.75], axis=1, interpolation=interpolation, method=method
  562. )
  563. exp = DataFrame([np.arange(1.0, 6.0)] * 2, index=[0.5, 0.75])
  564. if interpolation == "nearest":
  565. exp.iloc[1, -1] = np.nan
  566. tm.assert_frame_equal(res, exp)
  567. # full-nan column
  568. df["b"] = np.nan
  569. res = df.quantile(0.5, interpolation=interpolation, method=method)
  570. exp = Series([3.0, np.nan], index=["a", "b"], name=0.5)
  571. tm.assert_series_equal(res, exp)
  572. res = df.quantile([0.5, 0.75], interpolation=interpolation, method=method)
  573. exp = DataFrame({"a": [3.0, 4.0], "b": [np.nan, np.nan]}, index=[0.5, 0.75])
  574. tm.assert_frame_equal(res, exp)
  575. def test_quantile_nat(self, interp_method, request, using_array_manager):
  576. interpolation, method = interp_method
  577. if method == "table" and using_array_manager:
  578. request.node.add_marker(
  579. pytest.mark.xfail(reason="Axis name incorrectly set.")
  580. )
  581. # full NaT column
  582. df = DataFrame({"a": [pd.NaT, pd.NaT, pd.NaT]})
  583. res = df.quantile(
  584. 0.5, numeric_only=False, interpolation=interpolation, method=method
  585. )
  586. exp = Series([pd.NaT], index=["a"], name=0.5)
  587. tm.assert_series_equal(res, exp)
  588. res = df.quantile(
  589. [0.5], numeric_only=False, interpolation=interpolation, method=method
  590. )
  591. exp = DataFrame({"a": [pd.NaT]}, index=[0.5])
  592. tm.assert_frame_equal(res, exp)
  593. # mixed non-null / full null column
  594. df = DataFrame(
  595. {
  596. "a": [
  597. Timestamp("2012-01-01"),
  598. Timestamp("2012-01-02"),
  599. Timestamp("2012-01-03"),
  600. ],
  601. "b": [pd.NaT, pd.NaT, pd.NaT],
  602. }
  603. )
  604. res = df.quantile(
  605. 0.5, numeric_only=False, interpolation=interpolation, method=method
  606. )
  607. exp = Series([Timestamp("2012-01-02"), pd.NaT], index=["a", "b"], name=0.5)
  608. tm.assert_series_equal(res, exp)
  609. res = df.quantile(
  610. [0.5], numeric_only=False, interpolation=interpolation, method=method
  611. )
  612. exp = DataFrame(
  613. [[Timestamp("2012-01-02"), pd.NaT]], index=[0.5], columns=["a", "b"]
  614. )
  615. tm.assert_frame_equal(res, exp)
  616. def test_quantile_empty_no_rows_floats(self, interp_method):
  617. interpolation, method = interp_method
  618. df = DataFrame(columns=["a", "b"], dtype="float64")
  619. res = df.quantile(0.5, interpolation=interpolation, method=method)
  620. exp = Series([np.nan, np.nan], index=["a", "b"], name=0.5)
  621. tm.assert_series_equal(res, exp)
  622. res = df.quantile([0.5], interpolation=interpolation, method=method)
  623. exp = DataFrame([[np.nan, np.nan]], columns=["a", "b"], index=[0.5])
  624. tm.assert_frame_equal(res, exp)
  625. res = df.quantile(0.5, axis=1, interpolation=interpolation, method=method)
  626. exp = Series([], index=[], dtype="float64", name=0.5)
  627. tm.assert_series_equal(res, exp)
  628. res = df.quantile([0.5], axis=1, interpolation=interpolation, method=method)
  629. exp = DataFrame(columns=[], index=[0.5])
  630. tm.assert_frame_equal(res, exp)
  631. def test_quantile_empty_no_rows_ints(self, interp_method):
  632. interpolation, method = interp_method
  633. df = DataFrame(columns=["a", "b"], dtype="int64")
  634. res = df.quantile(0.5, interpolation=interpolation, method=method)
  635. exp = Series([np.nan, np.nan], index=["a", "b"], name=0.5)
  636. tm.assert_series_equal(res, exp)
  637. def test_quantile_empty_no_rows_dt64(self, interp_method):
  638. interpolation, method = interp_method
  639. # datetimes
  640. df = DataFrame(columns=["a", "b"], dtype="datetime64[ns]")
  641. res = df.quantile(
  642. 0.5, numeric_only=False, interpolation=interpolation, method=method
  643. )
  644. exp = Series(
  645. [pd.NaT, pd.NaT], index=["a", "b"], dtype="datetime64[ns]", name=0.5
  646. )
  647. tm.assert_series_equal(res, exp)
  648. # Mixed dt64/dt64tz
  649. df["a"] = df["a"].dt.tz_localize("US/Central")
  650. res = df.quantile(
  651. 0.5, numeric_only=False, interpolation=interpolation, method=method
  652. )
  653. exp = exp.astype(object)
  654. tm.assert_series_equal(res, exp)
  655. # both dt64tz
  656. df["b"] = df["b"].dt.tz_localize("US/Central")
  657. res = df.quantile(
  658. 0.5, numeric_only=False, interpolation=interpolation, method=method
  659. )
  660. exp = exp.astype(df["b"].dtype)
  661. tm.assert_series_equal(res, exp)
  662. def test_quantile_empty_no_columns(self, interp_method):
  663. # GH#23925 _get_numeric_data may drop all columns
  664. interpolation, method = interp_method
  665. df = DataFrame(pd.date_range("1/1/18", periods=5))
  666. df.columns.name = "captain tightpants"
  667. result = df.quantile(
  668. 0.5, numeric_only=True, interpolation=interpolation, method=method
  669. )
  670. expected = Series([], index=[], name=0.5, dtype=np.float64)
  671. expected.index.name = "captain tightpants"
  672. tm.assert_series_equal(result, expected)
  673. result = df.quantile(
  674. [0.5], numeric_only=True, interpolation=interpolation, method=method
  675. )
  676. expected = DataFrame([], index=[0.5], columns=[])
  677. expected.columns.name = "captain tightpants"
  678. tm.assert_frame_equal(result, expected)
  679. def test_quantile_item_cache(
  680. self, using_array_manager, interp_method, using_copy_on_write
  681. ):
  682. # previous behavior incorrect retained an invalid _item_cache entry
  683. interpolation, method = interp_method
  684. df = DataFrame(np.random.randn(4, 3), columns=["A", "B", "C"])
  685. df["D"] = df["A"] * 2
  686. ser = df["A"]
  687. if not using_array_manager:
  688. assert len(df._mgr.blocks) == 2
  689. df.quantile(numeric_only=False, interpolation=interpolation, method=method)
  690. if using_copy_on_write:
  691. ser.iloc[0] = 99
  692. assert df.iloc[0, 0] == df["A"][0]
  693. assert df.iloc[0, 0] != 99
  694. else:
  695. ser.values[0] = 99
  696. assert df.iloc[0, 0] == df["A"][0]
  697. assert df.iloc[0, 0] == 99
  698. def test_invalid_method(self):
  699. with pytest.raises(ValueError, match="Invalid method: foo"):
  700. DataFrame(range(1)).quantile(0.5, method="foo")
  701. def test_table_invalid_interpolation(self):
  702. with pytest.raises(ValueError, match="Invalid interpolation: foo"):
  703. DataFrame(range(1)).quantile(0.5, method="table", interpolation="foo")
  704. class TestQuantileExtensionDtype:
  705. # TODO: tests for axis=1?
  706. # TODO: empty case?
  707. @pytest.fixture(
  708. params=[
  709. pytest.param(
  710. pd.IntervalIndex.from_breaks(range(10)),
  711. marks=pytest.mark.xfail(reason="raises when trying to add Intervals"),
  712. ),
  713. pd.period_range("2016-01-01", periods=9, freq="D"),
  714. pd.date_range("2016-01-01", periods=9, tz="US/Pacific"),
  715. pd.timedelta_range("1 Day", periods=9),
  716. pd.array(np.arange(9), dtype="Int64"),
  717. pd.array(np.arange(9), dtype="Float64"),
  718. ],
  719. ids=lambda x: str(x.dtype),
  720. )
  721. def index(self, request):
  722. # NB: not actually an Index object
  723. idx = request.param
  724. idx.name = "A"
  725. return idx
  726. @pytest.fixture
  727. def obj(self, index, frame_or_series):
  728. # bc index is not always an Index (yet), we need to re-patch .name
  729. obj = frame_or_series(index).copy()
  730. if frame_or_series is Series:
  731. obj.name = "A"
  732. else:
  733. obj.columns = ["A"]
  734. return obj
  735. def compute_quantile(self, obj, qs):
  736. if isinstance(obj, Series):
  737. result = obj.quantile(qs)
  738. else:
  739. result = obj.quantile(qs, numeric_only=False)
  740. return result
  741. def test_quantile_ea(self, request, obj, index):
  742. # result should be invariant to shuffling
  743. indexer = np.arange(len(index), dtype=np.intp)
  744. np.random.shuffle(indexer)
  745. obj = obj.iloc[indexer]
  746. qs = [0.5, 0, 1]
  747. result = self.compute_quantile(obj, qs)
  748. if np_version_under1p21 and index.dtype == "timedelta64[ns]":
  749. msg = "failed on Numpy 1.20.3; TypeError: data type 'Int64' not understood"
  750. mark = pytest.mark.xfail(reason=msg, raises=TypeError)
  751. request.node.add_marker(mark)
  752. exp_dtype = index.dtype
  753. if index.dtype == "Int64":
  754. # match non-nullable casting behavior
  755. exp_dtype = "Float64"
  756. # expected here assumes len(index) == 9
  757. expected = Series(
  758. [index[4], index[0], index[-1]], dtype=exp_dtype, index=qs, name="A"
  759. )
  760. expected = type(obj)(expected)
  761. tm.assert_equal(result, expected)
  762. def test_quantile_ea_with_na(self, obj, index):
  763. obj.iloc[0] = index._na_value
  764. obj.iloc[-1] = index._na_value
  765. # result should be invariant to shuffling
  766. indexer = np.arange(len(index), dtype=np.intp)
  767. np.random.shuffle(indexer)
  768. obj = obj.iloc[indexer]
  769. qs = [0.5, 0, 1]
  770. result = self.compute_quantile(obj, qs)
  771. # expected here assumes len(index) == 9
  772. expected = Series(
  773. [index[4], index[1], index[-2]], dtype=index.dtype, index=qs, name="A"
  774. )
  775. expected = type(obj)(expected)
  776. tm.assert_equal(result, expected)
  777. def test_quantile_ea_all_na(self, request, obj, index):
  778. obj.iloc[:] = index._na_value
  779. # Check dtypes were preserved; this was once a problem see GH#39763
  780. assert np.all(obj.dtypes == index.dtype)
  781. # result should be invariant to shuffling
  782. indexer = np.arange(len(index), dtype=np.intp)
  783. np.random.shuffle(indexer)
  784. obj = obj.iloc[indexer]
  785. qs = [0.5, 0, 1]
  786. result = self.compute_quantile(obj, qs)
  787. expected = index.take([-1, -1, -1], allow_fill=True, fill_value=index._na_value)
  788. expected = Series(expected, index=qs, name="A")
  789. expected = type(obj)(expected)
  790. tm.assert_equal(result, expected)
  791. def test_quantile_ea_scalar(self, request, obj, index):
  792. # scalar qs
  793. # result should be invariant to shuffling
  794. indexer = np.arange(len(index), dtype=np.intp)
  795. np.random.shuffle(indexer)
  796. obj = obj.iloc[indexer]
  797. qs = 0.5
  798. result = self.compute_quantile(obj, qs)
  799. if np_version_under1p21 and index.dtype == "timedelta64[ns]":
  800. msg = "failed on Numpy 1.20.3; TypeError: data type 'Int64' not understood"
  801. mark = pytest.mark.xfail(reason=msg, raises=TypeError)
  802. request.node.add_marker(mark)
  803. exp_dtype = index.dtype
  804. if index.dtype == "Int64":
  805. exp_dtype = "Float64"
  806. expected = Series({"A": index[4]}, dtype=exp_dtype, name=0.5)
  807. if isinstance(obj, Series):
  808. expected = expected["A"]
  809. assert result == expected
  810. else:
  811. tm.assert_series_equal(result, expected)
  812. @pytest.mark.parametrize(
  813. "dtype, expected_data, expected_index, axis",
  814. [
  815. ["float64", [], [], 1],
  816. ["int64", [], [], 1],
  817. ["float64", [np.nan, np.nan], ["a", "b"], 0],
  818. ["int64", [np.nan, np.nan], ["a", "b"], 0],
  819. ],
  820. )
  821. def test_empty_numeric(self, dtype, expected_data, expected_index, axis):
  822. # GH 14564
  823. df = DataFrame(columns=["a", "b"], dtype=dtype)
  824. result = df.quantile(0.5, axis=axis)
  825. expected = Series(
  826. expected_data, name=0.5, index=Index(expected_index), dtype="float64"
  827. )
  828. tm.assert_series_equal(result, expected)
  829. @pytest.mark.parametrize(
  830. "dtype, expected_data, expected_index, axis, expected_dtype",
  831. [
  832. ["datetime64[ns]", [], [], 1, "datetime64[ns]"],
  833. ["datetime64[ns]", [pd.NaT, pd.NaT], ["a", "b"], 0, "datetime64[ns]"],
  834. ],
  835. )
  836. def test_empty_datelike(
  837. self, dtype, expected_data, expected_index, axis, expected_dtype
  838. ):
  839. # GH 14564
  840. df = DataFrame(columns=["a", "b"], dtype=dtype)
  841. result = df.quantile(0.5, axis=axis, numeric_only=False)
  842. expected = Series(
  843. expected_data, name=0.5, index=Index(expected_index), dtype=expected_dtype
  844. )
  845. tm.assert_series_equal(result, expected)
  846. @pytest.mark.parametrize(
  847. "expected_data, expected_index, axis",
  848. [
  849. [[np.nan, np.nan], range(2), 1],
  850. [[], [], 0],
  851. ],
  852. )
  853. def test_datelike_numeric_only(self, expected_data, expected_index, axis):
  854. # GH 14564
  855. df = DataFrame(
  856. {
  857. "a": pd.to_datetime(["2010", "2011"]),
  858. "b": [0, 5],
  859. "c": pd.to_datetime(["2011", "2012"]),
  860. }
  861. )
  862. result = df[["a", "c"]].quantile(0.5, axis=axis, numeric_only=True)
  863. expected = Series(
  864. expected_data, name=0.5, index=Index(expected_index), dtype=np.float64
  865. )
  866. tm.assert_series_equal(result, expected)