test_sparse.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475
  1. """
  2. This file contains a minimal set of tests for compliance with the extension
  3. array interface test suite, and should contain no other tests.
  4. The test suite for the full functionality of the array is located in
  5. `pandas/tests/arrays/`.
  6. The tests in this file are inherited from the BaseExtensionTests, and only
  7. minimal tweaks should be applied to get the tests passing (by overwriting a
  8. parent method).
  9. Additional tests should either be added to one of the BaseExtensionTests
  10. classes (if they are relevant for the extension interface for all dtypes), or
  11. be added to the array-specific tests in `pandas/tests/arrays/`.
  12. """
  13. import numpy as np
  14. import pytest
  15. from pandas.errors import PerformanceWarning
  16. import pandas as pd
  17. from pandas import SparseDtype
  18. import pandas._testing as tm
  19. from pandas.arrays import SparseArray
  20. from pandas.tests.extension import base
  21. def make_data(fill_value):
  22. if np.isnan(fill_value):
  23. data = np.random.uniform(size=100)
  24. else:
  25. data = np.random.randint(1, 100, size=100)
  26. if data[0] == data[1]:
  27. data[0] += 1
  28. data[2::3] = fill_value
  29. return data
  30. @pytest.fixture
  31. def dtype():
  32. return SparseDtype()
  33. @pytest.fixture(params=[0, np.nan])
  34. def data(request):
  35. """Length-100 PeriodArray for semantics test."""
  36. res = SparseArray(make_data(request.param), fill_value=request.param)
  37. return res
  38. @pytest.fixture
  39. def data_for_twos():
  40. return SparseArray(np.ones(100) * 2)
  41. @pytest.fixture(params=[0, np.nan])
  42. def data_missing(request):
  43. """Length 2 array with [NA, Valid]"""
  44. return SparseArray([np.nan, 1], fill_value=request.param)
  45. @pytest.fixture(params=[0, np.nan])
  46. def data_repeated(request):
  47. """Return different versions of data for count times"""
  48. def gen(count):
  49. for _ in range(count):
  50. yield SparseArray(make_data(request.param), fill_value=request.param)
  51. yield gen
  52. @pytest.fixture(params=[0, np.nan])
  53. def data_for_sorting(request):
  54. return SparseArray([2, 3, 1], fill_value=request.param)
  55. @pytest.fixture(params=[0, np.nan])
  56. def data_missing_for_sorting(request):
  57. return SparseArray([2, np.nan, 1], fill_value=request.param)
  58. @pytest.fixture
  59. def na_value():
  60. return np.nan
  61. @pytest.fixture
  62. def na_cmp():
  63. return lambda left, right: pd.isna(left) and pd.isna(right)
  64. @pytest.fixture(params=[0, np.nan])
  65. def data_for_grouping(request):
  66. return SparseArray([1, 1, np.nan, np.nan, 2, 2, 1, 3], fill_value=request.param)
  67. @pytest.fixture(params=[0, np.nan])
  68. def data_for_compare(request):
  69. return SparseArray([0, 0, np.nan, -2, -1, 4, 2, 3, 0, 0], fill_value=request.param)
  70. class BaseSparseTests:
  71. def _check_unsupported(self, data):
  72. if data.dtype == SparseDtype(int, 0):
  73. pytest.skip("Can't store nan in int array.")
  74. @pytest.mark.xfail(reason="SparseArray does not support setitem")
  75. def test_ravel(self, data):
  76. super().test_ravel(data)
  77. class TestDtype(BaseSparseTests, base.BaseDtypeTests):
  78. def test_array_type_with_arg(self, data, dtype):
  79. assert dtype.construct_array_type() is SparseArray
  80. class TestInterface(BaseSparseTests, base.BaseInterfaceTests):
  81. def test_copy(self, data):
  82. # __setitem__ does not work, so we only have a smoke-test
  83. data.copy()
  84. def test_view(self, data):
  85. # __setitem__ does not work, so we only have a smoke-test
  86. data.view()
  87. class TestConstructors(BaseSparseTests, base.BaseConstructorsTests):
  88. pass
  89. class TestReshaping(BaseSparseTests, base.BaseReshapingTests):
  90. def test_concat_mixed_dtypes(self, data):
  91. # https://github.com/pandas-dev/pandas/issues/20762
  92. # This should be the same, aside from concat([sparse, float])
  93. df1 = pd.DataFrame({"A": data[:3]})
  94. df2 = pd.DataFrame({"A": [1, 2, 3]})
  95. df3 = pd.DataFrame({"A": ["a", "b", "c"]}).astype("category")
  96. dfs = [df1, df2, df3]
  97. # dataframes
  98. result = pd.concat(dfs)
  99. expected = pd.concat(
  100. [x.apply(lambda s: np.asarray(s).astype(object)) for x in dfs]
  101. )
  102. self.assert_frame_equal(result, expected)
  103. @pytest.mark.parametrize(
  104. "columns",
  105. [
  106. ["A", "B"],
  107. pd.MultiIndex.from_tuples(
  108. [("A", "a"), ("A", "b")], names=["outer", "inner"]
  109. ),
  110. ],
  111. )
  112. def test_stack(self, data, columns):
  113. super().test_stack(data, columns)
  114. def test_concat_columns(self, data, na_value):
  115. self._check_unsupported(data)
  116. super().test_concat_columns(data, na_value)
  117. def test_concat_extension_arrays_copy_false(self, data, na_value):
  118. self._check_unsupported(data)
  119. super().test_concat_extension_arrays_copy_false(data, na_value)
  120. def test_align(self, data, na_value):
  121. self._check_unsupported(data)
  122. super().test_align(data, na_value)
  123. def test_align_frame(self, data, na_value):
  124. self._check_unsupported(data)
  125. super().test_align_frame(data, na_value)
  126. def test_align_series_frame(self, data, na_value):
  127. self._check_unsupported(data)
  128. super().test_align_series_frame(data, na_value)
  129. def test_merge(self, data, na_value):
  130. self._check_unsupported(data)
  131. super().test_merge(data, na_value)
  132. @pytest.mark.xfail(reason="SparseArray does not support setitem")
  133. def test_transpose(self, data):
  134. super().test_transpose(data)
  135. class TestGetitem(BaseSparseTests, base.BaseGetitemTests):
  136. def test_get(self, data):
  137. ser = pd.Series(data, index=[2 * i for i in range(len(data))])
  138. if np.isnan(ser.values.fill_value):
  139. assert np.isnan(ser.get(4)) and np.isnan(ser.iloc[2])
  140. else:
  141. assert ser.get(4) == ser.iloc[2]
  142. assert ser.get(2) == ser.iloc[1]
  143. def test_reindex(self, data, na_value):
  144. self._check_unsupported(data)
  145. super().test_reindex(data, na_value)
  146. # Skipping TestSetitem, since we don't implement it.
  147. class TestIndex(base.BaseIndexTests):
  148. pass
  149. class TestMissing(BaseSparseTests, base.BaseMissingTests):
  150. def test_isna(self, data_missing):
  151. sarr = SparseArray(data_missing)
  152. expected_dtype = SparseDtype(bool, pd.isna(data_missing.dtype.fill_value))
  153. expected = SparseArray([True, False], dtype=expected_dtype)
  154. result = sarr.isna()
  155. tm.assert_sp_array_equal(result, expected)
  156. # test isna for arr without na
  157. sarr = sarr.fillna(0)
  158. expected_dtype = SparseDtype(bool, pd.isna(data_missing.dtype.fill_value))
  159. expected = SparseArray([False, False], fill_value=False, dtype=expected_dtype)
  160. self.assert_equal(sarr.isna(), expected)
  161. def test_fillna_limit_pad(self, data_missing):
  162. with tm.assert_produces_warning(PerformanceWarning, check_stacklevel=False):
  163. super().test_fillna_limit_pad(data_missing)
  164. def test_fillna_limit_backfill(self, data_missing):
  165. with tm.assert_produces_warning(PerformanceWarning, check_stacklevel=False):
  166. super().test_fillna_limit_backfill(data_missing)
  167. def test_fillna_no_op_returns_copy(self, data, request):
  168. if np.isnan(data.fill_value):
  169. request.node.add_marker(
  170. pytest.mark.xfail(reason="returns array with different fill value")
  171. )
  172. with tm.assert_produces_warning(PerformanceWarning, check_stacklevel=False):
  173. super().test_fillna_no_op_returns_copy(data)
  174. def test_fillna_series_method(self, data_missing):
  175. with tm.assert_produces_warning(PerformanceWarning, check_stacklevel=False):
  176. super().test_fillna_limit_backfill(data_missing)
  177. @pytest.mark.xfail(reason="Unsupported")
  178. def test_fillna_series(self):
  179. # this one looks doable.
  180. super().test_fillna_series()
  181. def test_fillna_frame(self, data_missing):
  182. # Have to override to specify that fill_value will change.
  183. fill_value = data_missing[1]
  184. result = pd.DataFrame({"A": data_missing, "B": [1, 2]}).fillna(fill_value)
  185. if pd.isna(data_missing.fill_value):
  186. dtype = SparseDtype(data_missing.dtype, fill_value)
  187. else:
  188. dtype = data_missing.dtype
  189. expected = pd.DataFrame(
  190. {
  191. "A": data_missing._from_sequence([fill_value, fill_value], dtype=dtype),
  192. "B": [1, 2],
  193. }
  194. )
  195. self.assert_frame_equal(result, expected)
  196. class TestMethods(BaseSparseTests, base.BaseMethodsTests):
  197. _combine_le_expected_dtype = "Sparse[bool]"
  198. def test_fillna_copy_frame(self, data_missing, using_copy_on_write):
  199. arr = data_missing.take([1, 1])
  200. df = pd.DataFrame({"A": arr}, copy=False)
  201. filled_val = df.iloc[0, 0]
  202. result = df.fillna(filled_val)
  203. if hasattr(df._mgr, "blocks"):
  204. if using_copy_on_write:
  205. assert df.values.base is result.values.base
  206. else:
  207. assert df.values.base is not result.values.base
  208. assert df.A._values.to_dense() is arr.to_dense()
  209. def test_fillna_copy_series(self, data_missing, using_copy_on_write):
  210. arr = data_missing.take([1, 1])
  211. ser = pd.Series(arr, copy=False)
  212. filled_val = ser[0]
  213. result = ser.fillna(filled_val)
  214. if using_copy_on_write:
  215. assert ser._values is result._values
  216. else:
  217. assert ser._values is not result._values
  218. assert ser._values.to_dense() is arr.to_dense()
  219. @pytest.mark.xfail(reason="Not Applicable")
  220. def test_fillna_length_mismatch(self, data_missing):
  221. super().test_fillna_length_mismatch(data_missing)
  222. def test_where_series(self, data, na_value):
  223. assert data[0] != data[1]
  224. cls = type(data)
  225. a, b = data[:2]
  226. ser = pd.Series(cls._from_sequence([a, a, b, b], dtype=data.dtype))
  227. cond = np.array([True, True, False, False])
  228. result = ser.where(cond)
  229. new_dtype = SparseDtype("float", 0.0)
  230. expected = pd.Series(
  231. cls._from_sequence([a, a, na_value, na_value], dtype=new_dtype)
  232. )
  233. self.assert_series_equal(result, expected)
  234. other = cls._from_sequence([a, b, a, b], dtype=data.dtype)
  235. cond = np.array([True, False, True, True])
  236. result = ser.where(cond, other)
  237. expected = pd.Series(cls._from_sequence([a, b, b, b], dtype=data.dtype))
  238. self.assert_series_equal(result, expected)
  239. def test_combine_first(self, data, request):
  240. if data.dtype.subtype == "int":
  241. # Right now this is upcasted to float, just like combine_first
  242. # for Series[int]
  243. mark = pytest.mark.xfail(
  244. reason="TODO(SparseArray.__setitem__) will preserve dtype."
  245. )
  246. request.node.add_marker(mark)
  247. super().test_combine_first(data)
  248. def test_searchsorted(self, data_for_sorting, as_series):
  249. with tm.assert_produces_warning(PerformanceWarning, check_stacklevel=False):
  250. super().test_searchsorted(data_for_sorting, as_series)
  251. def test_shift_0_periods(self, data):
  252. # GH#33856 shifting with periods=0 should return a copy, not same obj
  253. result = data.shift(0)
  254. data._sparse_values[0] = data._sparse_values[1]
  255. assert result._sparse_values[0] != result._sparse_values[1]
  256. @pytest.mark.parametrize("method", ["argmax", "argmin"])
  257. def test_argmin_argmax_all_na(self, method, data, na_value):
  258. # overriding because Sparse[int64, 0] cannot handle na_value
  259. self._check_unsupported(data)
  260. super().test_argmin_argmax_all_na(method, data, na_value)
  261. @pytest.mark.parametrize("box", [pd.array, pd.Series, pd.DataFrame])
  262. def test_equals(self, data, na_value, as_series, box):
  263. self._check_unsupported(data)
  264. super().test_equals(data, na_value, as_series, box)
  265. class TestCasting(BaseSparseTests, base.BaseCastingTests):
  266. def test_astype_str(self, data):
  267. # pre-2.0 this would give a SparseDtype even if the user asked
  268. # for a non-sparse dtype.
  269. result = pd.Series(data[:5]).astype(str)
  270. expected = pd.Series([str(x) for x in data[:5]], dtype=object)
  271. self.assert_series_equal(result, expected)
  272. @pytest.mark.xfail(raises=TypeError, reason="no sparse StringDtype")
  273. def test_astype_string(self, data):
  274. super().test_astype_string(data)
  275. class TestArithmeticOps(BaseSparseTests, base.BaseArithmeticOpsTests):
  276. series_scalar_exc = None
  277. frame_scalar_exc = None
  278. divmod_exc = None
  279. series_array_exc = None
  280. def _skip_if_different_combine(self, data):
  281. if data.fill_value == 0:
  282. # arith ops call on dtype.fill_value so that the sparsity
  283. # is maintained. Combine can't be called on a dtype in
  284. # general, so we can't make the expected. This is tested elsewhere
  285. pytest.skip("Incorrected expected from Series.combine and tested elsewhere")
  286. def test_arith_series_with_scalar(self, data, all_arithmetic_operators):
  287. self._skip_if_different_combine(data)
  288. super().test_arith_series_with_scalar(data, all_arithmetic_operators)
  289. def test_arith_series_with_array(self, data, all_arithmetic_operators):
  290. self._skip_if_different_combine(data)
  291. super().test_arith_series_with_array(data, all_arithmetic_operators)
  292. def test_arith_frame_with_scalar(self, data, all_arithmetic_operators, request):
  293. if data.dtype.fill_value != 0:
  294. pass
  295. elif all_arithmetic_operators.strip("_") not in [
  296. "mul",
  297. "rmul",
  298. "floordiv",
  299. "rfloordiv",
  300. "pow",
  301. "mod",
  302. "rmod",
  303. ]:
  304. mark = pytest.mark.xfail(reason="result dtype.fill_value mismatch")
  305. request.node.add_marker(mark)
  306. super().test_arith_frame_with_scalar(data, all_arithmetic_operators)
  307. def _check_divmod_op(self, ser, op, other, exc=NotImplementedError):
  308. # We implement divmod
  309. super()._check_divmod_op(ser, op, other, exc=None)
  310. class TestComparisonOps(BaseSparseTests):
  311. def _compare_other(self, data_for_compare: SparseArray, comparison_op, other):
  312. op = comparison_op
  313. result = op(data_for_compare, other)
  314. assert isinstance(result, SparseArray)
  315. assert result.dtype.subtype == np.bool_
  316. if isinstance(other, SparseArray):
  317. fill_value = op(data_for_compare.fill_value, other.fill_value)
  318. else:
  319. fill_value = np.all(
  320. op(np.asarray(data_for_compare.fill_value), np.asarray(other))
  321. )
  322. expected = SparseArray(
  323. op(data_for_compare.to_dense(), np.asarray(other)),
  324. fill_value=fill_value,
  325. dtype=np.bool_,
  326. )
  327. tm.assert_sp_array_equal(result, expected)
  328. def test_scalar(self, data_for_compare: SparseArray, comparison_op):
  329. self._compare_other(data_for_compare, comparison_op, 0)
  330. self._compare_other(data_for_compare, comparison_op, 1)
  331. self._compare_other(data_for_compare, comparison_op, -1)
  332. self._compare_other(data_for_compare, comparison_op, np.nan)
  333. @pytest.mark.xfail(reason="Wrong indices")
  334. def test_array(self, data_for_compare: SparseArray, comparison_op):
  335. arr = np.linspace(-4, 5, 10)
  336. self._compare_other(data_for_compare, comparison_op, arr)
  337. @pytest.mark.xfail(reason="Wrong indices")
  338. def test_sparse_array(self, data_for_compare: SparseArray, comparison_op):
  339. arr = data_for_compare + 1
  340. self._compare_other(data_for_compare, comparison_op, arr)
  341. arr = data_for_compare * 2
  342. self._compare_other(data_for_compare, comparison_op, arr)
  343. class TestPrinting(BaseSparseTests, base.BasePrintingTests):
  344. @pytest.mark.xfail(reason="Different repr")
  345. def test_array_repr(self, data, size):
  346. super().test_array_repr(data, size)
  347. class TestParsing(BaseSparseTests, base.BaseParsingTests):
  348. @pytest.mark.parametrize("engine", ["c", "python"])
  349. def test_EA_types(self, engine, data):
  350. expected_msg = r".*must implement _from_sequence_of_strings.*"
  351. with pytest.raises(NotImplementedError, match=expected_msg):
  352. super().test_EA_types(engine, data)
  353. class TestNoNumericAccumulations(base.BaseAccumulateTests):
  354. @pytest.mark.parametrize("skipna", [True, False])
  355. def test_accumulate_series(self, data, all_numeric_accumulations, skipna):
  356. pass