test_categorical.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. """
  2. This file contains a minimal set of tests for compliance with the extension
  3. array interface test suite, and should contain no other tests.
  4. The test suite for the full functionality of the array is located in
  5. `pandas/tests/arrays/`.
  6. The tests in this file are inherited from the BaseExtensionTests, and only
  7. minimal tweaks should be applied to get the tests passing (by overwriting a
  8. parent method).
  9. Additional tests should either be added to one of the BaseExtensionTests
  10. classes (if they are relevant for the extension interface for all dtypes), or
  11. be added to the array-specific tests in `pandas/tests/arrays/`.
  12. """
  13. import string
  14. import numpy as np
  15. import pytest
  16. import pandas as pd
  17. from pandas import (
  18. Categorical,
  19. CategoricalIndex,
  20. Timestamp,
  21. )
  22. import pandas._testing as tm
  23. from pandas.api.types import CategoricalDtype
  24. from pandas.tests.extension import base
  25. def make_data():
  26. while True:
  27. values = np.random.choice(list(string.ascii_letters), size=100)
  28. # ensure we meet the requirements
  29. # 1. first two not null
  30. # 2. first and second are different
  31. if values[0] != values[1]:
  32. break
  33. return values
  34. @pytest.fixture
  35. def dtype():
  36. return CategoricalDtype()
  37. @pytest.fixture
  38. def data():
  39. """Length-100 array for this type.
  40. * data[0] and data[1] should both be non missing
  41. * data[0] and data[1] should not be equal
  42. """
  43. return Categorical(make_data())
  44. @pytest.fixture
  45. def data_missing():
  46. """Length 2 array with [NA, Valid]"""
  47. return Categorical([np.nan, "A"])
  48. @pytest.fixture
  49. def data_for_sorting():
  50. return Categorical(["A", "B", "C"], categories=["C", "A", "B"], ordered=True)
  51. @pytest.fixture
  52. def data_missing_for_sorting():
  53. return Categorical(["A", None, "B"], categories=["B", "A"], ordered=True)
  54. @pytest.fixture
  55. def na_value():
  56. return np.nan
  57. @pytest.fixture
  58. def data_for_grouping():
  59. return Categorical(["a", "a", None, None, "b", "b", "a", "c"])
  60. class TestDtype(base.BaseDtypeTests):
  61. pass
  62. class TestInterface(base.BaseInterfaceTests):
  63. @pytest.mark.xfail(reason="Memory usage doesn't match")
  64. def test_memory_usage(self, data):
  65. # Is this deliberate?
  66. super().test_memory_usage(data)
  67. def test_contains(self, data, data_missing):
  68. # GH-37867
  69. # na value handling in Categorical.__contains__ is deprecated.
  70. # See base.BaseInterFaceTests.test_contains for more details.
  71. na_value = data.dtype.na_value
  72. # ensure data without missing values
  73. data = data[~data.isna()]
  74. # first elements are non-missing
  75. assert data[0] in data
  76. assert data_missing[0] in data_missing
  77. # check the presence of na_value
  78. assert na_value in data_missing
  79. assert na_value not in data
  80. # Categoricals can contain other nan-likes than na_value
  81. for na_value_obj in tm.NULL_OBJECTS:
  82. if na_value_obj is na_value:
  83. continue
  84. assert na_value_obj not in data
  85. assert na_value_obj in data_missing # this line differs from super method
  86. class TestConstructors(base.BaseConstructorsTests):
  87. def test_empty(self, dtype):
  88. cls = dtype.construct_array_type()
  89. result = cls._empty((4,), dtype=dtype)
  90. assert isinstance(result, cls)
  91. # the dtype we passed is not initialized, so will not match the
  92. # dtype on our result.
  93. assert result.dtype == CategoricalDtype([])
  94. class TestReshaping(base.BaseReshapingTests):
  95. pass
  96. class TestGetitem(base.BaseGetitemTests):
  97. @pytest.mark.skip(reason="Backwards compatibility")
  98. def test_getitem_scalar(self, data):
  99. # CategoricalDtype.type isn't "correct" since it should
  100. # be a parent of the elements (object). But don't want
  101. # to break things by changing.
  102. super().test_getitem_scalar(data)
  103. class TestSetitem(base.BaseSetitemTests):
  104. pass
  105. class TestIndex(base.BaseIndexTests):
  106. pass
  107. class TestMissing(base.BaseMissingTests):
  108. pass
  109. class TestReduce(base.BaseNoReduceTests):
  110. pass
  111. class TestAccumulate(base.BaseAccumulateTests):
  112. @pytest.mark.parametrize("skipna", [True, False])
  113. def test_accumulate_series(self, data, all_numeric_accumulations, skipna):
  114. pass
  115. class TestMethods(base.BaseMethodsTests):
  116. @pytest.mark.xfail(reason="Unobserved categories included")
  117. def test_value_counts(self, all_data, dropna):
  118. return super().test_value_counts(all_data, dropna)
  119. def test_combine_add(self, data_repeated):
  120. # GH 20825
  121. # When adding categoricals in combine, result is a string
  122. orig_data1, orig_data2 = data_repeated(2)
  123. s1 = pd.Series(orig_data1)
  124. s2 = pd.Series(orig_data2)
  125. result = s1.combine(s2, lambda x1, x2: x1 + x2)
  126. expected = pd.Series(
  127. [a + b for (a, b) in zip(list(orig_data1), list(orig_data2))]
  128. )
  129. self.assert_series_equal(result, expected)
  130. val = s1.iloc[0]
  131. result = s1.combine(val, lambda x1, x2: x1 + x2)
  132. expected = pd.Series([a + val for a in list(orig_data1)])
  133. self.assert_series_equal(result, expected)
  134. class TestCasting(base.BaseCastingTests):
  135. @pytest.mark.parametrize("cls", [Categorical, CategoricalIndex])
  136. @pytest.mark.parametrize("values", [[1, np.nan], [Timestamp("2000"), pd.NaT]])
  137. def test_cast_nan_to_int(self, cls, values):
  138. # GH 28406
  139. s = cls(values)
  140. msg = "Cannot (cast|convert)"
  141. with pytest.raises((ValueError, TypeError), match=msg):
  142. s.astype(int)
  143. @pytest.mark.parametrize(
  144. "expected",
  145. [
  146. pd.Series(["2019", "2020"], dtype="datetime64[ns, UTC]"),
  147. pd.Series([0, 0], dtype="timedelta64[ns]"),
  148. pd.Series([pd.Period("2019"), pd.Period("2020")], dtype="period[A-DEC]"),
  149. pd.Series([pd.Interval(0, 1), pd.Interval(1, 2)], dtype="interval"),
  150. pd.Series([1, np.nan], dtype="Int64"),
  151. ],
  152. )
  153. def test_cast_category_to_extension_dtype(self, expected):
  154. # GH 28668
  155. result = expected.astype("category").astype(expected.dtype)
  156. tm.assert_series_equal(result, expected)
  157. @pytest.mark.parametrize(
  158. "dtype, expected",
  159. [
  160. (
  161. "datetime64[ns]",
  162. np.array(["2015-01-01T00:00:00.000000000"], dtype="datetime64[ns]"),
  163. ),
  164. (
  165. "datetime64[ns, MET]",
  166. pd.DatetimeIndex(
  167. [Timestamp("2015-01-01 00:00:00+0100", tz="MET")]
  168. ).array,
  169. ),
  170. ],
  171. )
  172. def test_consistent_casting(self, dtype, expected):
  173. # GH 28448
  174. result = Categorical(["2015-01-01"]).astype(dtype)
  175. assert result == expected
  176. class TestArithmeticOps(base.BaseArithmeticOpsTests):
  177. def test_arith_frame_with_scalar(self, data, all_arithmetic_operators, request):
  178. # frame & scalar
  179. op_name = all_arithmetic_operators
  180. if op_name == "__rmod__":
  181. request.node.add_marker(
  182. pytest.mark.xfail(
  183. reason="rmod never called when string is first argument"
  184. )
  185. )
  186. super().test_arith_frame_with_scalar(data, op_name)
  187. def test_arith_series_with_scalar(self, data, all_arithmetic_operators, request):
  188. op_name = all_arithmetic_operators
  189. if op_name == "__rmod__":
  190. request.node.add_marker(
  191. pytest.mark.xfail(
  192. reason="rmod never called when string is first argument"
  193. )
  194. )
  195. super().test_arith_series_with_scalar(data, op_name)
  196. def test_add_series_with_extension_array(self, data):
  197. ser = pd.Series(data)
  198. with pytest.raises(TypeError, match="cannot perform|unsupported operand"):
  199. ser + data
  200. def test_divmod_series_array(self):
  201. # GH 23287
  202. # skipping because it is not implemented
  203. pass
  204. def _check_divmod_op(self, s, op, other, exc=NotImplementedError):
  205. return super()._check_divmod_op(s, op, other, exc=TypeError)
  206. class TestComparisonOps(base.BaseComparisonOpsTests):
  207. def _compare_other(self, s, data, op, other):
  208. op_name = f"__{op.__name__}__"
  209. if op_name == "__eq__":
  210. result = op(s, other)
  211. expected = s.combine(other, lambda x, y: x == y)
  212. assert (result == expected).all()
  213. elif op_name == "__ne__":
  214. result = op(s, other)
  215. expected = s.combine(other, lambda x, y: x != y)
  216. assert (result == expected).all()
  217. else:
  218. msg = "Unordered Categoricals can only compare equality or not"
  219. with pytest.raises(TypeError, match=msg):
  220. op(data, other)
  221. @pytest.mark.parametrize(
  222. "categories",
  223. [["a", "b"], [0, 1], [Timestamp("2019"), Timestamp("2020")]],
  224. )
  225. def test_not_equal_with_na(self, categories):
  226. # https://github.com/pandas-dev/pandas/issues/32276
  227. c1 = Categorical.from_codes([-1, 0], categories=categories)
  228. c2 = Categorical.from_codes([0, 1], categories=categories)
  229. result = c1 != c2
  230. assert result.all()
  231. class TestParsing(base.BaseParsingTests):
  232. pass
  233. class Test2DCompat(base.NDArrayBacked2DTests):
  234. def test_repr_2d(self, data):
  235. # Categorical __repr__ doesn't include "Categorical", so we need
  236. # to special-case
  237. res = repr(data.reshape(1, -1))
  238. assert res.count("\nCategories") == 1
  239. res = repr(data.reshape(-1, 1))
  240. assert res.count("\nCategories") == 1