test_missing.py 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154
  1. import numpy as np
  2. import pytest
  3. import pandas as pd
  4. from pandas import (
  5. DataFrame,
  6. Index,
  7. date_range,
  8. )
  9. import pandas._testing as tm
  10. @pytest.mark.parametrize("func", ["ffill", "bfill"])
  11. def test_groupby_column_index_name_lost_fill_funcs(func):
  12. # GH: 29764 groupby loses index sometimes
  13. df = DataFrame(
  14. [[1, 1.0, -1.0], [1, np.nan, np.nan], [1, 2.0, -2.0]],
  15. columns=Index(["type", "a", "b"], name="idx"),
  16. )
  17. df_grouped = df.groupby(["type"])[["a", "b"]]
  18. result = getattr(df_grouped, func)().columns
  19. expected = Index(["a", "b"], name="idx")
  20. tm.assert_index_equal(result, expected)
  21. @pytest.mark.parametrize("func", ["ffill", "bfill"])
  22. def test_groupby_fill_duplicate_column_names(func):
  23. # GH: 25610 ValueError with duplicate column names
  24. df1 = DataFrame({"field1": [1, 3, 4], "field2": [1, 3, 4]})
  25. df2 = DataFrame({"field1": [1, np.nan, 4]})
  26. df_grouped = pd.concat([df1, df2], axis=1).groupby(by=["field2"])
  27. expected = DataFrame(
  28. [[1, 1.0], [3, np.nan], [4, 4.0]], columns=["field1", "field1"]
  29. )
  30. result = getattr(df_grouped, func)()
  31. tm.assert_frame_equal(result, expected)
  32. def test_ffill_missing_arguments():
  33. # GH 14955
  34. df = DataFrame({"a": [1, 2], "b": [1, 1]})
  35. with pytest.raises(ValueError, match="Must specify a fill"):
  36. df.groupby("b").fillna()
  37. @pytest.mark.parametrize(
  38. "method, expected", [("ffill", [None, "a", "a"]), ("bfill", ["a", "a", None])]
  39. )
  40. def test_fillna_with_string_dtype(method, expected):
  41. # GH 40250
  42. df = DataFrame({"a": pd.array([None, "a", None], dtype="string"), "b": [0, 0, 0]})
  43. grp = df.groupby("b")
  44. result = grp.fillna(method=method)
  45. expected = DataFrame({"a": pd.array(expected, dtype="string")})
  46. tm.assert_frame_equal(result, expected)
  47. def test_fill_consistency():
  48. # GH9221
  49. # pass thru keyword arguments to the generated wrapper
  50. # are set if the passed kw is None (only)
  51. df = DataFrame(
  52. index=pd.MultiIndex.from_product(
  53. [["value1", "value2"], date_range("2014-01-01", "2014-01-06")]
  54. ),
  55. columns=Index(["1", "2"], name="id"),
  56. )
  57. df["1"] = [
  58. np.nan,
  59. 1,
  60. np.nan,
  61. np.nan,
  62. 11,
  63. np.nan,
  64. np.nan,
  65. 2,
  66. np.nan,
  67. np.nan,
  68. 22,
  69. np.nan,
  70. ]
  71. df["2"] = [
  72. np.nan,
  73. 3,
  74. np.nan,
  75. np.nan,
  76. 33,
  77. np.nan,
  78. np.nan,
  79. 4,
  80. np.nan,
  81. np.nan,
  82. 44,
  83. np.nan,
  84. ]
  85. expected = df.groupby(level=0, axis=0).fillna(method="ffill")
  86. result = df.T.groupby(level=0, axis=1).fillna(method="ffill").T
  87. tm.assert_frame_equal(result, expected)
  88. @pytest.mark.parametrize("method", ["ffill", "bfill"])
  89. @pytest.mark.parametrize("dropna", [True, False])
  90. @pytest.mark.parametrize("has_nan_group", [True, False])
  91. def test_ffill_handles_nan_groups(dropna, method, has_nan_group):
  92. # GH 34725
  93. df_without_nan_rows = DataFrame([(1, 0.1), (2, 0.2)])
  94. ridx = [-1, 0, -1, -1, 1, -1]
  95. df = df_without_nan_rows.reindex(ridx).reset_index(drop=True)
  96. group_b = np.nan if has_nan_group else "b"
  97. df["group_col"] = pd.Series(["a"] * 3 + [group_b] * 3)
  98. grouped = df.groupby(by="group_col", dropna=dropna)
  99. result = getattr(grouped, method)(limit=None)
  100. expected_rows = {
  101. ("ffill", True, True): [-1, 0, 0, -1, -1, -1],
  102. ("ffill", True, False): [-1, 0, 0, -1, 1, 1],
  103. ("ffill", False, True): [-1, 0, 0, -1, 1, 1],
  104. ("ffill", False, False): [-1, 0, 0, -1, 1, 1],
  105. ("bfill", True, True): [0, 0, -1, -1, -1, -1],
  106. ("bfill", True, False): [0, 0, -1, 1, 1, -1],
  107. ("bfill", False, True): [0, 0, -1, 1, 1, -1],
  108. ("bfill", False, False): [0, 0, -1, 1, 1, -1],
  109. }
  110. ridx = expected_rows.get((method, dropna, has_nan_group))
  111. expected = df_without_nan_rows.reindex(ridx).reset_index(drop=True)
  112. # columns are a 'take' on df.columns, which are object dtype
  113. expected.columns = expected.columns.astype(object)
  114. tm.assert_frame_equal(result, expected)
  115. @pytest.mark.parametrize("min_count, value", [(2, np.nan), (-1, 1.0)])
  116. @pytest.mark.parametrize("func", ["first", "last", "max", "min"])
  117. def test_min_count(func, min_count, value):
  118. # GH#37821
  119. df = DataFrame({"a": [1] * 3, "b": [1, np.nan, np.nan], "c": [np.nan] * 3})
  120. result = getattr(df.groupby("a"), func)(min_count=min_count)
  121. expected = DataFrame({"b": [value], "c": [np.nan]}, index=Index([1], name="a"))
  122. tm.assert_frame_equal(result, expected)
  123. def test_indices_with_missing():
  124. # GH 9304
  125. df = DataFrame({"a": [1, 1, np.nan], "b": [2, 3, 4], "c": [5, 6, 7]})
  126. g = df.groupby(["a", "b"])
  127. result = g.indices
  128. expected = {(1.0, 2): np.array([0]), (1.0, 3): np.array([1])}
  129. assert result == expected