BUG: updated rolling and expanding count for consistency (GH26996)

Updated the behavior of rolling and expanding count so that it becomes consistent with all other rolling and expanding functions.
Also updated many test cases to reflect this change of behavior.

Author: fujiaxiang

pandas/core/window/rolling.py

@@ -1186,13 +1186,10 @@ def count(self):
         window = self._get_window()
         window = min(window, len(obj)) if not self.center else window
 
-        # We set the default value min_periods to be 0 because count method
-        # is meant to count NAs, we don't want it by default requires all
-        # values in the window to be valid to produce a valid count
-        min_periods = 0 if self.min_periods is None else self.min_periods
-
-        # this is required as window is mutate above
-        min_periods = min(min_periods, window)
+        min_periods = self.min_periods
+        if min_periods is not None and not self.center:
+            # this is required as window is mutated above
+            min_periods = min(min_periods, window)
 
         results = []
         for b in blocks:
@@ -1665,7 +1662,11 @@ def _get_cov(X, Y):
             mean = lambda x: x.rolling(
                 window, self.min_periods, center=self.center
             ).mean(**kwargs)
-            count = (X + Y).rolling(window=window, center=self.center).count(**kwargs)
+            count = (
+                (X + Y)
+                .rolling(window=window, min_periods=0, center=self.center)
+                .count(**kwargs)
+            )
             bias_adj = count / (count - ddof)
             return (mean(X * Y) - mean(X) * mean(Y)) * bias_adj
 

pandas/tests/window/moments/test_moments_expanding.py

@@ -40,9 +40,9 @@ def test_expanding_corr(self):
         tm.assert_almost_equal(rolling_result, result)
 
     def test_expanding_count(self):
-        result = self.series.expanding().count()
+        result = self.series.expanding(min_periods=0).count()
         tm.assert_almost_equal(
-            result, self.series.rolling(window=len(self.series)).count()
+            result, self.series.rolling(window=len(self.series), min_periods=0).count()
         )
 
     def test_expanding_quantile(self):
@@ -369,7 +369,7 @@ def test_expanding_consistency(self, min_periods):
             )
             self._test_moments_consistency(
                 min_periods=min_periods,
-                count=lambda x: x.expanding().count(),
+                count=lambda x: x.expanding(min_periods=min_periods).count(),
                 mean=lambda x: x.expanding(min_periods=min_periods).mean(),
                 corr=lambda x, y: x.expanding(min_periods=min_periods).corr(y),
                 var_unbiased=lambda x: x.expanding(min_periods=min_periods).var(),

pandas/tests/window/moments/test_moments_rolling.py

@@ -777,8 +777,8 @@ def get_result(obj, window, min_periods=None, center=False):
                 series_result = get_result(series, window=win, min_periods=minp)
                 frame_result = get_result(frame, window=win, min_periods=minp)
             else:
-                series_result = get_result(series, window=win)
-                frame_result = get_result(frame, window=win)
+                series_result = get_result(series, window=win, min_periods=0)
+                frame_result = get_result(frame, window=win, min_periods=0)
 
             last_date = series_result.index[-1]
             prev_date = last_date - 24 * offsets.BDay()
@@ -851,10 +851,11 @@ def get_result(obj, window, min_periods=None, center=False):
                     pd.concat([obj, Series([np.NaN] * 9)]), 20, min_periods=15
                 )[9:].reset_index(drop=True)
             else:
-                result = get_result(obj, 20, center=True)
-                expected = get_result(pd.concat([obj, Series([np.NaN] * 9)]), 20)[
-                    9:
-                ].reset_index(drop=True)
+                result = get_result(obj, 20, min_periods=0, center=True)
+                print(result)
+                expected = get_result(
+                    pd.concat([obj, Series([np.NaN] * 9)]), 20, min_periods=0
+                )[9:].reset_index(drop=True)
 
             tm.assert_series_equal(result, expected)
 
@@ -893,21 +894,27 @@ def get_result(obj, window, min_periods=None, center=False):
             else:
                 series_xp = (
                     get_result(
-                        self.series.reindex(list(self.series.index) + s), window=25
+                        self.series.reindex(list(self.series.index) + s),
+                        window=25,
+                        min_periods=0,
                     )
                     .shift(-12)
                     .reindex(self.series.index)
                 )
                 frame_xp = (
                     get_result(
-                        self.frame.reindex(list(self.frame.index) + s), window=25
+                        self.frame.reindex(list(self.frame.index) + s),
+                        window=25,
+                        min_periods=0,
                     )
                     .shift(-12)
                     .reindex(self.frame.index)
                 )
 
-                series_rs = get_result(self.series, window=25, center=True)
-                frame_rs = get_result(self.frame, window=25, center=True)
+                series_rs = get_result(
+                    self.series, window=25, min_periods=0, center=True
+                )
+                frame_rs = get_result(self.frame, window=25, min_periods=0, center=True)
 
             if fill_value is not None:
                 series_xp = series_xp.fillna(fill_value)
@@ -964,7 +971,11 @@ def test_rolling_consistency(self, window, min_periods, center):
 
             self._test_moments_consistency_is_constant(
                 min_periods=min_periods,
-                count=lambda x: (x.rolling(window=window, center=center).count()),
+                count=lambda x: (
+                    x.rolling(
+                        window=window, min_periods=min_periods, center=center
+                    ).count()
+                ),
                 mean=lambda x: (
                     x.rolling(
                         window=window, min_periods=min_periods, center=center
@@ -989,19 +1000,26 @@ def test_rolling_consistency(self, window, min_periods, center):
                     ).var(ddof=0)
                 ),
                 var_debiasing_factors=lambda x: (
-                    x.rolling(window=window, center=center)
+                    x.rolling(window=window, min_periods=min_periods, center=center)
                     .count()
                     .divide(
-                        (x.rolling(window=window, center=center).count() - 1.0).replace(
-                            0.0, np.nan
-                        )
+                        (
+                            x.rolling(
+                                window=window, min_periods=min_periods, center=center
+                            ).count()
+                            - 1.0
+                        ).replace(0.0, np.nan)
                     )
                 ),
             )
 
             self._test_moments_consistency(
                 min_periods=min_periods,
-                count=lambda x: (x.rolling(window=window, center=center).count()),
+                count=lambda x: (
+                    x.rolling(
+                        window=window, min_periods=min_periods, center=center
+                    ).count()
+                ),
                 mean=lambda x: (
                     x.rolling(
                         window=window, min_periods=min_periods, center=center
@@ -1071,7 +1089,7 @@ def test_rolling_consistency(self, window, min_periods, center):
                     if name == "count":
                         rolling_f_result = rolling_f()
                         rolling_apply_f_result = x.rolling(
-                            window=window, min_periods=0, center=center
+                            window=window, min_periods=min_periods, center=center
                         ).apply(func=f, raw=True)
                     else:
                         if name in ["cov", "corr"]:

pandas/tests/window/test_api.py

@@ -237,10 +237,10 @@ def test_count_nonnumeric_types(self):
             columns=cols,
         )
 
-        result = df.rolling(window=2).count()
+        result = df.rolling(window=2, min_periods=0).count()
         tm.assert_frame_equal(result, expected)
 
-        result = df.rolling(1).count()
+        result = df.rolling(1, min_periods=0).count()
         expected = df.notna().astype(float)
         tm.assert_frame_equal(result, expected)
 

pandas/tests/window/test_dtypes.py

@@ -34,7 +34,7 @@ class Dtype:
     def get_expects(self):
         expects = {
             "sr1": {
-                "count": Series([1, 2, 2, 2, 2], dtype="float64"),
+                "count": Series([np.nan, 2, 2, 2, 2], dtype="float64"),
                 "max": Series([np.nan, 1, 2, 3, 4], dtype="float64"),
                 "min": Series([np.nan, 0, 1, 2, 3], dtype="float64"),
                 "sum": Series([np.nan, 1, 3, 5, 7], dtype="float64"),
@@ -44,7 +44,7 @@ def get_expects(self):
                 "median": Series([np.nan, 0.5, 1.5, 2.5, 3.5], dtype="float64"),
             },
             "sr2": {
-                "count": Series([1, 2, 2, 2, 2], dtype="float64"),
+                "count": Series([np.nan, 2, 2, 2, 2], dtype="float64"),
                 "max": Series([np.nan, 10, 8, 6, 4], dtype="float64"),
                 "min": Series([np.nan, 8, 6, 4, 2], dtype="float64"),
                 "sum": Series([np.nan, 18, 14, 10, 6], dtype="float64"),
@@ -54,7 +54,7 @@ def get_expects(self):
                 "median": Series([np.nan, 9, 7, 5, 3], dtype="float64"),
             },
             "sr3": {
-                "count": Series([1, 2, 2, 1, 1], dtype="float64"),
+                "count": Series([np.nan, 2, 2, 1, 1], dtype="float64"),
                 "max": Series([np.nan, 1, 2, np.nan, np.nan], dtype="float64"),
                 "min": Series([np.nan, 0, 1, np.nan, np.nan], dtype="float64"),
                 "sum": Series([np.nan, 1, 3, np.nan, np.nan], dtype="float64"),
@@ -67,7 +67,7 @@ def get_expects(self):
             },
             "df": {
                 "count": DataFrame(
-                    {0: Series([1, 2, 2, 2, 2]), 1: Series([1, 2, 2, 2, 2])},
+                    {0: Series([np.nan, 2, 2, 2, 2]), 1: Series([np.nan, 2, 2, 2, 2])},
                     dtype="float64",
                 ),
                 "max": DataFrame(

pandas/tests/window/test_rolling.py

@@ -344,7 +344,7 @@ def test_rolling_axis_count(self, axis_frame):
         else:
             expected = DataFrame({"x": [1.0, 1.0, 1.0], "y": [2.0, 2.0, 2.0]})
 
-        result = df.rolling(2, axis=axis_frame).count()
+        result = df.rolling(2, axis=axis_frame, min_periods=0).count()
         tm.assert_frame_equal(result, expected)
 
     def test_readonly_array(self):
@@ -469,7 +469,7 @@ def test_rolling_count_default_min_periods_with_null_values(test_series):
     # we want to by default produce a valid count even if
     # there are very few valid entries in the window
     values = [1, 2, 3, np.nan, 4, 5, 6]
-    expected_counts = [1.0, 2.0, 3.0, 2.0, 2.0, 2.0, 3.0]
+    expected_counts = [np.nan, np.nan, 3.0, 2.0, 2.0, 2.0, 3.0]
 
     if test_series:
         ser = Series(values)