ENH: Added a min_count keyword to stat funcs

The current default is 1, reproducing the behavior of pandas 0.21. The current
test suite should pass.

Currently, only nansum and nanprod actually do anything with `min_count`. It
will not be hard to adjust other nan* methods use it if we want. This was just
simplest for now.

Additional tests for the new behavior have been added.

Author: TomAugspurger

pandas/core/categorical.py

@@ -1956,7 +1956,7 @@ def _reduce(self, op, name, axis=0, skipna=True, numeric_only=None,
             raise TypeError(msg.format(op=name))
         return func(numeric_only=numeric_only, **kwds)
 
-    def min(self, numeric_only=None, **kwargs):
+    def min(self, numeric_only=None, min_count=1, **kwargs):
         """ The minimum value of the object.
 
         Only ordered `Categoricals` have a minimum!
@@ -1981,7 +1981,7 @@ def min(self, numeric_only=None, **kwargs):
         else:
             return self.categories[pointer]
 
-    def max(self, numeric_only=None, **kwargs):
+    def max(self, numeric_only=None, min_count=1, **kwargs):
         """ The maximum value of the object.
 
         Only ordered `Categoricals` have a maximum!

pandas/core/generic.py

@@ -7424,7 +7424,7 @@ def _add_series_only_operations(cls):
 
         axis_descr, name, name2 = _doc_parms(cls)
 
-        def nanptp(values, axis=0, skipna=True):
+        def nanptp(values, axis=0, min_count=1, skipna=True):
             nmax = nanops.nanmax(values, axis, skipna)
             nmin = nanops.nanmin(values, axis, skipna)
             return nmax - nmin
@@ -7528,6 +7528,9 @@ def _doc_parms(cls):
 numeric_only : boolean, default None
     Include only float, int, boolean columns. If None, will attempt to use
     everything, then use only numeric data. Not implemented for Series.
+min_count : int, default 1
+    The required number of valid values to perform the operation. If fewer than
+    ``min_count`` non-NA values are present the result will be NA.
 
 Returns
 -------
@@ -7604,6 +7607,7 @@ def _make_stat_function(cls, name, name1, name2, axis_descr, desc, f):
                   axis_descr=axis_descr)
     @Appender(_num_doc)
     def stat_func(self, axis=None, skipna=None, level=None, numeric_only=None,
+                  min_count=1,
                   **kwargs):
         nv.validate_stat_func(tuple(), kwargs, fname=name)
         if skipna is None:
@@ -7614,7 +7618,7 @@ def stat_func(self, axis=None, skipna=None, level=None, numeric_only=None,
             return self._agg_by_level(name, axis=axis, level=level,
                                       skipna=skipna)
         return self._reduce(f, name, axis=axis, skipna=skipna,
-                            numeric_only=numeric_only)
+                            numeric_only=numeric_only, min_count=min_count)
 
     return set_function_name(stat_func, name, cls)
 

pandas/core/indexes/datetimelike.py

@@ -479,7 +479,7 @@ def tolist(self):
         """
         return list(self.astype(object))
 
-    def min(self, axis=None, *args, **kwargs):
+    def min(self, axis=None, min_count=1, *args, **kwargs):
         """
         Return the minimum value of the Index or minimum along
         an axis.
@@ -527,7 +527,7 @@ def argmin(self, axis=None, *args, **kwargs):
             i8[mask] = np.iinfo('int64').max
         return i8.argmin()
 
-    def max(self, axis=None, *args, **kwargs):
+    def max(self, axis=None, min_count=1, *args, **kwargs):
         """
         Return the maximum value of the Index or maximum along
         an axis.

pandas/core/nanops.py

@@ -101,14 +101,13 @@ def __call__(self, alt):
             bn_func = None
 
         @functools.wraps(alt)
-        def f(values, axis=None, skipna=True, **kwds):
+        def f(values, axis=None, skipna=True, min_count=1, **kwds):
             if len(self.kwargs) > 0:
                 for k, v in compat.iteritems(self.kwargs):
                     if k not in kwds:
                         kwds[k] = v
             try:
-                if values.size == 0:
-
+                if values.size < min_count:
                     # we either return np.nan or pd.NaT
                     if is_numeric_dtype(values):
                         values = values.astype('float64')
@@ -132,7 +131,8 @@ def f(values, axis=None, skipna=True, **kwds):
                     if _has_infs(result):
                         result = alt(values, axis=axis, skipna=skipna, **kwds)
                 else:
-                    result = alt(values, axis=axis, skipna=skipna, **kwds)
+                    result = alt(values, axis=axis, skipna=skipna,
+                                 min_count=min_count, **kwds)
             except Exception:
                 try:
                     result = alt(values, axis=axis, skipna=skipna, **kwds)
@@ -292,34 +292,53 @@ def _wrap_results(result, dtype):
     return result
 
 
-def nanany(values, axis=None, skipna=True):
+def _na_for_min_count(values, axis):
+    # we either return np.nan or pd.NaT
+    if is_numeric_dtype(values):
+        values = values.astype('float64')
+    fill_value = na_value_for_dtype(values.dtype)
+
+    if values.ndim == 1:
+        return fill_value
+    else:
+        result_shape = (values.shape[:axis] +
+                        values.shape[axis + 1:])
+        result = np.empty(result_shape, dtype=values.dtype)
+        result.fill(fill_value)
+        return result
+
+
+def nanany(values, axis=None, skipna=True, min_count=1):
     values, mask, dtype, _ = _get_values(values, skipna, False, copy=skipna)
     return values.any(axis)
 
 
-def nanall(values, axis=None, skipna=True):
+def nanall(values, axis=None, skipna=True, min_count=1):
     values, mask, dtype, _ = _get_values(values, skipna, True, copy=skipna)
     return values.all(axis)
 
 
 @disallow('M8')
 @bottleneck_switch()
-def nansum(values, axis=None, skipna=True):
+def nansum(values, axis=None, skipna=True, min_count=1):
+    if len(values) < min_count:
+        return _na_for_min_count(values, axis=axis)
+
     values, mask, dtype, dtype_max = _get_values(values, skipna, 0)
     dtype_sum = dtype_max
     if is_float_dtype(dtype):
         dtype_sum = dtype
     elif is_timedelta64_dtype(dtype):
         dtype_sum = np.float64
     the_sum = values.sum(axis, dtype=dtype_sum)
-    the_sum = _maybe_null_out(the_sum, axis, mask)
+    the_sum = _maybe_null_out(the_sum, axis, mask, min_count=min_count)
 
     return _wrap_results(the_sum, dtype)
 
 
 @disallow('M8')
 @bottleneck_switch()
-def nanmean(values, axis=None, skipna=True):
+def nanmean(values, axis=None, skipna=True, min_count=1):
     values, mask, dtype, dtype_max = _get_values(values, skipna, 0)
 
     dtype_sum = dtype_max
@@ -345,7 +364,7 @@ def nanmean(values, axis=None, skipna=True):
 
 @disallow('M8')
 @bottleneck_switch()
-def nanmedian(values, axis=None, skipna=True):
+def nanmedian(values, axis=None, skipna=True, min_count=1):
 
     values, mask, dtype, dtype_max = _get_values(values, skipna)
 
@@ -405,14 +424,14 @@ def _get_counts_nanvar(mask, axis, ddof, dtype=float):
 
 @disallow('M8')
 @bottleneck_switch(ddof=1)
-def nanstd(values, axis=None, skipna=True, ddof=1):
+def nanstd(values, axis=None, skipna=True, ddof=1, min_count=1):
     result = np.sqrt(nanvar(values, axis=axis, skipna=skipna, ddof=ddof))
     return _wrap_results(result, values.dtype)
 
 
 @disallow('M8')
 @bottleneck_switch(ddof=1)
-def nanvar(values, axis=None, skipna=True, ddof=1):
+def nanvar(values, axis=None, skipna=True, ddof=1, min_count=1):
 
     values = _values_from_object(values)
     dtype = values.dtype
@@ -452,7 +471,7 @@ def nanvar(values, axis=None, skipna=True, ddof=1):
 
 
 @disallow('M8', 'm8')
-def nansem(values, axis=None, skipna=True, ddof=1):
+def nansem(values, axis=None, skipna=True, ddof=1, min_count=1):
     var = nanvar(values, axis, skipna, ddof=ddof)
 
     mask = isna(values)
@@ -492,7 +511,7 @@ def reduction(values, axis=None, skipna=True):
 
 
 @disallow('O')
-def nanargmax(values, axis=None, skipna=True):
+def nanargmax(values, axis=None, skipna=True, min_count=1):
     """
     Returns -1 in the NA case
     """
@@ -503,7 +522,7 @@ def nanargmax(values, axis=None, skipna=True):
 
 
 @disallow('O')
-def nanargmin(values, axis=None, skipna=True):
+def nanargmin(values, axis=None, skipna=True, min_count=1):
     """
     Returns -1 in the NA case
     """
@@ -514,7 +533,7 @@ def nanargmin(values, axis=None, skipna=True):
 
 
 @disallow('M8', 'm8')
-def nanskew(values, axis=None, skipna=True):
+def nanskew(values, axis=None, skipna=True, min_count=1):
     """ Compute the sample skewness.
 
     The statistic computed here is the adjusted Fisher-Pearson standardized
@@ -573,7 +592,7 @@ def nanskew(values, axis=None, skipna=True):
 
 
 @disallow('M8', 'm8')
-def nankurt(values, axis=None, skipna=True):
+def nankurt(values, axis=None, skipna=True, min_count=1):
     """ Compute the sample excess kurtosis.
 
     The statistic computed here is the adjusted Fisher-Pearson standardized
@@ -641,13 +660,16 @@ def nankurt(values, axis=None, skipna=True):
 
 
 @disallow('M8', 'm8')
-def nanprod(values, axis=None, skipna=True):
+def nanprod(values, axis=None, skipna=True, min_count=1):
+    if len(values) < min_count:
+        return _na_for_min_count(values, axis=axis)
+
     mask = isna(values)
     if skipna and not is_any_int_dtype(values):
         values = values.copy()
         values[mask] = 1
     result = values.prod(axis)
-    return _maybe_null_out(result, axis, mask)
+    return _maybe_null_out(result, axis, mask, min_count=min_count)
 
 
 def _maybe_arg_null_out(result, axis, mask, skipna):
@@ -683,7 +705,7 @@ def _get_counts(mask, axis, dtype=float):
         return np.array(count, dtype=dtype)
 
 
-def _maybe_null_out(result, axis, mask):
+def _maybe_null_out(result, axis, mask, min_count=1):
     if axis is not None and getattr(result, 'ndim', False):
         null_mask = (mask.shape[axis] - mask.sum(axis)) == 0
         if np.any(null_mask):
@@ -698,7 +720,7 @@ def _maybe_null_out(result, axis, mask):
                 result[null_mask] = None
     elif result is not tslib.NaT:
         null_mask = mask.size - mask.sum()
-        if null_mask == 0:
+        if null_mask < min_count:
             result = np.nan
 
     return result
@@ -714,7 +736,7 @@ def _zero_out_fperr(arg):
 
 
 @disallow('M8', 'm8')
-def nancorr(a, b, method='pearson', min_periods=None):
+def nancorr(a, b, method='pearson', min_periods=None, min_count=1):
     """
     a, b: ndarrays
     """
@@ -761,7 +783,7 @@ def _spearman(a, b):
 
 
 @disallow('M8', 'm8')
-def nancov(a, b, min_periods=None):
+def nancov(a, b, min_periods=None, min_count=1):
     if len(a) != len(b):
         raise AssertionError('Operands to nancov must have same size')
 

pandas/tests/series/test_analytics.py

@@ -1759,6 +1759,48 @@ def test_value_counts_categorical_not_ordered(self):
         tm.assert_series_equal(idx.value_counts(normalize=True), exp)
 
 
+class TestMinCount():
+    @pytest.mark.parametrize("use_bottleneck", [True, False])
+    @pytest.mark.parametrize("method", [("sum", 0), ("prod", 1)])
+    def test_min_count_empty(self, method, use_bottleneck):
+        method, unit = method
+        s = pd.Series()
+
+        with pd.option_context("compute.use_bottleneck", use_bottleneck):
+            result = getattr(s, method)(min_count=0)
+            assert result == unit
+
+            result = getattr(s, method)(min_count=1)
+            assert np.isnan(result)
+
+        s = pd.Series([1])
+        with pd.option_context("compute.use_bottleneck", use_bottleneck):
+            result = getattr(s, method)(min_count=0)
+            assert result == 1
+
+            result = getattr(s, method)(min_count=1)
+            assert result == 1
+
+            result = getattr(s, method)(min_count=2)
+            assert np.isnan(result)
+
+    @pytest.mark.parametrize("use_bottleneck", [True, False])
+    @pytest.mark.parametrize("method", [("sum", 0), ("prod", 1)])
+    def test_min_count_with_na(self, method, use_bottleneck):
+        method, unit = method
+        s = pd.Series([np.nan])
+        with pd.option_context("compute.use_bottleneck", use_bottleneck):
+            result = getattr(s, method)(min_count=0)
+            assert result == unit
+
+            result = getattr(s, method)(min_count=1)
+            assert np.isnan(result)
+
+            s = pd.Series([np.nan, unit])
+            result = getattr(s, method)(min_count=1)
+            assert result == unit
+
+
 @pytest.fixture
 def s_main_dtypes():
     df = pd.DataFrame(