API: Sum / Prod of empty / all-NA (groupby)

Author: TomAugspurger

pandas/_libs/groupby_helper.pxi.in

@@ -89,7 +89,7 @@ def group_add_{{name}}(ndarray[{{dest_type2}}, ndim=2] out,
         for i in range(ncounts):
             for j in range(K):
                 if nobs[i, j] == 0:
-                    out[i, j] = NAN
+                    out[i, j] = 0
                 else:
                     out[i, j] = sumx[i, j]
 
@@ -148,7 +148,7 @@ def group_prod_{{name}}(ndarray[{{dest_type2}}, ndim=2] out,
         for i in range(ncounts):
             for j in range(K):
                 if nobs[i, j] == 0:
-                    out[i, j] = NAN
+                    out[i, j] = 1
                 else:
                     out[i, j] = prodx[i, j]
 

pandas/tests/groupby/test_aggregate.py

@@ -813,8 +813,6 @@ def test_cython_agg_empty_buckets(self):
         ops = [('mean', np.mean),
                ('median', lambda x: np.median(x) if len(x) > 0 else np.nan),
                ('var', lambda x: np.var(x, ddof=1)),
-               ('add', lambda x: np.sum(x) if len(x) > 0 else np.nan),
-               ('prod', np.prod),
                ('min', np.min),
                ('max', np.max), ]
 
@@ -830,6 +828,23 @@ def test_cython_agg_empty_buckets(self):
                 exc.args += ('operation: %s' % op,)
                 raise
 
+    def test_cython_agg_empty_buckets_nanops(self):
+        # Bug in python agg func not being evaluated on empty buckets
+        df = pd.DataFrame([11, 12, 13], columns=['a'])
+        grps = range(0, 25, 5)
+        result = df.groupby(pd.cut(df['a'], grps))._cython_agg_general('add')
+        intervals = pd.interval_range(0, 20, freq=5)
+        expected = pd.DataFrame(
+            {"a": [0, 0, 36, 0]},
+            index=pd.CategoricalIndex(intervals, name='a', ordered=True))
+        tm.assert_frame_equal(result, expected)
+
+        result = df.groupby(pd.cut(df['a'], grps))._cython_agg_general('prod')
+        expected = pd.DataFrame(
+            {"a": [1, 1, 1716, 1]},
+            index=pd.CategoricalIndex(intervals, name='a', ordered=True))
+        tm.assert_frame_equal(result, expected)
+
     def test_agg_over_numpy_arrays(self):
         # GH 3788
         df = pd.DataFrame([[1, np.array([10, 20, 30])],
@@ -925,3 +940,17 @@ def test_agg_structs_series(self, structure, expected):
         result = df.groupby('A')['C'].aggregate(structure)
         expected.index.name = 'A'
         assert_series_equal(result, expected)
+
+    @pytest.mark.xfail(reason="agg functions not called on empty groups")
+    def test_agg_category_nansum(self):
+        categories = ['a', 'b', 'c']
+        df = pd.DataFrame({"A": pd.Categorical(['a', 'a', 'b'],
+                                               categories=categories),
+                           'B': [1, 2, 3]})
+        result = df.groupby("A").B.agg(np.nansum)
+        expected = pd.Series([3, 3, 0],
+                             index=pd.CategoricalIndex(['a', 'b', 'c'],
+                                                       categories=categories,
+                                                       name='A'),
+                             name='B')
+        tm.assert_series_equal(result, expected)

pandas/tests/groupby/test_categorical.py

@@ -37,7 +37,7 @@ def test_groupby(self):
         # single grouper
         gb = df.groupby("A")
         exp_idx = CategoricalIndex(['a', 'b', 'z'], name='A', ordered=True)
-        expected = DataFrame({'values': Series([3, 7, np.nan], index=exp_idx)})
+        expected = DataFrame({'values': Series([3, 7, 0], index=exp_idx)})
         result = gb.sum()
         tm.assert_frame_equal(result, expected)
 
@@ -662,3 +662,25 @@ def test_groupby_categorical_two_columns(self):
                          "C3": [nan, nan, nan, nan, 10, 100,
                                 nan, nan, nan, nan, 200, 34]}, index=idx)
         tm.assert_frame_equal(res, exp)
+
+    def test_sum_zero(self):
+        df = pd.DataFrame({"A": pd.Categorical(['a', 'b', 'a'],
+                                               categories=['a', 'b', 'c']),
+                           'B': [1, 2, 1]})
+        result = df.groupby("A").B.sum()
+        expected = pd.Series([2, 2, 0],
+                             index=pd.CategoricalIndex(['a', 'b', 'c'],
+                                                       name='A'),
+                             name='B')
+        tm.assert_series_equal(result, expected)
+
+    def test_prod_one(self):
+        df = pd.DataFrame({"A": pd.Categorical(['a', 'b', 'a'],
+                                               categories=['a', 'b', 'c']),
+                           'B': [1, 2, 1]})
+        result = df.groupby("A").B.prod()
+        expected = pd.Series([1, 2, 1],
+                             index=pd.CategoricalIndex(['a', 'b', 'c'],
+                                                       name='A'),
+                             name='B')
+        tm.assert_series_equal(result, expected)

pandas/tests/groupby/test_groupby.py

@@ -2704,7 +2704,7 @@ def h(df, arg3):
 
         # Assert the results here
         index = pd.Index(['A', 'B', 'C'], name='group')
-        expected = pd.Series([-79.5160891089, -78.4839108911, None],
+        expected = pd.Series([-79.5160891089, -78.4839108911, -80],
                              index=index)
 
         assert_series_equal(expected, result)

pandas/tests/groupby/test_timegrouper.py

@@ -41,12 +41,12 @@ def test_groupby_with_timegrouper(self):
             df = df.set_index(['Date'])
 
             expected = DataFrame(
-                {'Quantity': np.nan},
+                {'Quantity': 0},
                 index=date_range('20130901 13:00:00',
                                  '20131205 13:00:00', freq='5D',
                                  name='Date', closed='left'))
             expected.iloc[[0, 6, 18], 0] = np.array(
-                [24., 6., 9.], dtype='float64')
+                [24, 6, 9], dtype='int64')
 
             result1 = df.resample('5D') .sum()
             assert_frame_equal(result1, expected)
@@ -261,9 +261,10 @@ def test_timegrouper_with_reg_groups(self):
         for freq in ['D', 'M', 'A', 'Q-APR']:
             expected = df.groupby('user_id')[
                 'whole_cost'].resample(
-                    freq).sum().dropna().reorder_levels(
+                    freq).sum().reorder_levels(
                         ['date', 'user_id']).sort_index().astype('int64')
             expected.name = 'whole_cost'
+            expected = expected[expected > 0]
 
             result1 = df.sort_index().groupby([pd.Grouper(freq=freq),
                                                'user_id'])['whole_cost'].sum()