BUG: groupby ffill adds labels as extra column (#21521)

Author: Adam

doc/source/whatsnew/v0.25.0.rst

@@ -391,6 +391,7 @@ Groupby/Resample/Rolling
 - Bug in :meth:`pandas.core.window.Rolling.min` and :meth:`pandas.core.window.Rolling.max` that caused a memory leak (:issue:`25893`)
 - Bug in :meth:`pandas.core.groupby.GroupBy.idxmax` and :meth:`pandas.core.groupby.GroupBy.idxmin` with datetime column would return incorrect dtype (:issue:`25444`, :issue:`15306`)
 - Bug in :meth:`pandas.core.groupby.GroupBy.cumsum`, :meth:`pandas.core.groupby.GroupBy.cumprod`, :meth:`pandas.core.groupby.GroupBy.cummin` and :meth:`pandas.core.groupby.GroupBy.cummax` with categorical column having absent categories, would return incorrect result or segfault (:issue:`16771`)
+- Bug in :meth:`pandas.core.groupby.DataFrameGroupBy.ffill` and :meth:`pandas.core.groupby.DataFrameGroupBy.bfill` when group labels are not in frame, would concat them with the return value. (:issue:`21521`)
 
 Reshaping
 ^^^^^^^^^

pandas/core/groupby/generic.py

@@ -1472,8 +1472,14 @@ def _apply_to_column_groupbys(self, func):
     def _fill(self, direction, limit=None):
         """Overridden method to join grouped columns in output"""
         res = super(DataFrameGroupBy, self)._fill(direction, limit=limit)
+
+        # Columns with group labels get removed by GroupBy._fill before calling
+        # into cython; if they're supposed to be part of the output, we have to
+        # add them back here.  We add all group labels which came from the
+        # input DataFrame, and are in the list of selected columns.
         output = OrderedDict(
-            (grp.name, grp.grouper) for grp in self.grouper.groupings)
+            (grp.name, grp.grouper) for grp in self.grouper.groupings
+            if grp.in_axis and grp.name in self._selected_obj)
 
         from pandas import concat
         return concat((self._wrap_transformed_output(output), res), axis=1)

pandas/core/groupby/groupby.py

@@ -2127,7 +2127,8 @@ def pct_change(self, periods=1, fill_method='pad', limit=None, freq=None,
                                                      limit=limit, freq=freq,
                                                      axis=axis))
         filled = getattr(self, fill_method)(limit=limit)
-        filled = filled.drop(self.grouper.names, axis=1)
+        filled = filled.drop(filled.columns.intersection(self.grouper.names),
+                             axis=1)
         fill_grp = filled.groupby(self.grouper.labels)
         shifted = fill_grp.shift(periods=periods, freq=freq)
         return (filled / shifted) - 1

pandas/tests/groupby/test_transform.py

@@ -880,3 +880,14 @@ def test_transform_absent_categories(func):
     result = getattr(df.y.groupby(df.x), func)()
     expected = df.y
     assert_series_equal(result, expected)
+
+
+@pytest.mark.parametrize('func', ['ffill', 'bfill'])
+@pytest.mark.parametrize('key, val', [('level', 0), ('by', Series([0]))])
+def test_ffill_not_in_axis(func, key, val):
+    # GH 21521
+    df = pd.DataFrame([[0]])
+    result = getattr(df.groupby(**{key: val}), func)()
+    expected = df
+
+    assert_frame_equal(result, expected)