From 444209d18e6802a8e72284a7546df11cfa2ab065 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 28 Aug 2019 22:48:27 -0500 Subject: [PATCH 01/20] Use try / except on empty Series --- pandas/core/apply.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 5c8599dbb054b..d9afb898220af 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -213,7 +213,12 @@ def apply_empty_result(self): pass if reduce: - return self.obj._constructor_sliced(np.nan, index=self.agg_axis) + try: + r = self.f(Series([]), *self.args, **self.kwds) + except: + r = np.nan + + return self.obj._constructor_sliced(r, index=self.agg_axis) else: return self.obj.copy() From b749cca52b839d978c29cd9435843a1357fd3be3 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Wed, 28 Aug 2019 23:10:20 -0500 Subject: [PATCH 02/20] Fix PEP8 --- pandas/core/apply.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index d9afb898220af..24acfc2dc9b28 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -215,7 +215,7 @@ def apply_empty_result(self): if reduce: try: r = self.f(Series([]), *self.args, **self.kwds) - except: + except Exception: r = np.nan return self.obj._constructor_sliced(r, index=self.agg_axis) From d9b620b705e0f5b6528dedb85d548510d31589f1 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 29 Aug 2019 06:53:45 -0500 Subject: [PATCH 03/20] Remove try / except --- pandas/core/apply.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 24acfc2dc9b28..819f8f578f77b 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -213,11 +213,7 @@ def apply_empty_result(self): pass if reduce: - try: - r = self.f(Series([]), *self.args, **self.kwds) - except Exception: - r = np.nan - + r = self.f(Series([]), *self.args, **self.kwds) return self.obj._constructor_sliced(r, index=self.agg_axis) else: return self.obj.copy() From a82479a2b2a8d7862777de57e9bdb91abf65b200 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 29 Aug 2019 07:39:48 -0500 Subject: [PATCH 04/20] Only apply on non-empty agg_axis --- pandas/core/apply.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 819f8f578f77b..b3d5210d5fb66 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -212,7 +212,7 @@ def apply_empty_result(self): except Exception: pass - if reduce: + if reduce and len(self.agg_axis): r = self.f(Series([]), *self.args, **self.kwds) return self.obj._constructor_sliced(r, index=self.agg_axis) else: From aa4b33be0a3e3e5bbb34c9b3b339e5d9b4308553 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 29 Aug 2019 18:04:04 -0500 Subject: [PATCH 05/20] Add tests --- pandas/tests/frame/test_apply.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index 92912ff9ec093..be176b47dd106 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -116,6 +116,26 @@ def test_apply_with_reduce_empty(self): # Ensure that x.append hasn't been called assert x == [] + def test_apply_funcs_over_empty(self): + # GH 28213 + df = DataFrame(columns=["a", "b", "c"]) + + result = df.apply(np.sum) + expected = df.sum() + assert_series_equal(result, expected) + + result = df.apply(np.prod) + expected = df.prod() + assert_series_equal(result, expected) + + result = df.apply(np.any) + expected = df.any() + assert_series_equal(result, expected) + + result = df.apply(np.all) + expected = df.all() + assert_series_equal(result, expected) + def test_apply_deprecate_reduce(self): empty_frame = DataFrame() From 61f432f05dd7af28072a7b487fbd3558d5e0f0f5 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 29 Aug 2019 18:13:55 -0500 Subject: [PATCH 06/20] Move condition --- pandas/core/apply.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index b3d5210d5fb66..b82a5e490a255 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -212,8 +212,12 @@ def apply_empty_result(self): except Exception: pass - if reduce and len(self.agg_axis): - r = self.f(Series([]), *self.args, **self.kwds) + if reduce: + if len(self.agg_axis): + r = self.f(Series([]), *self.args, **self.kwds) + else: + r = np.nan + return self.obj._constructor_sliced(r, index=self.agg_axis) else: return self.obj.copy() From cb68153f4b6fca19440ec6b79a0d1128c002ec11 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 29 Aug 2019 21:03:39 -0500 Subject: [PATCH 07/20] Don't unpack args twice --- pandas/core/apply.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index b82a5e490a255..fed75481d7d1b 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -204,17 +204,15 @@ def apply_empty_result(self): from pandas import Series if not reduce: - - EMPTY_SERIES = Series([]) try: - r = self.f(EMPTY_SERIES, *self.args, **self.kwds) + r = self.f(Series([])) reduce = not isinstance(r, Series) except Exception: pass if reduce: if len(self.agg_axis): - r = self.f(Series([]), *self.args, **self.kwds) + r = self.f(Series([])) else: r = np.nan From f4c7a9750c001129b27a1934378a2181a745b8ff Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 03:46:03 -0500 Subject: [PATCH 08/20] Add nunique test --- pandas/tests/frame/test_apply.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index be176b47dd106..3f2ab91b064fe 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -136,6 +136,10 @@ def test_apply_funcs_over_empty(self): expected = df.all() assert_series_equal(result, expected) + result = df.nunique() + expected = Series(0, index=df.columns) + assert_series_equal(result, expected) + def test_apply_deprecate_reduce(self): empty_frame = DataFrame() From 633a6b8e3ff146ab9107b8088c6cf51eadf24653 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 17:29:56 -0500 Subject: [PATCH 09/20] Add to release notes --- doc/source/whatsnew/v1.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index 050a26cc86d42..f61ebfb1f2446 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -84,6 +84,7 @@ Performance improvements Bug fixes ~~~~~~~~~ +- Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`) Categorical ^^^^^^^^^^^ From 1993c7c2bc572e77c8b3418ec7b2d7ee4e0f3273 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 18:06:37 -0500 Subject: [PATCH 10/20] Parametrize tests --- pandas/tests/frame/test_apply.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index 3f2ab91b064fe..555069c049645 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -116,28 +116,13 @@ def test_apply_with_reduce_empty(self): # Ensure that x.append hasn't been called assert x == [] - def test_apply_funcs_over_empty(self): + @pytest.mark.parametrize("func", ["sum", "prod", "any", "all"]) + def test_apply_funcs_over_empty(self, func): # GH 28213 df = DataFrame(columns=["a", "b", "c"]) - result = df.apply(np.sum) - expected = df.sum() - assert_series_equal(result, expected) - - result = df.apply(np.prod) - expected = df.prod() - assert_series_equal(result, expected) - - result = df.apply(np.any) - expected = df.any() - assert_series_equal(result, expected) - - result = df.apply(np.all) - expected = df.all() - assert_series_equal(result, expected) - - result = df.nunique() - expected = Series(0, index=df.columns) + result = df.apply(eval(func)) + expected = operator.methodcaller(func)(df) assert_series_equal(result, expected) def test_apply_deprecate_reduce(self): From a0c09eb2fa2ee2b81b49641d1e63f0e0a7ec56d8 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 19:15:52 -0500 Subject: [PATCH 11/20] Revert "Parametrize tests" This reverts commit 1993c7c2bc572e77c8b3418ec7b2d7ee4e0f3273. --- pandas/tests/frame/test_apply.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index 555069c049645..3f2ab91b064fe 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -116,13 +116,28 @@ def test_apply_with_reduce_empty(self): # Ensure that x.append hasn't been called assert x == [] - @pytest.mark.parametrize("func", ["sum", "prod", "any", "all"]) - def test_apply_funcs_over_empty(self, func): + def test_apply_funcs_over_empty(self): # GH 28213 df = DataFrame(columns=["a", "b", "c"]) - result = df.apply(eval(func)) - expected = operator.methodcaller(func)(df) + result = df.apply(np.sum) + expected = df.sum() + assert_series_equal(result, expected) + + result = df.apply(np.prod) + expected = df.prod() + assert_series_equal(result, expected) + + result = df.apply(np.any) + expected = df.any() + assert_series_equal(result, expected) + + result = df.apply(np.all) + expected = df.all() + assert_series_equal(result, expected) + + result = df.nunique() + expected = Series(0, index=df.columns) assert_series_equal(result, expected) def test_apply_deprecate_reduce(self): From 51de6ef470e8b9e8556a946f8645b961de58f07d Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 18:06:37 -0500 Subject: [PATCH 12/20] Parametrize tests --- pandas/tests/frame/test_apply.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index 3f2ab91b064fe..555069c049645 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -116,28 +116,13 @@ def test_apply_with_reduce_empty(self): # Ensure that x.append hasn't been called assert x == [] - def test_apply_funcs_over_empty(self): + @pytest.mark.parametrize("func", ["sum", "prod", "any", "all"]) + def test_apply_funcs_over_empty(self, func): # GH 28213 df = DataFrame(columns=["a", "b", "c"]) - result = df.apply(np.sum) - expected = df.sum() - assert_series_equal(result, expected) - - result = df.apply(np.prod) - expected = df.prod() - assert_series_equal(result, expected) - - result = df.apply(np.any) - expected = df.any() - assert_series_equal(result, expected) - - result = df.apply(np.all) - expected = df.all() - assert_series_equal(result, expected) - - result = df.nunique() - expected = Series(0, index=df.columns) + result = df.apply(eval(func)) + expected = operator.methodcaller(func)(df) assert_series_equal(result, expected) def test_apply_deprecate_reduce(self): From 91ef6572db27e5b7e880ab257697f583eb25dfa0 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 20:08:47 -0500 Subject: [PATCH 13/20] Use getattr --- pandas/tests/frame/test_apply.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index 555069c049645..7dc5152c46ec2 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -121,8 +121,8 @@ def test_apply_funcs_over_empty(self, func): # GH 28213 df = DataFrame(columns=["a", "b", "c"]) - result = df.apply(eval(func)) - expected = operator.methodcaller(func)(df) + result = df.apply(getattr(np, func)) + expected = getattr(df, func)() assert_series_equal(result, expected) def test_apply_deprecate_reduce(self): From 4a9ea0b8e05e71f52e7d8a936ebef02424c144c3 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 20:33:13 -0500 Subject: [PATCH 14/20] Add nunique test --- pandas/tests/frame/test_apply.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index 7dc5152c46ec2..fbc3c9e2e98eb 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -125,6 +125,18 @@ def test_apply_funcs_over_empty(self, func): expected = getattr(df, func)() assert_series_equal(result, expected) + def test_nunique_empty(self): + # GH 28213 + df = DataFrame(columns=["a", "b", "c"]) + + result = df.nunique() + expected = Series(0, index=df.columns) + assert_series_equal(result, expected) + + result = df.T.nunique() + expected = Series([]) + assert_series_equal(result, expected) + def test_apply_deprecate_reduce(self): empty_frame = DataFrame() From 42b22092d33a4bab329b3221a695f2ea1b637cd4 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Fri, 30 Aug 2019 21:01:10 -0500 Subject: [PATCH 15/20] Set index --- pandas/tests/frame/test_apply.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index fbc3c9e2e98eb..0328232213588 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -134,7 +134,7 @@ def test_nunique_empty(self): assert_series_equal(result, expected) result = df.T.nunique() - expected = Series([]) + expected = Series([], index=pd.Index([])) assert_series_equal(result, expected) def test_apply_deprecate_reduce(self): From de0e1ddd2d8a536e69b87dbaff419996cba02d63 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Mon, 2 Sep 2019 17:24:21 -0500 Subject: [PATCH 16/20] Move note under reshaping --- doc/source/whatsnew/v1.0.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v1.0.0.rst b/doc/source/whatsnew/v1.0.0.rst index f61ebfb1f2446..afc4788cbeebc 100644 --- a/doc/source/whatsnew/v1.0.0.rst +++ b/doc/source/whatsnew/v1.0.0.rst @@ -84,7 +84,6 @@ Performance improvements Bug fixes ~~~~~~~~~ -- Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`) Categorical ^^^^^^^^^^^ @@ -183,6 +182,7 @@ Groupby/resample/rolling Reshaping ^^^^^^^^^ +- Bug in :meth:`DataFrame.apply` that caused incorrect output with empty :class:`DataFrame` (:issue:`28202`, :issue:`21959`) - - From 9d6ae49ab446e98f8a79b52234774ff9a4796635 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Sat, 7 Sep 2019 21:58:38 -0400 Subject: [PATCH 17/20] Use should_reduce --- pandas/core/apply.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index f9d01d515bb0f..ec175a0c98f79 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -206,7 +206,7 @@ def apply_empty_result(self): if not should_reduce: try: r = self.f(Series([])) - reduce = not isinstance(r, Series) + should_reduce = not isinstance(r, Series) except Exception: pass else: From 2067c1a0f6d681a9cc52a6cd5541363a051548c7 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Sun, 8 Sep 2019 20:13:22 -0400 Subject: [PATCH 18/20] Fix merge --- pandas/core/apply.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index ec175a0c98f79..61d093d19e4be 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -206,7 +206,6 @@ def apply_empty_result(self): if not should_reduce: try: r = self.f(Series([])) - should_reduce = not isinstance(r, Series) except Exception: pass else: From 9442a04f08860287032e3e82026500bd85825345 Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Mon, 9 Sep 2019 18:14:44 -0400 Subject: [PATCH 19/20] Narrow exception catching --- pandas/core/apply.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 61d093d19e4be..d2d2ed650d9d9 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -206,7 +206,7 @@ def apply_empty_result(self): if not should_reduce: try: r = self.f(Series([])) - except Exception: + except (TypeError, KeyError): pass else: should_reduce = not isinstance(r, Series) From f3077aaa4f0cb273774061f41b2dcf68eb2a488e Mon Sep 17 00:00:00 2001 From: Daniel Saxton <> Date: Thu, 12 Sep 2019 09:19:07 -0400 Subject: [PATCH 20/20] Revert "Narrow exception catching" This reverts commit 9442a04f08860287032e3e82026500bd85825345. --- pandas/core/apply.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index d2d2ed650d9d9..61d093d19e4be 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -206,7 +206,7 @@ def apply_empty_result(self): if not should_reduce: try: r = self.f(Series([])) - except (TypeError, KeyError): + except Exception: pass else: should_reduce = not isinstance(r, Series)