From fdf33f5492a4d38eb7027b7b2f5d8782d1a5e64b Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 3 Feb 2020 14:20:56 +0100 Subject: [PATCH 1/4] REGR: fix non-reduction apply with tz-aware objects --- pandas/core/apply.py | 10 ++++++---- pandas/tests/frame/test_apply.py | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 81e1d84880f60..2a50fce5ae68e 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -299,10 +299,12 @@ def apply_standard(self): result = libreduction.compute_reduction( values, self.f, axis=self.axis, dummy=dummy, labels=labels ) - except ValueError as err: - if "Function does not reduce" not in str(err): - # catch only ValueError raised intentionally in libreduction - raise + except ValueError: + # TODO there are still cases that give another error (eg GH-31505) + # if "Function does not reduce" not in str(err): + # # catch only ValueError raised intentionally in libreduction + # raise + pass except TypeError: # e.g. test_apply_ignore_failures we just ignore if not self.ignore_failures: diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index e98f74e133ea9..ad4f86b57d435 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -703,6 +703,12 @@ def apply_list(row): ) tm.assert_series_equal(result, expected) + def test_apply_noreduction_tzaware_object(self): + # https://github.com/pandas-dev/pandas/issues/31505 + df = pd.DataFrame({"foo": [pd.Timestamp("2020", tz="UTC")]}, dtype="object") + result = df.apply(lambda col: col.copy()) + tm.assert_frame_equal(result, df) + class TestInferOutputShape: # the user has supplied an opaque UDF where From a4768fff77df7317bfb33811b0c8382d432dc1b6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 4 Feb 2020 17:41:18 +0100 Subject: [PATCH 2/4] specify dtype to avoid inferring objects inside libreduction --- pandas/_libs/reduction.pyx | 3 ++- pandas/core/apply.py | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pandas/_libs/reduction.pyx b/pandas/_libs/reduction.pyx index 89164c527002a..43d253f632f0f 100644 --- a/pandas/_libs/reduction.pyx +++ b/pandas/_libs/reduction.pyx @@ -114,7 +114,8 @@ cdef class Reducer: if self.typ is not None: # In this case, we also have self.index name = labels[i] - cached_typ = self.typ(chunk, index=self.index, name=name) + cached_typ = self.typ( + chunk, index=self.index, name=name, dtype=arr.dtype) # use the cached_typ if possible if cached_typ is not None: diff --git a/pandas/core/apply.py b/pandas/core/apply.py index 2a50fce5ae68e..81e1d84880f60 100644 --- a/pandas/core/apply.py +++ b/pandas/core/apply.py @@ -299,12 +299,10 @@ def apply_standard(self): result = libreduction.compute_reduction( values, self.f, axis=self.axis, dummy=dummy, labels=labels ) - except ValueError: - # TODO there are still cases that give another error (eg GH-31505) - # if "Function does not reduce" not in str(err): - # # catch only ValueError raised intentionally in libreduction - # raise - pass + except ValueError as err: + if "Function does not reduce" not in str(err): + # catch only ValueError raised intentionally in libreduction + raise except TypeError: # e.g. test_apply_ignore_failures we just ignore if not self.ignore_failures: From 7486b60e6d17b3117296a0b248ab314f8cb2754d Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Tue, 4 Feb 2020 22:38:22 +0100 Subject: [PATCH 3/4] add whatsnew --- doc/source/whatsnew/v1.0.1.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.0.1.rst b/doc/source/whatsnew/v1.0.1.rst index f9c756b2518af..8d00ad4e7729a 100644 --- a/doc/source/whatsnew/v1.0.1.rst +++ b/doc/source/whatsnew/v1.0.1.rst @@ -20,6 +20,7 @@ Fixed regressions - Fixed regression in ``DataFrame.__setitem__`` raising an ``AttributeError`` with a :class:`MultiIndex` and a non-monotonic indexer (:issue:`31449`) - Fixed regression in :class:`Series` multiplication when multiplying a numeric :class:`Series` with >10000 elements with a timedelta-like scalar (:issue:`31457`) - Fixed regression in :meth:`GroupBy.apply` if called with a function which returned a non-pandas non-scalar object (e.g. a list or numpy array) (:issue:`31441`) +- Fixed regression in :meth:`DataFrame.apply` with object dtype and non-reducing function (:issue:`31505`) - Fixed regression in :meth:`to_datetime` when parsing non-nanosecond resolution datetimes (:issue:`31491`) - Fixed regression in :meth:`~DataFrame.to_csv` where specifying an ``na_rep`` might truncate the values written (:issue:`31447`) - Fixed regression in :class:`Categorical` construction with ``numpy.str_`` categories (:issue:`31499`) From 302907d9e9163efdba4b42409fa9551f9e5a7d0e Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 5 Feb 2020 08:38:55 +0100 Subject: [PATCH 4/4] update test --- pandas/tests/frame/test_apply.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/frame/test_apply.py b/pandas/tests/frame/test_apply.py index ad4f86b57d435..fe6abef97acc4 100644 --- a/pandas/tests/frame/test_apply.py +++ b/pandas/tests/frame/test_apply.py @@ -706,7 +706,9 @@ def apply_list(row): def test_apply_noreduction_tzaware_object(self): # https://github.com/pandas-dev/pandas/issues/31505 df = pd.DataFrame({"foo": [pd.Timestamp("2020", tz="UTC")]}, dtype="object") - result = df.apply(lambda col: col.copy()) + result = df.apply(lambda x: x) + tm.assert_frame_equal(result, df) + result = df.apply(lambda x: x.copy()) tm.assert_frame_equal(result, df)