BUG/Compat: dt64/td64 cummin, cummax on np 1.18 (#30460)

Author: jbrockmendel

pandas/core/generic.py

@@ -11131,11 +11131,35 @@ def cum_func(self, axis=None, skipna=True, *args, **kwargs):
             axis = self._get_axis_number(axis)
 
         y = com.values_from_object(self).copy()
+        d = self._construct_axes_dict()
+        d["copy"] = False
+
+        if issubclass(y.dtype.type, (np.datetime64, np.timedelta64)):
+            # numpy 1.18 started sorting NaTs at the end instead of beginning,
+            #  so we need to work around to maintain backwards-consistency.
+            orig_dtype = y.dtype
+            if accum_func == np.minimum.accumulate:
+                # Note: the accum_func comparison fails as an "is" comparison
+                # Note that "y" is always a copy, so we can safely modify it
+                mask = isna(self)
+                y = y.view("i8")
+                y[mask] = np.iinfo(np.int64).max
+
+            result = accum_func(y.view("i8"), axis).view(orig_dtype)
+            if skipna:
+                mask = isna(self)
+                np.putmask(result, mask, iNaT)
+            elif accum_func == np.minimum.accumulate:
+                # Restore NaTs that we masked previously
+                nz = (~np.asarray(mask)).nonzero()[0]
+                if len(nz):
+                    # everything up to the first non-na entry stays NaT
+                    result[: nz[0]] = iNaT
+
+            if self.ndim == 1:
+                # restore dt64tz dtype
+                d["dtype"] = self.dtype
 
-        if skipna and issubclass(y.dtype.type, (np.datetime64, np.timedelta64)):
-            result = accum_func(y, axis)
-            mask = isna(self)
-            np.putmask(result, mask, iNaT)
         elif skipna and not issubclass(y.dtype.type, (np.integer, np.bool_)):
             mask = isna(self)
             np.putmask(y, mask, mask_a)
@@ -11144,8 +11168,6 @@ def cum_func(self, axis=None, skipna=True, *args, **kwargs):
         else:
             result = accum_func(y, axis)
 
-        d = self._construct_axes_dict()
-        d["copy"] = False
         return self._constructor(result, **d).__finalize__(self)
 
     return set_function_name(cum_func, name, cls)

pandas/tests/series/test_cumulative.py

@@ -10,8 +10,6 @@
 import numpy as np
 import pytest
 
-from pandas.compat.numpy import _np_version_under1p18
-
 import pandas as pd
 import pandas.util.testing as tm
 
@@ -63,53 +61,54 @@ def test_cummax(self, datetime_series):
 
         tm.assert_series_equal(result, expected)
 
-    @pytest.mark.xfail(
-        not _np_version_under1p18, reason="numpy 1.18 changed min/max behavior for NaT"
-    )
-    def test_cummin_datetime64(self):
+    @pytest.mark.parametrize("tz", [None, "US/Pacific"])
+    def test_cummin_datetime64(self, tz):
         s = pd.Series(
-            pd.to_datetime(["NaT", "2000-1-2", "NaT", "2000-1-1", "NaT", "2000-1-3"])
+            pd.to_datetime(
+                ["NaT", "2000-1-2", "NaT", "2000-1-1", "NaT", "2000-1-3"]
+            ).tz_localize(tz)
         )
 
         expected = pd.Series(
-            pd.to_datetime(["NaT", "2000-1-2", "NaT", "2000-1-1", "NaT", "2000-1-1"])
+            pd.to_datetime(
+                ["NaT", "2000-1-2", "NaT", "2000-1-1", "NaT", "2000-1-1"]
+            ).tz_localize(tz)
         )
         result = s.cummin(skipna=True)
         tm.assert_series_equal(expected, result)
 
         expected = pd.Series(
             pd.to_datetime(
                 ["NaT", "2000-1-2", "2000-1-2", "2000-1-1", "2000-1-1", "2000-1-1"]
-            )
+            ).tz_localize(tz)
         )
         result = s.cummin(skipna=False)
         tm.assert_series_equal(expected, result)
 
-    @pytest.mark.xfail(
-        not _np_version_under1p18, reason="numpy 1.18 changed min/max behavior for NaT"
-    )
-    def test_cummax_datetime64(self):
+    @pytest.mark.parametrize("tz", [None, "US/Pacific"])
+    def test_cummax_datetime64(self, tz):
         s = pd.Series(
-            pd.to_datetime(["NaT", "2000-1-2", "NaT", "2000-1-1", "NaT", "2000-1-3"])
+            pd.to_datetime(
+                ["NaT", "2000-1-2", "NaT", "2000-1-1", "NaT", "2000-1-3"]
+            ).tz_localize(tz)
         )
 
         expected = pd.Series(
-            pd.to_datetime(["NaT", "2000-1-2", "NaT", "2000-1-2", "NaT", "2000-1-3"])
+            pd.to_datetime(
+                ["NaT", "2000-1-2", "NaT", "2000-1-2", "NaT", "2000-1-3"]
+            ).tz_localize(tz)
         )
         result = s.cummax(skipna=True)
         tm.assert_series_equal(expected, result)
 
         expected = pd.Series(
             pd.to_datetime(
                 ["NaT", "2000-1-2", "2000-1-2", "2000-1-2", "2000-1-2", "2000-1-3"]
-            )
+            ).tz_localize(tz)
         )
         result = s.cummax(skipna=False)
         tm.assert_series_equal(expected, result)
 
-    @pytest.mark.xfail(
-        not _np_version_under1p18, reason="numpy 1.18 changed min/max behavior for NaT"
-    )
     def test_cummin_timedelta64(self):
         s = pd.Series(pd.to_timedelta(["NaT", "2 min", "NaT", "1 min", "NaT", "3 min"]))
 
@@ -125,9 +124,6 @@ def test_cummin_timedelta64(self):
         result = s.cummin(skipna=False)
         tm.assert_series_equal(expected, result)
 
-    @pytest.mark.xfail(
-        not _np_version_under1p18, reason="numpy 1.18 changed min/max behavior for NaT"
-    )
     def test_cummax_timedelta64(self):
         s = pd.Series(pd.to_timedelta(["NaT", "2 min", "NaT", "1 min", "NaT", "3 min"]))