COMPAT: numpy_compat for >= 1.11 for np.datetime64 tz changes, #12100

closes #12100

Author: Jeff Reback <jeff@reback.net>

Closes #12127 from jreback/dt64_compat and squashes the following commits:

acb7cfd [Jeff Reback] COMPAT: numpy_compat for >= 1.11 for np.datetime64 tz changes, #12100

Author: jreback

pandas/__init__.py

@@ -4,6 +4,15 @@
 
 __docformat__ = 'restructuredtext'
 
+# use the closest tagged version if possible
+from ._version import get_versions
+v = get_versions()
+__version__ = v.get('closest-tag',v['version'])
+del get_versions, v
+
+# numpy compat
+from pandas.compat.numpy_compat import *
+
 try:
     from pandas import hashtable, tslib, lib
 except ImportError as e:  # pragma: no cover
@@ -16,29 +25,8 @@
 from datetime import datetime
 import numpy as np
 
-
-# XXX: HACK for NumPy 1.5.1 to suppress warnings
-try:
-    np.seterr(all='ignore')
-except Exception:  # pragma: no cover
-    pass
-
-# numpy versioning
-from distutils.version import LooseVersion
-_np_version = np.version.short_version
-_np_version_under1p8 = LooseVersion(_np_version) < '1.8'
-_np_version_under1p9 = LooseVersion(_np_version) < '1.9'
-
-
 from pandas.info import __doc__
 
-
-if LooseVersion(_np_version) < '1.7.0':
-    raise ImportError('pandas {0} is incompatible with numpy < 1.7.0, '
-                      'your numpy version is {1}. Please upgrade numpy to'
-                      ' >= 1.7.0 to use pandas version {0}'.format(__version__,
-                                                                   _np_version))
-
 # let init-time option registration happen
 import pandas.core.config_init
 
@@ -62,9 +50,3 @@
 from pandas.util.nosetester import NoseTester
 test = NoseTester().test
 del NoseTester
-
-# use the closest tagged version if possible
-from ._version import get_versions
-v = get_versions()
-__version__ = v.get('closest-tag',v['version'])
-del get_versions, v

pandas/compat/numpy_compat.py

@@ -0,0 +1,74 @@
+""" support numpy compatiblitiy across versions """
+
+from distutils.version import LooseVersion
+from pandas.compat import string_types, string_and_binary_types
+import numpy as np
+
+# TODO: HACK for NumPy 1.5.1 to suppress warnings
+# is this necessary?
+try:
+    np.seterr(all='ignore')
+except Exception:  # pragma: no cover
+    pass
+
+# numpy versioning
+_np_version = np.version.short_version
+_np_version_under1p8 = LooseVersion(_np_version) < '1.8'
+_np_version_under1p9 = LooseVersion(_np_version) < '1.9'
+_np_version_under1p10 = LooseVersion(_np_version) < '1.10'
+_np_version_under1p11 = LooseVersion(_np_version) < '1.11'
+
+if LooseVersion(_np_version) < '1.7.0':
+    from pandas import __version__
+    raise ImportError('pandas {0} is incompatible with numpy < 1.7.0\n'
+                      'your numpy version is {1}.\n'
+                      'Please upgrade numpy to >= 1.7.0 to use '
+                      'this pandas version'.format(__version__, _np_version))
+
+
+def tz_replacer(s):
+    if isinstance(s, string_types):
+        if s.endswith('Z'):
+            s = s[:-1]
+        elif s.endswith('-0000'):
+            s = s[:-5]
+    return s
+
+
+def np_datetime64_compat(s, *args, **kwargs):
+    """
+    provide compat for construction of strings to numpy datetime64's with
+    tz-changes in 1.11 that make '2015-01-01 09:00:00Z' show a deprecation
+    warning, when need to pass '2015-01-01 09:00:00'
+    """
+
+    if not _np_version_under1p11:
+        s = tz_replacer(s)
+    return np.datetime64(s, *args, **kwargs)
+
+
+def np_array_datetime64_compat(arr, *args, **kwargs):
+    """
+    provide compat for construction of an array of strings to a
+    np.array(..., dtype=np.datetime64(..))
+    tz-changes in 1.11 that make '2015-01-01 09:00:00Z' show a deprecation
+    warning, when need to pass '2015-01-01 09:00:00'
+    """
+
+    if not _np_version_under1p11:
+
+        # is_list_like
+        if hasattr(arr, '__iter__') and not \
+           isinstance(arr, string_and_binary_types):
+            arr = [tz_replacer(s) for s in arr]
+        else:
+            arr = tz_replacer(s)
+
+    return np.array(arr, *args, **kwargs)
+
+__all__ = ['_np_version',
+           '_np_version_under1p8',
+           '_np_version_under1p9',
+           '_np_version_under1p10',
+           '_np_version_under1p11',
+           ]

pandas/io/tests/test_date_converters.py

@@ -10,6 +10,7 @@
 from pandas.util.testing import assert_frame_equal
 import pandas.io.date_converters as conv
 import pandas.util.testing as tm
+from pandas.compat.numpy_compat import np_array_datetime64_compat
 
 
 class TestConverters(tm.TestCase):
@@ -119,15 +120,16 @@ def test_dateparser_resolution_if_not_ns(self):
 """
 
         def date_parser(date, time):
-            datetime = np.array(date + 'T' + time + 'Z', dtype='datetime64[s]')
+            datetime = np_array_datetime64_compat(
+                date + 'T' + time + 'Z', dtype='datetime64[s]')
             return datetime
 
         df = read_csv(StringIO(data), date_parser=date_parser,
                       parse_dates={'datetime': ['date', 'time']},
                       index_col=['datetime', 'prn'])
 
-        datetimes = np.array(['2013-11-03T19:00:00Z'] * 3,
-                             dtype='datetime64[s]')
+        datetimes = np_array_datetime64_compat(['2013-11-03T19:00:00Z'] * 3,
+                                               dtype='datetime64[s]')
         df_correct = DataFrame(data={'rxstatus': ['00E80000'] * 3},
                                index=MultiIndex.from_tuples(
                                    [(datetimes[0], 126),

pandas/tseries/tests/test_offsets.py

@@ -8,6 +8,7 @@
 
 import numpy as np
 
+from pandas.compat.numpy_compat import np_datetime64_compat
 from pandas.core.datetools import (bday, BDay, CDay, BQuarterEnd, BMonthEnd,
                                    BusinessHour, CBMonthEnd, CBMonthBegin,
                                    BYearEnd, MonthEnd, MonthBegin, BYearBegin,
@@ -201,7 +202,7 @@ def setUp(self):
                           'Second': Timestamp('2011-01-01 09:00:01'),
                           'Milli': Timestamp('2011-01-01 09:00:00.001000'),
                           'Micro': Timestamp('2011-01-01 09:00:00.000001'),
-                          'Nano': Timestamp(np.datetime64(
+                          'Nano': Timestamp(np_datetime64_compat(
                               '2011-01-01T09:00:00.000000001Z'))}
 
     def test_return_type(self):
@@ -292,7 +293,7 @@ def _check_offsetfunc_works(self, offset, funcname, dt, expected,
 
     def test_apply(self):
         sdt = datetime(2011, 1, 1, 9, 0)
-        ndt = np.datetime64('2011-01-01 09:00Z')
+        ndt = np_datetime64_compat('2011-01-01 09:00Z')
 
         for offset in self.offset_types:
             for dt in [sdt, ndt]:
@@ -333,7 +334,7 @@ def test_rollforward(self):
         norm_expected.update(normalized)
 
         sdt = datetime(2011, 1, 1, 9, 0)
-        ndt = np.datetime64('2011-01-01 09:00Z')
+        ndt = np_datetime64_compat('2011-01-01 09:00Z')
 
         for offset in self.offset_types:
             for dt in [sdt, ndt]:
@@ -391,7 +392,7 @@ def test_rollback(self):
         norm_expected.update(normalized)
 
         sdt = datetime(2011, 1, 1, 9, 0)
-        ndt = np.datetime64('2011-01-01 09:00Z')
+        ndt = np_datetime64_compat('2011-01-01 09:00Z')
 
         for offset in self.offset_types:
             for dt in [sdt, ndt]:
@@ -1394,7 +1395,7 @@ class TestCustomBusinessDay(Base):
 
     def setUp(self):
         self.d = datetime(2008, 1, 1)
-        self.nd = np.datetime64('2008-01-01 00:00:00Z')
+        self.nd = np_datetime64_compat('2008-01-01 00:00:00Z')
 
         tm._skip_if_no_cday()
         self.offset = CDay()

pandas/tseries/tests/test_period.py

@@ -22,14 +22,14 @@
 import pandas as pd
 import numpy as np
 from numpy.random import randn
-from pandas.compat import range, lrange, lmap, zip
+from pandas.compat import range, lrange, lmap, zip, text_type, PY3
+from pandas.compat.numpy_compat import np_datetime64_compat
 
 from pandas import Series, DataFrame, _np_version_under1p9
 from pandas import tslib
 from pandas.util.testing import (assert_series_equal, assert_almost_equal,
                                  assertRaisesRegexp)
 import pandas.util.testing as tm
-from pandas import compat
 
 
 class TestPeriodProperties(tm.TestCase):
@@ -329,8 +329,8 @@ def test_period_constructor(self):
         i1 = Period(date(2007, 1, 1), freq='M')
         i2 = Period(datetime(2007, 1, 1), freq='M')
         i3 = Period(np.datetime64('2007-01-01'), freq='M')
-        i4 = Period(np.datetime64('2007-01-01 00:00:00Z'), freq='M')
-        i5 = Period(np.datetime64('2007-01-01 00:00:00.000Z'), freq='M')
+        i4 = Period(np_datetime64_compat('2007-01-01 00:00:00Z'), freq='M')
+        i5 = Period(np_datetime64_compat('2007-01-01 00:00:00.000Z'), freq='M')
         self.assertEqual(i1, i2)
         self.assertEqual(i1, i3)
         self.assertEqual(i1, i4)
@@ -340,14 +340,15 @@ def test_period_constructor(self):
         expected = Period(datetime(2007, 1, 1, 9, 0, 0, 1000), freq='L')
         self.assertEqual(i1, expected)
 
-        expected = Period(np.datetime64('2007-01-01 09:00:00.001Z'), freq='L')
+        expected = Period(np_datetime64_compat(
+            '2007-01-01 09:00:00.001Z'), freq='L')
         self.assertEqual(i1, expected)
 
         i1 = Period('2007-01-01 09:00:00.00101')
         expected = Period(datetime(2007, 1, 1, 9, 0, 0, 1010), freq='U')
         self.assertEqual(i1, expected)
 
-        expected = Period(np.datetime64('2007-01-01 09:00:00.00101Z'),
+        expected = Period(np_datetime64_compat('2007-01-01 09:00:00.00101Z'),
                           freq='U')
         self.assertEqual(i1, expected)
 
@@ -406,8 +407,8 @@ def test_period_constructor_offsets(self):
         i1 = Period(date(2007, 1, 1), freq='M')
         i2 = Period(datetime(2007, 1, 1), freq='M')
         i3 = Period(np.datetime64('2007-01-01'), freq='M')
-        i4 = Period(np.datetime64('2007-01-01 00:00:00Z'), freq='M')
-        i5 = Period(np.datetime64('2007-01-01 00:00:00.000Z'), freq='M')
+        i4 = Period(np_datetime64_compat('2007-01-01 00:00:00Z'), freq='M')
+        i5 = Period(np_datetime64_compat('2007-01-01 00:00:00.000Z'), freq='M')
         self.assertEqual(i1, i2)
         self.assertEqual(i1, i3)
         self.assertEqual(i1, i4)
@@ -417,14 +418,15 @@ def test_period_constructor_offsets(self):
         expected = Period(datetime(2007, 1, 1, 9, 0, 0, 1000), freq='L')
         self.assertEqual(i1, expected)
 
-        expected = Period(np.datetime64('2007-01-01 09:00:00.001Z'), freq='L')
+        expected = Period(np_datetime64_compat(
+            '2007-01-01 09:00:00.001Z'), freq='L')
         self.assertEqual(i1, expected)
 
         i1 = Period('2007-01-01 09:00:00.00101')
         expected = Period(datetime(2007, 1, 1, 9, 0, 0, 1010), freq='U')
         self.assertEqual(i1, expected)
 
-        expected = Period(np.datetime64('2007-01-01 09:00:00.00101Z'),
+        expected = Period(np_datetime64_compat('2007-01-01 09:00:00.00101Z'),
                           freq='U')
         self.assertEqual(i1, expected)
 
@@ -462,7 +464,7 @@ def test_strftime(self):
         p = Period('2000-1-1 12:34:12', freq='S')
         res = p.strftime('%Y-%m-%d %H:%M:%S')
         self.assertEqual(res, '2000-01-01 12:34:12')
-        tm.assertIsInstance(res, compat.text_type)  # GH3363
+        tm.assertIsInstance(res, text_type)  # GH3363
 
     def test_sub_delta(self):
         left, right = Period('2011', freq='A'), Period('2007', freq='A')
@@ -2957,9 +2959,9 @@ def test_map_with_string_constructor(self):
         index = PeriodIndex(raw, freq='A')
         types = str,
 
-        if compat.PY3:
+        if PY3:
             # unicode
-            types += compat.text_type,
+            types += text_type,
 
         for t in types:
             expected = np.array(lmap(t, raw), dtype=object)

pandas/tseries/tests/test_timeseries.py

@@ -15,6 +15,7 @@
                     Period, DatetimeIndex, Int64Index, to_datetime,
                     bdate_range, Float64Index, NaT, timedelta_range, Timedelta)
 
+from pandas.compat.numpy_compat import np_datetime64_compat
 import pandas.core.datetools as datetools
 import pandas.tseries.offsets as offsets
 import pandas.tseries.tools as tools
@@ -2496,11 +2497,11 @@ def test_comparisons_nat(self):
                                   '2014-05-01', '2014-07-01'])
         didx2 = pd.DatetimeIndex(['2014-02-01', '2014-03-01', pd.NaT, pd.NaT,
                                   '2014-06-01', '2014-07-01'])
-        darr = np.array([np.datetime64('2014-02-01 00:00Z'),
-                         np.datetime64('2014-03-01 00:00Z'),
-                         np.datetime64('nat'), np.datetime64('nat'),
-                         np.datetime64('2014-06-01 00:00Z'),
-                         np.datetime64('2014-07-01 00:00Z')])
+        darr = np.array([np_datetime64_compat('2014-02-01 00:00Z'),
+                         np_datetime64_compat('2014-03-01 00:00Z'),
+                         np_datetime64_compat('nat'), np.datetime64('nat'),
+                         np_datetime64_compat('2014-06-01 00:00Z'),
+                         np_datetime64_compat('2014-07-01 00:00Z')])
 
         if _np_version_under1p8:
             # cannot test array because np.datetime('nat') returns today's date

pandas/tseries/tests/test_tslib.py

@@ -15,6 +15,9 @@
 import pandas.tseries.offsets as offsets
 import pandas.util.testing as tm
 import pandas.compat as compat
+from pandas.compat.numpy_compat import (np_datetime64_compat,
+                                        np_array_datetime64_compat)
+
 from pandas.util.testing import assert_series_equal, _skip_if_has_locale
 
 
@@ -694,7 +697,7 @@ def test_parsing_valid_dates(self):
         arr = np.array(['01-01-2013', '01-02-2013'], dtype=object)
         self.assert_numpy_array_equal(
             tslib.array_to_datetime(arr),
-            np.array(
+            np_array_datetime64_compat(
                 [
                     '2013-01-01T00:00:00.000000000-0000',
                     '2013-01-02T00:00:00.000000000-0000'
@@ -706,7 +709,7 @@ def test_parsing_valid_dates(self):
         arr = np.array(['Mon Sep 16 2013', 'Tue Sep 17 2013'], dtype=object)
         self.assert_numpy_array_equal(
             tslib.array_to_datetime(arr),
-            np.array(
+            np_array_datetime64_compat(
                 [
                     '2013-09-16T00:00:00.000000000-0000',
                     '2013-09-17T00:00:00.000000000-0000'
@@ -752,7 +755,7 @@ def test_coercing_dates_outside_of_datetime64_ns_bounds(self):
         arr = np.array(['1/1/1000', '1/1/2000'], dtype=object)
         self.assert_numpy_array_equal(
             tslib.array_to_datetime(arr, errors='coerce'),
-            np.array(
+            np_array_datetime64_compat(
                 [
                     tslib.iNaT,
                     '2000-01-01T00:00:00.000000000-0000'
@@ -772,7 +775,7 @@ def test_coerce_of_invalid_datetimes(self):
         # With coercing, the invalid dates becomes iNaT
         self.assert_numpy_array_equal(
             tslib.array_to_datetime(arr, errors='coerce'),
-            np.array(
+            np_array_datetime64_compat(
                 [
                     '2013-01-01T00:00:00.000000000-0000',
                     tslib.iNaT,
@@ -863,7 +866,7 @@ def test_nanosecond_timestamp(self):
         self.assertEqual(t.value, expected)
         self.assertEqual(t.nanosecond, 5)
 
-        t = Timestamp(np.datetime64('2011-01-01 00:00:00.000000005Z'))
+        t = Timestamp(np_datetime64_compat('2011-01-01 00:00:00.000000005Z'))
         self.assertEqual(repr(t), "Timestamp('2011-01-01 00:00:00.000000005')")
         self.assertEqual(t.value, expected)
         self.assertEqual(t.nanosecond, 5)
@@ -879,7 +882,7 @@ def test_nanosecond_timestamp(self):
         self.assertEqual(t.value, expected)
         self.assertEqual(t.nanosecond, 10)
 
-        t = Timestamp(np.datetime64('2011-01-01 00:00:00.000000010Z'))
+        t = Timestamp(np_datetime64_compat('2011-01-01 00:00:00.000000010Z'))
         self.assertEqual(repr(t), "Timestamp('2011-01-01 00:00:00.000000010')")
         self.assertEqual(t.value, expected)
         self.assertEqual(t.nanosecond, 10)