ENH: Parse %z directive in format for to_datetime

return timedeltas as list

return timedeltas in a numpy array

some flake fixes

Extend logic of parsing timezones

address comment misspelling

Add additional tests

address timezone localization

Author: mroeschke

pandas/_libs/tslibs/strptime.pyx

@@ -20,6 +20,7 @@ except:
         except:
             from _dummy_thread import allocate_lock as _thread_allocate_lock
 
+import pytz
 
 from cython cimport Py_ssize_t
 from cpython cimport PyFloat_Check
@@ -29,7 +30,7 @@ cimport cython
 import numpy as np
 from numpy cimport ndarray, int64_t
 
-from datetime import date as datetime_date
+from datetime import date as datetime_date, timedelta as datetime_timedelta
 from cpython.datetime cimport datetime
 
 from np_datetime cimport (check_dts_bounds,
@@ -58,6 +59,8 @@ def array_strptime(ndarray[object] values, object fmt,
         Py_ssize_t i, n = len(values)
         pandas_datetimestruct dts
         ndarray[int64_t] iresult
+        ndarray[object] results_tzoffset
+        ndarray[object] results_tzname
         int year, month, day, minute, hour, second, weekday, julian, tz
         int week_of_year, week_of_year_start
         int64_t us, ns
@@ -109,6 +112,9 @@ def array_strptime(ndarray[object] values, object fmt,
     result = np.empty(n, dtype='M8[ns]')
     iresult = result.view('i8')
 
+    results_tzname = np.empty(n, dtype='object')
+    results_tzoffset = np.empty(n, dtype='object')
+
     dts.us = dts.ps = dts.as = 0
 
     cdef dict _parse_code_table = {
@@ -130,7 +136,8 @@ def array_strptime(ndarray[object] values, object fmt,
         'U': 15,
         'W': 16,
         'Z': 17,
-        'p': 18   # just an additional key, works only with I
+        'p': 18,  # just an additional key, works only with I
+        'z': 19,
     }
     cdef int parse_code
 
@@ -177,6 +184,8 @@ def array_strptime(ndarray[object] values, object fmt,
         month = day = 1
         hour = minute = second = ns = us = 0
         tz = -1
+        gmtoff = None
+        gmtoff_fraction = 0
         # Default to -1 to signify that values not known; not critical to have,
         # though
         week_of_year = -1
@@ -281,6 +290,32 @@ def array_strptime(ndarray[object] values, object fmt,
                         else:
                             tz = value
                             break
+            elif parse_code == 19:
+                z = found_dict['z']
+                if z == 'Z':
+                    gmtoff = 0
+                else:
+                    if z[3] == ':':
+                        z = z[:3] + z[4:]
+                        if len(z) > 5:
+                            if z[5] != ':':
+                                msg = "Inconsistent use of : in {0}"
+                                raise ValueError(msg.format(found_dict['z']))
+                            z = z[:5] + z[6:]
+                    hours = int(z[1:3])
+                    minutes = int(z[3:5])
+                    seconds = int(z[5:7] or 0)
+                    gmtoff = (hours * 60 * 60) + (minutes * 60) + seconds
+                    gmtoff_remainder = z[8:]
+                    # Pad to always return microseconds.
+                    pad_number = 6 - len(gmtoff_remainder)
+                    gmtoff_remainder_padding = "0" * pad_number
+                    gmtoff_fraction = int(gmtoff_remainder +
+                                          gmtoff_remainder_padding)
+                    if z.startswith("-"):
+                        gmtoff = -gmtoff
+                        gmtoff_fraction = -gmtoff_fraction
+
         # If we know the wk of the year and what day of that wk, we can figure
         # out the Julian day of the year.
         if julian == -1 and week_of_year != -1 and weekday != -1:
@@ -330,7 +365,16 @@ def array_strptime(ndarray[object] values, object fmt,
                 continue
             raise
 
-    return result
+        tzname = found_dict.get('Z')
+        if tzname is not None:
+            results_tzname[i] = tzname
+
+        if gmtoff is not None:
+            tzdelta = datetime_timedelta(seconds=gmtoff,
+                                         microseconds=gmtoff_fraction)
+            results_tzoffset[i] = tzdelta
+
+    return result, results_tzname, results_tzoffset
 
 
 """_getlang, LocaleTime, TimeRE, _calc_julian_from_U_or_W are vendored
@@ -538,6 +582,7 @@ class TimeRE(dict):
             # XXX: Does 'Y' need to worry about having less or more than
             #     4 digits?
             'Y': r"(?P<Y>\d\d\d\d)",
+            'z': r"(?P<z>[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)",
             'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
             'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
             'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),

pandas/core/tools/datetimes.py

@@ -1,7 +1,9 @@
 from datetime import datetime, timedelta, time
-import numpy as np
 from collections import MutableMapping
 
+import numpy as np
+import pytz
+
 from pandas._libs import tslib
 from pandas._libs.tslibs.strptime import array_strptime
 from pandas._libs.tslibs import parsing, conversion
@@ -27,6 +29,7 @@
     ABCDataFrame)
 from pandas.core.dtypes.missing import notna
 from pandas.core import algorithms
+from pandas.compat import PY3, zip
 
 
 def _guess_datetime_format_for_array(arr, **kwargs):
@@ -343,8 +346,74 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
                 # fallback
                 if result is None:
                     try:
-                        result = array_strptime(arg, format, exact=exact,
-                                                errors=errors)
+                        parsing_tzname = '%Z' in format
+                        parsing_tzoffset = '%z' in format
+                        if tz is not None and (parsing_tzname or
+                                               parsing_tzoffset):
+                            raise ValueError("Cannot pass a tz argument when "
+                                             "parsing strings with timezone "
+                                             "information.")
+                        result, tznames, tzoffsets = array_strptime(
+                            arg, format, exact=exact, errors=errors)
+                        if parsing_tzname and not parsing_tzoffset:
+                            if len(set(tznames)) == 1:
+                                tz = tznames[0]
+                                if box:
+                                    result = DatetimeIndex(result,
+                                                           tz=tz,
+                                                           name=name)
+                                else:
+                                    stamps = [tslib.Timestamp(res, tz=tz)
+                                              for res in result]
+                                    result = np.array(stamps, dtype=object)
+                            else:
+                                stamps = [tslib.Timestamp(res, tz=tz)
+                                          for res, tz in zip(result, tznames)]
+                                result = np.array(stamps, dtype=object)
+                            return result
+                        elif parsing_tzoffset and not parsing_tzname:
+                            # Should we convert these to pytz.FixedOffsets
+                            # or datetime.timezones?
+                            if len(set(tzoffsets)) == 1:
+                                offset_mins = tzoffsets[0].total_seconds() / 60
+                                tzoffset = pytz.FixedOffset(offset_mins)
+                                if box:
+                                    result = DatetimeIndex(result,
+                                                           tz=tzoffset,
+                                                           name=name)
+                                else:
+                                    stamps = []
+                                    for res, offset in zip(result, tzoffsets):
+                                        ts = tslib.Timestamp(res)
+                                        ts = ts.tz_localize(tzoffset)
+                                        stamps.append(ts)
+                                    result = np.array(stamps, dtype=object)
+                            else:
+                                stamps = []
+                                for res, offset in zip(result, tzoffsets):
+                                    offset_mins = offset.total_seconds() / 60
+                                    tzoffset = pytz.FixedOffset(offset_mins)
+                                    ts = tslib.Timestamp(res)
+                                    ts = ts.tz_localize(tzoffset)
+                                    stamps.append(ts)
+                                result = np.array(stamps, dtype=object)
+                            return result
+                        elif parsing_tzoffset and parsing_tzname:
+                            if not PY3:
+                                raise ValueError("Parsing tzoffsets are not "
+                                                 "not supported in Python 3")
+                            from datetime import timezone
+                            stamps = []
+                            for res, offset, tzname in zip(result, tzoffsets,
+                                                           tznames):
+                                # Do we need to validate these timezones?
+                                # e.g. UTC / +0100
+                                tzinfo = timezone(offset, tzname)
+                                ts = tslib.Timestamp(res, tzinfo=tzinfo)
+                                stamps.append(ts)
+                            result = np.array(stamps, dtype=object)
+                            return result
+
                     except tslib.OutOfBoundsDatetime:
                         if errors == 'raise':
                             raise

pandas/tests/indexes/datetimes/test_tools.py

@@ -8,7 +8,7 @@
 import dateutil
 import numpy as np
 from dateutil.parser import parse
-from datetime import datetime, date, time
+from datetime import datetime, timedelta, date, time
 from distutils.version import LooseVersion
 
 import pandas as pd
@@ -183,6 +183,63 @@ def test_to_datetime_format_weeks(self, cache):
         for s, format, dt in data:
             assert to_datetime(s, format=format, cache=cache) == dt
 
+    @pytest.mark.skipif(not PY3,
+                        reason="datetime.timezone not supported in PY2")
+    def test_to_datetime_parse_timezone(self):
+        from datetime import timezone
+        # %Z parsing only
+        fmt = '%Y-%m-%d %H:%M:%S %Z'
+        dates = ['2010-01-01 12:00:00 UTC'] * 2
+        result = pd.to_datetime(dates, format=fmt)
+        expected_dates = [pd.Timestamp('2010-01-01 12:00:00', tz='UTC')] * 2
+        expected = pd.DatetimeIndex(expected_dates)
+        tm.assert_index_equal(result, expected)
+
+        result = pd.to_datetime(dates, format=fmt, box=False)
+        expected = np.array(expected_dates, dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        dates = ['2010-01-01 12:00:00 UTC', '2010-01-01 12:00:00 GMT']
+        result = pd.to_datetime(dates, format=fmt)
+        expected_dates = [pd.Timestamp('2010-01-01 12:00:00', tz='UTC'),
+                          pd.Timestamp('2010-01-01 12:00:00', tz='GMT')]
+        expected = np.array(expected_dates, dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        # %z parsing only
+        dates = ['2010-01-01 12:00:00 +0100'] * 2
+        fmt = '%Y-%m-%d %H:%M:%S %z'
+        result = pd.to_datetime(dates, format=fmt)
+        expected_dates = [pd.Timestamp('2010-01-01 12:00:00',
+                                       tzinfo=pytz.FixedOffset(60))] * 2
+        expected = pd.DatetimeIndex(expected_dates)
+        tm.assert_index_equal(result, expected)
+
+        result = pd.to_datetime(dates, format=fmt, box=False)
+        expected = np.array(expected_dates, dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        dates = ['2010-01-01 12:00:00 +0100', '2010-01-01 12:00:00 -0100']
+        result = pd.to_datetime(dates, format=fmt)
+        expected_dates = [pd.Timestamp('2010-01-01 12:00:00',
+                                       tzinfo=pytz.FixedOffset(60)),
+                          pd.Timestamp('2010-01-01 12:00:00',
+                                       tzinfo=pytz.FixedOffset(-60))]
+        expected = np.array(expected_dates, dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        # %z and %Z parsing
+        dates = ['2010-01-01 12:00:00 UTC +0100'] * 2
+        fmt = '%Y-%m-%d %H:%M:%S %Z %z'
+        result = pd.to_datetime(dates, format=fmt)
+        tzinfo = timezone(timedelta(minutes=60), 'UTC')
+        expected_dates = [pd.Timestamp('2010-01-01 13:00:00', tzinfo=tzinfo)]
+        expected = np.array(expected_dates * 2, dtype=object)
+        tm.assert_numpy_array_equal(result, expected)
+
+        with pytest.raises(ValueError):
+            pd.to_datetime(dates, format=fmt, utc=True)
+
 
 class TestToDatetime(object):
     def test_to_datetime_pydatetime(self):