PERF: timedelta and datetime64 ops improvements

Author: jreback

doc/source/whatsnew/v0.17.0.txt

@@ -55,6 +55,7 @@ Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
 - 4x improvement in ``timedelta`` string parsing (:issue:`6755`)
+- 8x improvement in ``timedelta64`` and ``datetime64`` ops (:issue:`6755`)
 
 .. _whatsnew_0170.bug_fixes:
 

pandas/tseries/timedeltas.py

@@ -34,19 +34,13 @@ def _convert_listlike(arg, box, unit):
         if isinstance(arg, (list,tuple)) or ((hasattr(arg,'__iter__') and not hasattr(arg,'dtype'))):
             arg = np.array(list(arg), dtype='O')
 
+        # these are shortcutable
         if is_timedelta64_dtype(arg):
             value = arg.astype('timedelta64[ns]')
         elif is_integer_dtype(arg):
-
-            # these are shortcutable
-            value = arg.astype('timedelta64[{0}]'.format(unit)).astype('timedelta64[ns]')
+            value = arg.astype('timedelta64[{0}]'.format(unit)).astype('timedelta64[ns]', copy=False)
         else:
-            try:
-                value = tslib.array_to_timedelta64(_ensure_object(arg), unit=unit, coerce=coerce)
-            except:
-
-                # try to process strings fast; may need to fallback
-                value = np.array([ _coerce_scalar_to_timedelta_type(r, unit=unit, coerce=coerce) for r in arg ])
+            value = tslib.array_to_timedelta64(_ensure_object(arg), unit=unit, coerce=coerce)
             value = value.astype('timedelta64[ns]', copy=False)
 
         if box:

pandas/tseries/tools.py

@@ -263,6 +263,7 @@ def _convert_listlike(arg, box, format):
         if isinstance(arg, (list,tuple)):
             arg = np.array(arg, dtype='O')
 
+        # these are shortcutable
         if com.is_datetime64_ns_dtype(arg):
             if box and not isinstance(arg, DatetimeIndex):
                 try:
@@ -271,6 +272,12 @@ def _convert_listlike(arg, box, format):
                     pass
 
             return arg
+        elif format is None and com.is_integer_dtype(arg) and unit=='ns':
+            result = arg.astype('datetime64[ns]')
+            if box:
+                return DatetimeIndex(result, tz='utc' if utc else None)
+
+            return result
 
         arg = com._ensure_object(arg)
 

pandas/tslib.pyx

@@ -5,15 +5,18 @@ from numpy cimport (int8_t, int32_t, int64_t, import_array, ndarray,
                     NPY_INT64, NPY_DATETIME, NPY_TIMEDELTA)
 import numpy as np
 
+# GH3363
+from sys import version_info
+cdef bint PY2 = version_info[0] == 2
+cdef bint PY3 = not PY2
+
 from cpython cimport (
     PyTypeObject,
     PyFloat_Check,
     PyLong_Check,
     PyObject_RichCompareBool,
     PyObject_RichCompare,
     PyString_Check,
-    PyUnicode_Contains,
-    PyString_AsString,
     Py_GT, Py_GE, Py_EQ, Py_NE, Py_LT, Py_LE
 )
 
@@ -50,15 +53,11 @@ else:
     from dateutil.tz import gettz as _dateutil_gettz
 
 from pytz.tzinfo import BaseTzInfo as _pytz_BaseTzInfo
-from pandas.compat import parse_date, string_types, PY3, iteritems
+from pandas.compat import parse_date, string_types, iteritems
 
-from sys import version_info
 import operator
 import collections
 
-# GH3363
-cdef bint PY2 = version_info[0] == 2
-
 # initialize numpy
 import_array()
 #import_ufunc()
@@ -2291,12 +2290,10 @@ cdef inline parse_timedelta_string(object ts, coerce=False):
     """
 
     cdef:
-        char c
-        bytes bc
+        str c
         bint neg=0, have_dot=0, have_value=0, have_hhmmss=0
         object current_unit=None
-        Py_ssize_t i
-        int64_t result=0, m, r
+        int64_t result=0, m=0, r
         list number=[], frac=[], unit=[]
 
     # neg : tracks if we have a leading negative for the value
@@ -2307,39 +2304,38 @@ cdef inline parse_timedelta_string(object ts, coerce=False):
     if ts in _nat_strings or not len(ts):
         return iNaT
 
-    for c in PyString_AsString(ts):
-        bc = <bytes>c
+    for c in ts:
 
         # skip whitespace / commas
-        if bc == ' ' or bc == ',':
+        if c == ' ' or c == ',':
             pass
 
         # positive signs are ignored
-        elif bc == '+':
+        elif c == '+':
             pass
 
         # neg
-        elif bc == '-':
+        elif c == '-':
 
             if neg or have_value or have_hhmmss:
                 raise ValueError("only leading negative signs are allowed")
 
             neg = 1
 
         # number (ascii codes)
-        elif c >= 48 and c <= 57:
+        elif ord(c) >= 48 and ord(c) <= 57:
 
             if have_dot:
 
                 # we found a dot, but now its just a fraction
                 if len(unit):
-                    number.append(bc)
+                    number.append(c)
                     have_dot = 0
                 else:
-                    frac.append(bc)
+                    frac.append(c)
 
             elif not len(unit):
-                number.append(bc)
+                number.append(c)
 
             else:
 
@@ -2349,12 +2345,12 @@ cdef inline parse_timedelta_string(object ts, coerce=False):
                     if coerce:
                         return iNaT
                     raise
-                unit, number, frac = [], [bc], []
+                unit, number, frac = [], [c], []
 
                 result += timedelta_as_neg(r, neg)
 
         # hh:mm:ss.
-        elif bc == ':':
+        elif c == ':':
 
             # we flip this off if we have a leading value
             if have_value:
@@ -2381,7 +2377,7 @@ cdef inline parse_timedelta_string(object ts, coerce=False):
             unit, number = [], []
 
         # after the decimal point
-        elif bc == '.':
+        elif c == '.':
 
             if len(number) and current_unit is not None:
 
@@ -2400,7 +2396,7 @@ cdef inline parse_timedelta_string(object ts, coerce=False):
 
         # unit
         else:
-            unit.append(bc)
+            unit.append(c)
             have_value = 1
             have_dot = 0