PERF: improve perf of writing csv's with datetimes

Author: jreback

doc/source/whatsnew/v0.16.1.txt

@@ -89,7 +89,7 @@ API changes
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-
+- Improved csv write performance with mixed dtypes, including datetimes
 
 
 

pandas/core/internals.py

@@ -1870,23 +1870,12 @@ def to_native_types(self, slicer=None, na_rep=None, date_format=None,
         values = self.values
         if slicer is not None:
             values = values[:, slicer]
-        mask = isnull(values)
 
-        rvalues = np.empty(values.shape, dtype=object)
-        if na_rep is None:
-            na_rep = 'NaT'
-        rvalues[mask] = na_rep
-        imask = (~mask).ravel()
-
-        if date_format is None:
-            date_formatter = lambda x: Timestamp(x)._repr_base
-        else:
-            date_formatter = lambda x: Timestamp(x).strftime(date_format)
-
-        rvalues.flat[imask] = np.array([date_formatter(val) for val in
-                                        values.ravel()[imask]], dtype=object)
-
-        return rvalues.tolist()
+        result = tslib.format_array_from_datetime(values.view('i8').ravel(),
+                                                  tz=None,
+                                                  format=date_format,
+                                                  na_rep=na_rep).reshape(values.shape)
+        return result.tolist()
 
     def should_store(self, value):
         return issubclass(value.dtype.type, np.datetime64)

pandas/tslib.pyx

@@ -1398,6 +1398,62 @@ def parse_datetime_string(date_string, **kwargs):
     dt = parse_date(date_string, **kwargs)
     return dt
 
+def format_array_from_datetime(ndarray[int64_t] values, object tz=None, object format=None, object na_rep=None):
+    """
+    return a np object array of the string formatted values
+
+    Parameters
+    ----------
+    values : a 1-d i8 array
+    tz : the timezone (or None)
+    format : optional, default is None
+          a strftime capable string
+    na_rep : optional, default is None
+          a nat format
+
+    """
+    cdef int64_t val, ns, N = len(values)
+    cdef ndarray[object] result = np.empty(N, dtype=object)
+    cdef object ts, res
+    cdef _TSObject obj
+
+    obj = _TSObject()
+
+    if na_rep is None:
+       na_rep = 'NaT'
+
+    for i in range(N):
+       val = values[i]
+
+       if val == iNaT:
+          result[i] = na_rep
+       else:
+          if format is None and tz is None:
+
+            obj.value = val
+            pandas_datetime_to_datetimestruct(val, PANDAS_FR_ns, &obj.dts)
+            res = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (obj.dts.year,
+                                                   obj.dts.month,
+                                                   obj.dts.day,
+                                                   obj.dts.hour,
+                                                   obj.dts.min,
+                                                   obj.dts.sec)
+
+            ns = obj.dts.ps / 1000
+
+            if ns != 0:
+               res += '.%.9d' % (ns + 1000 * obj.dts.us)
+            elif obj.dts.us != 0:
+               res += '.%.6d' % obj.dts.us
+
+            result[i] = res
+
+          else:
+             ts = Timestamp(val, tz=tz)
+             result[i] = ts.strftime(format)
+
+    return result
+
 def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False,
                       format=None, utc=None, coerce=False, unit=None):
     cdef: