BUG GH11693 Support NaT series concatenation

Author: varunkumar-dev

doc/source/whatsnew/v0.18.0.txt

@@ -583,3 +583,5 @@ Bug Fixes
 - Bug in ``.skew`` and ``.kurt`` due to roundoff error for highly similar values (:issue:`11974`)
 
 - Bug in ``buffer_rd_bytes`` src->buffer could be freed more than once if reading failed, causing a segfault (:issue:`12098`)
+- Bug in ``pd.concat`` while concatenating tz-aware NaT series. (:issue:`11693`)
+- Bug in ``pd.concat`` while concatenating tz-aware series with time series. (:issue:`11755`)

pandas/tools/merge.py

@@ -979,8 +979,12 @@ def get_result(self):
 
             # stack blocks
             if self.axis == 0:
-                new_data = com._concat_compat([x._values for x in self.objs])
+                to_concat = [x._values for x in self.objs]
+                typs = com.get_dtype_kinds(to_concat)
+                new_data = com._concat_compat(to_concat)
                 name = com._consensus_name_attr(self.objs)
+                if 'datetimetz' in typs and ('datetime' in typs or 'object' in typs):
+                    return Series(new_data, index=self.new_axes[0], name=name, dtype='object').__finalize__(self, method='concat')
                 return (Series(new_data, index=self.new_axes[0], name=name)
                         .__finalize__(self, method='concat'))
 

pandas/tools/tests/test_merge.py

@@ -1024,6 +1024,47 @@ def test_merge_on_datetime64tz(self):
         result = pd.merge(left, right, on='key', how='outer')
         assert_frame_equal(result, expected)
 
+    def test_concat_Nat_series(self):
+        # GH 11693
+        # test for merging NaT series with datetime series.
+        x = pd.Series( pd.date_range('20151124 08:00', '20151124 09:00', freq='1h', tz = "US/Eastern"))
+        y = pd.Series( pd.date_range('20151124 10:00', '20151124 11:00', freq='1h', tz = "US/Eastern"))
+        y[:] = pd.NaT
+        expected = pd.Series([x[0], x[1], pd.NaT, pd.NaT], index=[0, 1, 0, 1])
+        tm.assert_series_equal(pd.concat([x,y]), expected)
+
+        # all NaT with tz
+        x[:] = pd.NaT
+        expected = pd.Series([pd.NaT for i in range(4)], index=[0, 1, 0, 1], dtype ='datetime64[ns, US/Eastern]')
+        tm.assert_series_equal(pd.concat([x,y]), expected)
+
+        #without tz
+        x = pd.Series( pd.date_range('20151124 08:00', '20151124 09:00', freq='1h'))
+        y = pd.Series( pd.date_range('20151124 10:00', '20151124 11:00', freq='1h'))
+        y[:] = pd.NaT
+        expected = pd.Series([x[0], x[1], pd.NaT, pd.NaT], index=[0, 1, 0, 1])
+        tm.assert_series_equal(pd.concat([x, y]), expected)
+
+        #all NaT without tz
+        x[:] = pd.NaT
+        expected = pd.Series([pd.NaT for i in range(4)], index=[0, 1, 0, 1], dtype ='datetime64[ns]')
+        tm.assert_series_equal(pd.concat([x,y]), expected)
+
+    def test_concat_tz_series(self):
+        #tz and no tz
+        #GH 11755
+        x = pd.Series(pd.date_range('20151124 08:00', '20151124 09:00', freq = '1h', tz = "UTC") )
+        y = pd.Series(pd.date_range('2012-01-01', '2012-01-02'))
+        expected = pd.Series([x[0], x[1], y[0], y[1]], index=[0, 1, 0, 1], dtype='object')
+        tm.assert_series_equal(pd.concat([x,y]), expected)
+
+        #tz and object
+        #GH 11887
+        x = pd.Series(pd.date_range('20151124 08:00', '20151124 09:00', freq = '1h', tz = "UTC") )
+        y = pd.Series(['a', 'b'])
+        expected = pd.Series([x[0], x[1], y[0], y[1]], index=[0, 1, 0, 1], dtype='object')
+        tm.assert_series_equal(pd.concat([x,y]), expected)
+
     def test_indicator(self):
         # PR #10054. xref #7412 and closes #8790.
         df1 = DataFrame({'col1': [0, 1], 'col_left': [

pandas/tseries/common.py

@@ -255,14 +255,15 @@ def _concat_compat(to_concat, axis=0):
 
     def convert_to_pydatetime(x, axis):
         # coerce to an object dtype
-        if x.dtype == _NS_DTYPE:
-
-            if hasattr(x, 'tz'):
-                x = x.asobject
 
+        # if dtype is of datetimetz or timezone
+        if x.dtype.kind == _NS_DTYPE.kind:
             shape = x.shape
             x = tslib.ints_to_pydatetime(x.view(np.int64).ravel())
             x = x.reshape(shape)
+            if hasattr(x, 'tz'):
+                x = x.asobject
+
         elif x.dtype == _TD_DTYPE:
             shape = x.shape
             x = tslib.ints_to_pytimedelta(x.view(np.int64).ravel())
@@ -275,6 +276,11 @@ def convert_to_pydatetime(x, axis):
     # datetimetz
     if 'datetimetz' in typs:
 
+        # if to_concat have 'datetime' or 'object', then we need to coerce to object
+        if 'datetime' in typs or 'object' in typs:
+            to_concat = [convert_to_pydatetime(x, axis) for x in to_concat]
+            return np.concatenate(to_concat,axis=axis)
+
         # we require ALL of the same tz for datetimetz
         tzs = set([getattr(x, 'tz', None) for x in to_concat]) - set([None])
         if len(tzs) == 1:

pandas/tslib.pyx

@@ -3554,6 +3554,10 @@ def tz_convert(ndarray[int64_t] vals, object tz1, object tz2):
     trans, deltas, typ = _get_dst_info(tz2)
     trans_len = len(trans)
 
+    #if all NaT, return all NaT
+    if (utc_dates==iNaT).all():
+        return utc_dates
+
     # use first non-NaT element
     # if all-NaT, return all-NaT
     if (result==NPY_NAT).all():