Merge pull request #8298 from jreback/iat

BUG: Bug in iat return boxing for Timestamp/Timedelta (GH7729)

Author: jreback

pandas/core/api.py

@@ -25,6 +25,7 @@
 from pandas.tseries.tools import to_datetime
 from pandas.tseries.index import (DatetimeIndex, Timestamp,
                                   date_range, bdate_range)
+from pandas.tseries.tdi import TimedeltaIndex, Timedelta
 from pandas.tseries.period import Period, PeriodIndex
 
 # legacy

pandas/core/common.py

@@ -1827,9 +1827,8 @@ def _possibly_convert_objects(values, convert_dates=True,
     if convert_timedeltas and values.dtype == np.object_:
 
         if convert_timedeltas == 'coerce':
-            from pandas.tseries.timedeltas import \
-                 _possibly_cast_to_timedelta
-            values = _possibly_cast_to_timedelta(values, coerce=True)
+            from pandas.tseries.timedeltas import to_timedelta
+            values = to_timedelta(values, coerce=True)
 
             # if we are all nans then leave me alone
             if not isnull(new_values).all():
@@ -1889,7 +1888,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
     """ try to cast the array/value to a datetimelike dtype, converting float
     nan to iNaT
     """
-    from pandas.tseries.timedeltas import _possibly_cast_to_timedelta
+    from pandas.tseries.timedeltas import to_timedelta
     from pandas.tseries.tools import to_datetime
 
     if dtype is not None:
@@ -1931,8 +1930,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
                         if is_datetime64:
                             value = to_datetime(value, coerce=coerce).values
                         elif is_timedelta64:
-                            value = _possibly_cast_to_timedelta(value,
-                                                                dtype=dtype)
+                            value = to_timedelta(value, coerce=coerce).values
                     except (AttributeError, ValueError):
                         pass
 
@@ -1949,7 +1947,7 @@ def _possibly_cast_to_datetime(value, dtype, coerce=False):
                 value = value.astype(_NS_DTYPE)
 
             elif dtype.kind == 'm' and dtype != _TD_DTYPE:
-                value = _possibly_cast_to_timedelta(value)
+                value = to_timedelta(value)
 
         # only do this if we have an array and the dtype of the array is not
         # setup already we are not an integer/object, so don't bother with this
@@ -2005,16 +2003,7 @@ def _try_timedelta(v):
             try:
                 return to_timedelta(v).values.reshape(shape)
             except:
-
-                # this is for compat with numpy < 1.7
-                # but string-likes will fail here
-
-                from pandas.tseries.timedeltas import \
-                     _possibly_cast_to_timedelta
-                try:
-                    return _possibly_cast_to_timedelta(v, coerce='compat').reshape(shape)
-                except:
-                    return v
+                return v
 
         # do a quick inference for perf
         sample = v[:min(3,len(v))]

pandas/core/ops.py

@@ -314,7 +314,7 @@ def _validate(self):
 
     def _convert_to_array(self, values, name=None, other=None):
         """converts values to ndarray"""
-        from pandas.tseries.timedeltas import _possibly_cast_to_timedelta
+        from pandas.tseries.timedeltas import to_timedelta
 
         coerce = True
         if not is_list_like(values):
@@ -337,7 +337,7 @@ def _convert_to_array(self, values, name=None, other=None):
                 values = tslib.array_to_datetime(values)
         elif inferred_type in ('timedelta', 'timedelta64'):
             # have a timedelta, convert to to ns here
-            values = _possibly_cast_to_timedelta(values, coerce=coerce, dtype='timedelta64[ns]')
+            values = to_timedelta(values, coerce=coerce)
         elif inferred_type == 'integer':
             # py3 compat where dtype is 'm' but is an integer
             if values.dtype.kind == 'm':
@@ -356,7 +356,7 @@ def _convert_to_array(self, values, name=None, other=None):
                                 "datetime/timedelta operations [{0}]".format(
                                     ', '.join([com.pprint_thing(v)
                                                for v in values[mask]])))
-            values = _possibly_cast_to_timedelta(os, coerce=coerce)
+            values = to_timedelta(os, coerce=coerce)
         elif inferred_type == 'floating':
 
             # all nan, so ok, use the other dtype (e.g. timedelta or datetime)

pandas/core/series.py

@@ -20,7 +20,8 @@
                                 _possibly_cast_to_datetime, _possibly_castable,
                                 _possibly_convert_platform, _try_sort,
                                 ABCSparseArray, _maybe_match_name, _coerce_to_dtype,
-                                _ensure_object, SettingWithCopyError)
+                                _ensure_object, SettingWithCopyError,
+                                _maybe_box_datetimelike)
 from pandas.core.index import (Index, MultiIndex, InvalidIndexError,
                                _ensure_index)
 from pandas.core.indexing import _check_bool_indexer, _maybe_convert_indices
@@ -781,7 +782,7 @@ def get_value(self, label, takeable=False):
         value : scalar value
         """
         if takeable is True:
-            return self.values[label]
+            return _maybe_box_datetimelike(self.values[label])
         return self.index.get_value(self.values, label)
 
     def set_value(self, label, value, takeable=False):

pandas/tests/test_indexing.py

@@ -14,7 +14,7 @@
 import pandas.core.common as com
 from pandas import option_context
 from pandas.core.api import (DataFrame, Index, Series, Panel, isnull,
-                             MultiIndex, Float64Index, Timestamp)
+                             MultiIndex, Float64Index, Timestamp, Timedelta)
 from pandas.util.testing import (assert_almost_equal, assert_series_equal,
                                  assert_frame_equal, assert_panel_equal,
                                  assert_attr_equal)
@@ -322,7 +322,7 @@ def _check(f, func, values = False):
             _check(d['ts'],    'at')
             _check(d['floats'],'at')
 
-    def test_at_timestamp(self):
+    def test_at_iat_coercion(self):
 
         # as timestamp is not a tuple!
         dates = date_range('1/1/2000', periods=8)
@@ -333,6 +333,22 @@ def test_at_timestamp(self):
         xp     = s.values[5]
         self.assertEqual(result, xp)
 
+        # GH 7729
+        # make sure we are boxing the returns
+        s = Series(['2014-01-01', '2014-02-02'], dtype='datetime64[ns]')
+        expected = Timestamp('2014-02-02')
+
+        for r in [ lambda : s.iat[1], lambda : s.iloc[1] ]:
+            result = r()
+            self.assertEqual(result, expected)
+
+        s = Series(['1 days','2 days'], dtype='timedelta64[ns]')
+        expected = Timedelta('2 days')
+
+        for r in [ lambda : s.iat[1], lambda : s.iloc[1] ]:
+            result = r()
+            self.assertEqual(result, expected)
+
     def test_iat_invalid_args(self):
         pass
 

pandas/tests/test_series.py

@@ -3286,15 +3286,12 @@ def test_bfill(self):
         assert_series_equal(ts.bfill(), ts.fillna(method='bfill'))
 
     def test_sub_of_datetime_from_TimeSeries(self):
-        from pandas.tseries.timedeltas import _possibly_cast_to_timedelta
+        from pandas.tseries.timedeltas import to_timedelta
         from datetime import datetime
         a = Timestamp(datetime(1993, 0o1, 0o7, 13, 30, 00))
         b = datetime(1993, 6, 22, 13, 30)
         a = Series([a])
-        result = _possibly_cast_to_timedelta(np.abs(a - b))
-        self.assertEqual(result.dtype, 'timedelta64[ns]')
-
-        result = _possibly_cast_to_timedelta(np.abs(b - a))
+        result = to_timedelta(np.abs(a - b))
         self.assertEqual(result.dtype, 'timedelta64[ns]')
 
     def test_datetime64_with_index(self):

pandas/tseries/tests/test_base.py

@@ -213,13 +213,9 @@ def test_sub_isub(self):
 
             for rng, other, expected in [(rng1, other1, expected1), (rng2, other2, expected2),
                                          (rng3, other3, expected3)]:
-                result_add = rng - other
-                result_union = rng.diff(other)
+                result_union = rng.difference(other)
 
-                tm.assert_index_equal(result_add, expected)
                 tm.assert_index_equal(result_union, expected)
-                rng -= other
-                tm.assert_index_equal(rng, expected)
 
             # offset
             offsets = [pd.offsets.Hour(2), timedelta(hours=2), np.timedelta64(2, 'h'),
@@ -859,13 +855,8 @@ def test_sub_isub(self):
                                      (rng3, other3, expected3), (rng4, other4, expected4),
                                      (rng5, other5, expected5), (rng6, other6, expected6),
                                      (rng7, other7, expected7),]:
-            result_add = rng - other
-            result_union = rng.diff(other)
-
-            tm.assert_index_equal(result_add, expected)
+            result_union = rng.difference(other)
             tm.assert_index_equal(result_union, expected)
-            rng -= other
-            tm.assert_index_equal(rng, expected)
 
         # offset
         # DateOffset

pandas/tseries/timedeltas.py

@@ -12,7 +12,7 @@
                                 is_timedelta64_dtype, _values_from_object,
                                 is_list_like, isnull, _ensure_object)
 
-def to_timedelta(arg, unit='ns', box=True):
+def to_timedelta(arg, unit='ns', box=True, coerce=False):
     """
     Convert argument to timedelta
 
@@ -23,6 +23,7 @@ def to_timedelta(arg, unit='ns', box=True):
     box : boolean, default True
         If True returns a Timedelta/TimedeltaIndex of the results
         if False returns a np.timedelta64 or ndarray of values of dtype timedelta64[ns]
+    coerce : force errors to NaT (False by default)
 
     Returns
     -------
@@ -43,14 +44,14 @@ def _convert_listlike(arg, box, unit):
             value = arg.astype('timedelta64[{0}]'.format(unit)).astype('timedelta64[ns]')
         else:
             try:
-                value = tslib.array_to_timedelta64(_ensure_object(arg), unit=unit)
+                value = tslib.array_to_timedelta64(_ensure_object(arg), unit=unit, coerce=coerce)
             except:
 
                 # try to process strings fast; may need to fallback
                 try:
                     value = np.array([ _get_string_converter(r, unit=unit)() for r in arg ],dtype='m8[ns]')
                 except:
-                    value = np.array([ _coerce_scalar_to_timedelta_type(r, unit=unit) for r in arg ])
+                    value = np.array([ _coerce_scalar_to_timedelta_type(r, unit=unit, coerce=coerce) for r in arg ])
 
         if box:
             from pandas import TimedeltaIndex
@@ -67,7 +68,7 @@ def _convert_listlike(arg, box, unit):
         return _convert_listlike(arg, box=box, unit=unit)
 
     # ...so it must be a scalar value. Return scalar.
-    return _coerce_scalar_to_timedelta_type(arg, unit=unit, box=box)
+    return _coerce_scalar_to_timedelta_type(arg, unit=unit, box=box, coerce=coerce)
 
 _unit_map = {
     'Y' : 'Y',
@@ -135,7 +136,7 @@ def _validate_timedelta_unit(arg):
 _full_search2 = re.compile(''.join(
     ["^\s*(?P<neg>-?)\s*"] + [ "(?P<" + p + ">\\d+\.?\d*\s*(" + ss + "))?\\s*" for p, ss in abbrevs ] + ['$']))
 
-def _coerce_scalar_to_timedelta_type(r, unit='ns', box=True):
+def _coerce_scalar_to_timedelta_type(r, unit='ns', box=True, coerce=False):
     """ convert strings to timedelta; coerce to Timedelta (if box), else np.timedelta64"""
 
     if isinstance(r, compat.string_types):
@@ -145,7 +146,7 @@ def _coerce_scalar_to_timedelta_type(r, unit='ns', box=True):
         r = converter()
         unit='ns'
 
-    result = tslib.convert_to_timedelta(r,unit)
+    result = tslib.convert_to_timedelta(r,unit,coerce)
     if box:
         result = tslib.Timedelta(result)
 
@@ -262,32 +263,3 @@ def convert(r=None, unit=None, m=m):
     # no converter
     raise ValueError("cannot create timedelta string converter for [{0}]".format(r))
 
-def _possibly_cast_to_timedelta(value, coerce=True, dtype=None):
-    """ try to cast to timedelta64, if already a timedeltalike, then make
-        sure that we are [ns] (as numpy 1.6.2 is very buggy in this regards,
-        don't force the conversion unless coerce is True
-
-        if dtype is passed then this is the target dtype
-        """
-
-    # deal with numpy not being able to handle certain timedelta operations
-    if isinstance(value, (ABCSeries, np.ndarray)):
-
-        # i8 conversions
-        if value.dtype == 'int64' and np.dtype(dtype) == 'timedelta64[ns]':
-            value = value.astype('timedelta64[ns]')
-            return value
-        elif value.dtype.kind == 'm':
-            if value.dtype != 'timedelta64[ns]':
-                value = value.astype('timedelta64[ns]')
-            return value
-
-    # we don't have a timedelta, but we want to try to convert to one (but
-    # don't force it)
-    if coerce:
-        new_value = tslib.array_to_timedelta64(
-            _values_from_object(value).astype(object), coerce=False)
-        if new_value.dtype == 'i8':
-            value = np.array(new_value, dtype='timedelta64[ns]')
-
-    return value

pandas/util/decorators.py

@@ -77,7 +77,7 @@ def wrapper(*args, **kwargs):
                 else:
                     new_arg_value = old_arg_value
                     msg = "the '%s' keyword is deprecated, " \
-                          "use '%s' instead" % (old_arg_name, new_arg_name)  
+                          "use '%s' instead" % (old_arg_name, new_arg_name)
                 warnings.warn(msg, FutureWarning)
                 if kwargs.get(new_arg_name, None) is not None:
                     msg = "Can only specify '%s' or '%s', not both" % \