Merge pull request #5177 from jreback/infer

API: make allclose comparison on dtype downcasting (GH5174)

Author: jreback

pandas/core/common.py

@@ -1053,7 +1053,7 @@ def _possibly_downcast_to_dtype(result, dtype):
             # do a test on the first element, if it fails then we are done
             r = result.ravel()
             arr = np.array([ r[0] ])
-            if (arr != arr.astype(dtype)).item():
+            if not np.allclose(arr,arr.astype(dtype)):
                 return result
 
             # a comparable, e.g. a Decimal may slip in here
@@ -1062,8 +1062,14 @@ def _possibly_downcast_to_dtype(result, dtype):
 
             if issubclass(result.dtype.type, (np.object_,np.number)) and notnull(result).all():
                 new_result = result.astype(dtype)
-                if (new_result == result).all():
-                    return new_result
+                try:
+                    if np.allclose(new_result,result):
+                        return new_result
+                except:
+
+                    # comparison of an object dtype with a number type could hit here
+                    if (new_result == result).all():
+                        return new_result
     except:
         pass
 

pandas/core/internals.py

@@ -376,10 +376,10 @@ def downcast(self, dtypes=None):
                 dtype = dtypes.get(item, self._downcast_dtype)
 
             if dtype is None:
-                nv = _block_shape(values[i])
+                nv = _block_shape(values[i],ndim=self.ndim)
             else:
                 nv = _possibly_downcast_to_dtype(values[i], dtype)
-                nv = _block_shape(nv)
+                nv = _block_shape(nv,ndim=self.ndim)
 
             blocks.append(make_block(nv, Index([item]), self.ref_items, ndim=self.ndim, fastpath=True))
 

pandas/tests/test_generic.py

@@ -59,7 +59,7 @@ def _axes(self):
         """ return the axes for my object typ """
         return self._typ._AXIS_ORDERS
 
-    def _construct(self, shape, value=None, **kwargs):
+    def _construct(self, shape, value=None, dtype=None, **kwargs):
         """ construct an object for the given shape
             if value is specified use that if its a scalar
             if value is an array, repeat it as needed """
@@ -74,7 +74,7 @@ def _construct(self, shape, value=None, **kwargs):
                     # remove the info axis
                     kwargs.pop(self._typ._info_axis_name,None)
                 else:
-                    arr = np.empty(shape)
+                    arr = np.empty(shape,dtype=dtype)
                     arr.fill(value)
             else:
                 fshape = np.prod(shape)
@@ -184,6 +184,32 @@ def test_numpy_1_7_compat_numeric_methods(self):
             if f is not None:
                 f(o)
 
+    def test_downcast(self):
+        # test close downcasting
+
+        o = self._construct(shape=4, value=9, dtype=np.int64)
+        result = o.copy()
+        result._data = o._data.downcast(dtypes='infer')
+        self._compare(result, o)
+
+        o = self._construct(shape=4, value=9.)
+        expected = o.astype(np.int64)
+        result = o.copy()
+        result._data = o._data.downcast(dtypes='infer')
+        self._compare(result, expected)
+
+        o = self._construct(shape=4, value=9.5)
+        result = o.copy()
+        result._data = o._data.downcast(dtypes='infer')
+        self._compare(result, o)
+
+        # are close
+        o = self._construct(shape=4, value=9.000000000005)
+        result = o.copy()
+        result._data = o._data.downcast(dtypes='infer')
+        expected = o.astype(np.int64)
+        self._compare(result, expected)
+
 class TestSeries(unittest.TestCase, Generic):
     _typ = Series
     _comparator = lambda self, x, y: assert_series_equal(x,y)
@@ -335,7 +361,7 @@ def test_interp_quad(self):
         _skip_if_no_scipy()
         sq = Series([1, 4, np.nan, 16], index=[1, 2, 3, 4])
         result = sq.interpolate(method='quadratic')
-        expected = Series([1., 4., 9., 16.], index=[1, 2, 3, 4])
+        expected = Series([1, 4, 9, 16], index=[1, 2, 3, 4])
         assert_series_equal(result, expected)
 
     def test_interp_scipy_basic(self):
@@ -589,7 +615,7 @@ def test_spline(self):
         _skip_if_no_scipy()
         s = Series([1, 2, np.nan, 4, 5, np.nan, 7])
         result = s.interpolate(method='spline', order=1)
-        expected = Series([1., 2, 3, 4, 5, 6, 7])  # dtype?
+        expected = Series([1, 2, 3, 4, 5, 6, 7])
         assert_series_equal(result, expected)