CLN: in common.py - revised _maybe_upcast to use _maybe_promote

     in rehashpe.py - removed block2d_to_block3d in favor of block2d_to_blocknd

Author: jreback

pandas/core/common.py

@@ -651,7 +651,7 @@ def _infer_dtype_from_scalar(val):
     # a 1-element ndarray
     if isinstance(val, pa.Array):
         if val.ndim != 0:
-            raise ValueError("invalid ndarray passed to _dtype_from_scalar")
+            raise ValueError("invalid ndarray passed to _infer_dtype_from_scalar")
 
         return val.item(), val.dtype
 
@@ -719,13 +719,21 @@ def _maybe_promote(dtype, fill_value=np.nan):
 
 
 def _maybe_upcast(values):
-    # TODO: convert remaining usage of _maybe_upcast to _maybe_promote
-    if issubclass(values.dtype.type, np.integer):
-        values = values.astype(np.float64)
-    elif issubclass(values.dtype.type, np.bool_):
-        values = values.astype(np.object_)
+    """ provide explicty type promotion and coercion """
+    new_dtype = _maybe_promote(values.dtype)
+    if new_dtype != values.dtype:
+        values = values.astype(new_dtype)
     return values
- 
+
+def _possibly_cast_item(obj, item, dtype):
+    chunk = obj[item]
+
+    if chunk.values.dtype != dtype:
+        if dtype in (np.object_, np.bool_):
+            obj[item] = chunk.astype(np.object_)
+        elif not issubclass(dtype, (np.integer, np.bool_)):  # pragma: no cover
+            raise ValueError("Unexpected dtype encountered: %s" % dtype)
+
 
 def _interp_wrapper(f, wrap_dtype, na_override=None):
     def wrapper(arr, mask, limit=None):
@@ -927,16 +935,6 @@ def _possibly_cast_to_datetime(value, dtype, coerce = False):
     return value
 
 
-def _possibly_cast_item(obj, item, dtype):
-    chunk = obj[item]
-
-    if chunk.values.dtype != dtype:
-        if dtype in (np.object_, np.bool_):
-            obj[item] = chunk.astype(np.object_)
-        elif not issubclass(dtype, (np.integer, np.bool_)):  # pragma: no cover
-            raise ValueError("Unexpected dtype encountered: %s" % dtype)
-
-
 def _is_bool_indexer(key):
     if isinstance(key, np.ndarray) and key.dtype == np.object_:
         key = np.asarray(key)

pandas/core/frame.py

@@ -1234,7 +1234,7 @@ def to_panel(self):
         panel : Panel
         """
         from pandas.core.panel import Panel
-        from pandas.core.reshape import block2d_to_block3d
+        from pandas.core.reshape import block2d_to_blocknd
 
         # only support this kind for now
         if (not isinstance(self.index, MultiIndex) or
@@ -1261,8 +1261,8 @@ def to_panel(self):
 
         new_blocks = []
         for block in selfsorted._data.blocks:
-            newb = block2d_to_block3d(block.values.T, block.items, shape,
-                                      major_labels, minor_labels,
+            newb = block2d_to_blocknd(block.values.T, block.items, shape,
+                                      [ major_labels, minor_labels ],
                                       ref_items=selfsorted.columns)
             new_blocks.append(newb)
 

pandas/core/reshape.py

@@ -9,7 +9,8 @@
 from pandas.core.frame import DataFrame
 
 from pandas.core.categorical import Categorical
-from pandas.core.common import notnull, _ensure_platform_int
+from pandas.core.common import (notnull, _ensure_platform_int, _maybe_promote,
+                                _maybe_upcast)
 from pandas.core.groupby import (get_group_index, _compress_group_index,
                                  decons_group_index)
 import pandas.core.common as com
@@ -148,11 +149,9 @@ def get_new_values(self):
         stride = values.shape[1]
         result_width = width * stride
 
-        new_values = np.empty((length, result_width), dtype=values.dtype)
-        new_mask = np.zeros((length, result_width), dtype=bool)
-
-        new_values = com._maybe_upcast(new_values)
+        new_values = np.empty((length, result_width), dtype=_maybe_promote(values.dtype))
         new_values.fill(np.nan)
+        new_mask = np.zeros((length, result_width), dtype=bool)
 
         # is there a simpler / faster way of doing this?
         for i in xrange(values.shape[1]):
@@ -761,40 +760,6 @@ def make_axis_dummies(frame, axis='minor', transform=None):
     return DataFrame(values, columns=items, index=frame.index)
 
 
-def block2d_to_block3d(values, items, shape, major_labels, minor_labels,
-                       ref_items=None):
-    """
-    Developer method for pivoting DataFrame -> Panel. Used in HDFStore and
-    DataFrame.to_panel
-    """
-    from pandas.core.internals import make_block
-    panel_shape = (len(items),) + shape
-
-    # TODO: lexsort depth needs to be 2!!
-
-    # Create observation selection vector using major and minor
-    # labels, for converting to panel format.
-    selector = minor_labels + shape[1] * major_labels
-    mask = np.zeros(np.prod(shape), dtype=bool)
-    mask.put(selector, True)
-
-    pvalues = np.empty(panel_shape, dtype=values.dtype)
-    if not issubclass(pvalues.dtype.type, (np.integer, np.bool_)):
-        pvalues.fill(np.nan)
-    elif not mask.all():
-        pvalues = com._maybe_upcast(pvalues)
-        pvalues.fill(np.nan)
-
-    values = values
-    for i in xrange(len(items)):
-        pvalues[i].flat[mask] = values[:, i]
-
-    if ref_items is None:
-        ref_items = items
-
-    return make_block(pvalues, items, ref_items)
-
-
 def block2d_to_blocknd(values, items, shape, labels, ref_items=None):
     """ pivot to the labels shape """
     from pandas.core.internals import make_block
@@ -812,7 +777,7 @@ def block2d_to_blocknd(values, items, shape, labels, ref_items=None):
     if not issubclass(pvalues.dtype.type, (np.integer, np.bool_)):
         pvalues.fill(np.nan)
     elif not mask.all():
-        pvalues = com._maybe_upcast(pvalues)
+        pvalues = _maybe_upcast(pvalues)
         pvalues.fill(np.nan)
 
     values = values

pandas/core/series.py

@@ -15,7 +15,7 @@
 import numpy.ma as ma
 
 from pandas.core.common import (isnull, notnull, _is_bool_indexer,
-                                _default_index, _maybe_upcast,
+                                _default_index, _maybe_promote,
                                 _asarray_tuplesafe, is_integer_dtype,
                                 _infer_dtype_from_scalar)
 from pandas.core.index import (Index, MultiIndex, InvalidIndexError,
@@ -2818,8 +2818,7 @@ def _get_values():
             return values
 
         if offset is None:
-            new_values = pa.empty(len(self), dtype=self.dtype)
-            new_values = _maybe_upcast(new_values)
+            new_values = pa.empty(len(self), dtype=_maybe_promote(self.dtype))
 
             if periods > 0:
                 new_values[periods:] = self.values[:-periods]

pandas/io/pytables.py

@@ -23,7 +23,7 @@
 from pandas.core.categorical import Categorical
 from pandas.core.common import _asarray_tuplesafe, _try_sort
 from pandas.core.internals import BlockManager, make_block, form_blocks
-from pandas.core.reshape import block2d_to_block3d, block2d_to_blocknd, factor_indexer
+from pandas.core.reshape import block2d_to_blocknd, factor_indexer
 from pandas.core.index import Int64Index
 import pandas.core.common as com
 from pandas.tools.merge import concat