BLD: py3 compat

TST/BUG: test/bugfix for GH4463

BUG: fix core/internals/setitem to work for boolean types (weird numpy bug!)

BUG: partial frame setting with dtype change (GH4204)

BUG: Indexing with dtype conversions fixed GH4463 (int->float), GH4204(boolean->float)

BUG: provide better ndarray compat

CLN: removed some duped methods

MERGE: fix an issue cropping up on the rebase

Author: jreback

doc/source/release.rst

@@ -184,6 +184,8 @@ and behaviors. Series formerly subclassed directly from ``ndarray``. (:issue:`40
 - Bug in Series update where the parent frame is not updating its cache based on
   changes (:issue:`4080`) or types (:issue:`3217`), fillna (:issue:`3386`)
 
+- Indexing with dtype conversions fixed (:issue:`4463`, :issue:`4204`)
+
 **Experimental Features**
 
 **Bug Fixes**

doc/source/v0.13.0.txt

@@ -202,6 +202,8 @@ and behaviors. Series formerly subclassed directly from ``ndarray``. (:issue:`40
 - Bug in Series update where the parent frame is not updating its cached based on
   changes (:issue:`4080`) or types (:issue:`3217`), fillna (:issue:`3386`)
 
+- Indexing with dtype conversions fixed (:issue:`4463`, :issue:`4204`)
+
 Bug Fixes
 ~~~~~~~~~
 

pandas/compat/pickle_compat.py

@@ -4,7 +4,7 @@
 import pickle
 import numpy as np
 import pandas
-from pandas.util import py3compat
+from pandas import compat
 from pandas.core.series import Series
 from pandas.sparse.series import SparseSeries
 
@@ -20,7 +20,7 @@ def load_reduce(self):
         elif n == 'DeprecatedSparseSeries':
             stack[-1] = object.__new__(SparseSeries)
             return
-            
+
     try:
         value = func(*args)
     except:
@@ -30,7 +30,7 @@ def load_reduce(self):
 
     stack[-1] = value
 
-if py3compat.PY3:
+if compat.PY3:
     class Unpickler(pickle._Unpickler):
         pass
 else:

pandas/core/frame.py

@@ -410,6 +410,8 @@ def __init__(self, data=None, index=None, columns=None, dtype=None,
                 if columns is None:
                     columns = data_columns
                 mgr = self._init_dict(data, index, columns, dtype=dtype)
+            elif getattr(data,'name',None):
+                mgr = self._init_dict({ data.name : data }, index, columns, dtype=dtype)
             else:
                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
                                          copy=copy)
@@ -4853,9 +4855,12 @@ def convert(v):
         # we could have a 1-dim or 2-dim list here
         # this is equiv of np.asarray, but does object conversion
         # and platform dtype preservation
-        if com.is_list_like(values[0]) or hasattr(values[0], 'len'):
-            values = np.array([convert(v) for v in values])
-        else:
+        try:
+            if com.is_list_like(values[0]) or hasattr(values[0], 'len'):
+                values = np.array([convert(v) for v in values])
+            else:
+                values = convert(values)
+        except:
             values = convert(values)
 
     else:
@@ -4945,18 +4950,23 @@ def _list_of_series_to_arrays(data, columns, coerce_float=False, dtype=None):
     from pandas.core.index import _get_combined_index
 
     if columns is None:
-        columns = _get_combined_index([s.index for s in data])
+        columns = _get_combined_index([s.index for s in data if getattr(s,'index',None) is not None ])
 
     indexer_cache = {}
 
     aligned_values = []
     for s in data:
-        index = s.index
+        index = getattr(s,'index',None)
+        if index is None:
+            index = _default_index(len(s))
+
         if id(index) in indexer_cache:
             indexer = indexer_cache[id(index)]
         else:
             indexer = indexer_cache[id(index)] = index.get_indexer(columns)
-        aligned_values.append(com.take_1d(s.values, indexer))
+
+        values = _values_from_object(s)
+        aligned_values.append(com.take_1d(values, indexer))
 
     values = np.vstack(aligned_values)
 
@@ -5000,13 +5010,13 @@ def _convert_object_array(content, columns, coerce_float=False, dtype=None):
 
 def _get_names_from_index(data):
     index = lrange(len(data))
-    has_some_name = any([s.name is not None for s in data])
+    has_some_name = any([getattr(s,'name',None) is not None for s in data])
     if not has_some_name:
         return index
 
     count = 0
     for i, s in enumerate(data):
-        n = s.name
+        n = getattr(s,'name',None)
         if n is not None:
             index[i] = n
         else:

pandas/core/generic.py

@@ -1,27 +1,22 @@
 # pylint: disable=W0231,E1101
 import warnings
-from pandas import compat
-import itertools
 import operator
 import weakref
 import numpy as np
 import pandas.lib as lib
-from pandas.core.base import PandasObject
 
+from pandas.core.base import PandasObject
 from pandas.core.index import Index, MultiIndex, _ensure_index
 import pandas.core.indexing as indexing
 from pandas.core.indexing import _maybe_convert_indices
 from pandas.tseries.index import DatetimeIndex
 from pandas.core.internals import BlockManager
-import pandas.lib as lib
-from pandas.util import py3compat
 import pandas.core.common as com
+from pandas import compat
 from pandas.compat import map, zip
 from pandas.core.common import (isnull, notnull, is_list_like,
                                 _values_from_object,
                                 _infer_dtype_from_scalar, _maybe_promote)
-from pandas.core.base import PandasObject
-
 
 class NDFrame(PandasObject):
 
@@ -78,10 +73,6 @@ def _init_mgr(self, mgr, axes=None, dtype=None, copy=False):
     def _constructor(self):
         raise NotImplementedError
 
-    def __hash__(self):
-        raise TypeError('{0!r} objects are mutable, thus they cannot be'
-                        ' hashed'.format(self.__class__.__name__))
-
     def __unicode__(self):
         # unicode representation based upon iterating over self
         # (since, by definition, `PandasContainers` are iterable)
@@ -111,12 +102,12 @@ def _setup_axes(
             """
 
         cls._AXIS_ORDERS = axes
-        cls._AXIS_NUMBERS = dict([(a, i) for i, a in enumerate(axes)])
+        cls._AXIS_NUMBERS = dict((a, i) for i, a in enumerate(axes))
         cls._AXIS_LEN = len(axes)
         cls._AXIS_ALIASES = aliases or dict()
-        cls._AXIS_IALIASES = dict([(v, k)
-                                  for k, v in cls._AXIS_ALIASES.items()])
-        cls._AXIS_NAMES = dict([(i, a) for i, a in enumerate(axes)])
+        cls._AXIS_IALIASES = dict((v, k)
+                                  for k, v in cls._AXIS_ALIASES.items())
+        cls._AXIS_NAMES = dict(enumerate(axes))
         cls._AXIS_SLICEMAP = slicers or None
         cls._AXIS_REVERSED = axes_are_reversed
 
@@ -271,23 +262,6 @@ def axes(self):
         the block manager shows then reversed """
         return [self._get_axis(a) for a in self._AXIS_ORDERS]
 
-    def _construct_axes_dict(self, axes=None, **kwargs):
-        """ return an axes dictionary for myself """
-        d = dict([(a, getattr(self, a)) for a in (axes or self._AXIS_ORDERS)])
-        d.update(kwargs)
-        return d
-
-    @staticmethod
-    def _construct_axes_dict_from(self, axes, **kwargs):
-        """ return an axes dictionary for the passed axes """
-        d = dict([(a, ax) for a, ax in zip(self._AXIS_ORDERS, axes)])
-        d.update(kwargs)
-        return d
-
-    @property
-    def values(self):
-        return self._data.as_matrix()
-
     @property
     def ndim(self):
         return self._data.ndim
@@ -445,9 +419,6 @@ def rename_axis(self, mapper, axis=0, copy=True):
     def _indexed_same(self, other):
         return all([self._get_axis(a).equals(other._get_axis(a)) for a in self._AXIS_ORDERS])
 
-    def reindex(self, *args, **kwds):
-        raise NotImplementedError
-
     def __neg__(self):
         arr = operator.neg(_values_from_object(self))
         return self._wrap_array(arr, self.axes, copy=False)
@@ -460,7 +431,8 @@ def __invert__(self):
     # Iteration
 
     def __hash__(self):
-        raise TypeError
+        raise TypeError('{0!r} objects are mutable, thus they cannot be'
+                        ' hashed'.format(self.__class__.__name__))
 
     def __iter__(self):
         """
@@ -483,7 +455,6 @@ def iterkv(self, *args, **kwargs):
                       "release, use ``iteritems`` instead.", DeprecationWarning)
         return self.iteritems(*args, **kwargs)
 
-
     def __len__(self):
         """Returns length of info axis """
         return len(self._info_axis)
@@ -1142,7 +1113,7 @@ def filter(self, items=None, like=None, regex=None, axis=None):
         if items is not None:
             return self.reindex(**{axis_name: [r for r in items if r in axis_values]})
         elif like:
-            matchf = lambda x: (like in x if isinstance(x, basestring)
+            matchf = lambda x: (like in x if isinstance(x, compat.string_types)
                                 else like in str(x))
             return self.select(matchf, axis=axis_name)
         elif regex:
@@ -1285,6 +1256,7 @@ def get_dtype_counts(self):
 
     def get_ftype_counts(self):
         """ return the counts of ftypes in this frame """
+        from pandas import Series
         return Series(self._data.get_ftype_counts())
 
     def as_blocks(self, columns=None):
@@ -1446,7 +1418,7 @@ def fillna(self, value=None, method=None, axis=0, inplace=False,
                                               'by column')
 
                 result = self if inplace else self.copy()
-                for k, v in value.iteritems():
+                for k, v in compat.iteritems(value):
                     if k not in result:
                         continue
                     obj = result[k]
@@ -1595,7 +1567,7 @@ def is_dictlike(x):
                 regex = True
 
             items = to_replace.items()
-            keys, values = itertools.izip(*items)
+            keys, values = zip(*items)
 
             are_mappings = [is_dictlike(v) for v in values]
 
@@ -1629,7 +1601,7 @@ def is_dictlike(x):
             if is_dictlike(to_replace):
                 if is_dictlike(value):  # {'A' : NA} -> {'A' : 0}
                     new_data = self._data
-                    for c, src in to_replace.iteritems():
+                    for c, src in compat.iteritems(to_replace):
                         if c in value and c in self:
                             new_data = new_data.replace(src, value[c],
                                                         filter=[c],
@@ -1639,7 +1611,7 @@ def is_dictlike(x):
                 # {'A': NA} -> 0
                 elif not isinstance(value, (list, np.ndarray)):
                     new_data = self._data
-                    for k, src in to_replace.iteritems():
+                    for k, src in compat.iteritems(to_replace):
                         if k in self:
                             new_data = new_data.replace(src, value,
                                                         filter=[k],
@@ -1679,7 +1651,7 @@ def is_dictlike(x):
                 if is_dictlike(value):  # NA -> {'A' : 0, 'B' : -1}
                     new_data = self._data
 
-                    for k, v in value.iteritems():
+                    for k, v in compat.iteritems(value):
                         if k in self:
                             new_data = new_data.replace(to_replace, v,
                                                         filter=[k],
@@ -1729,7 +1701,7 @@ def interpolate(self, to_replace, method='pad', axis=0, inplace=False,
 
         method = com._clean_fill_method(method)
 
-        if isinstance(to_replace, (dict, Series)):
+        if isinstance(to_replace, (dict, com.ABCSeries)):
             if axis == 0:
                 return self.replace(to_replace, method=method, inplace=inplace,
                                     limit=limit, axis=axis)

pandas/core/index.py

@@ -15,7 +15,6 @@
 from pandas.core.common import isnull
 import pandas.core.common as com
 from pandas.core.common import _values_from_object
-from pandas.util import py3compat
 from pandas.core.config import get_option
 import warnings
 
@@ -808,7 +807,7 @@ def get_value(self, series, key):
         k = _values_from_object(key)
         try:
             return self._engine.get_value(s, k)
-        except KeyError, e1:
+        except KeyError as e1:
             if len(self) > 0 and self.inferred_type == 'integer':
                 raise
 
@@ -1447,7 +1446,7 @@ def __new__(cls, data, dtype=None, copy=False, name=None, fastpath=False):
                     data = list(data)
                 data = np.asarray(data)
 
-            if issubclass(data.dtype.type, basestring):
+            if issubclass(data.dtype.type, compat.string_types):
                 raise TypeError('String dtype not supported, you may need '
                                 'to explicitly cast to int')
             elif issubclass(data.dtype.type, np.integer):
@@ -1865,7 +1864,7 @@ def get_value(self, series, key):
         k = _values_from_object(key)
         try:
             return self._engine.get_value(s, k)
-        except KeyError, e1:
+        except KeyError as e1:
             try:
                 # TODO: what if a level contains tuples??
                 loc = self.get_loc(key)

pandas/core/indexing.py

@@ -6,7 +6,8 @@
 from pandas.compat import range, zip
 import pandas.compat as compat
 import pandas.core.common as com
-from pandas.core.common import _is_bool_indexer, ABCSeries, ABCDataFrame
+from pandas.core.common import (_is_bool_indexer,
+                                ABCSeries, ABCDataFrame, ABCPanel)
 import pandas.lib as lib
 
 import numpy as np
@@ -104,7 +105,6 @@ def _convert_tuple(self, key):
     def _setitem_with_indexer(self, indexer, value):
 
         # also has the side effect of consolidating in-place
-
         # mmm, spaghetti
 
         if self.obj._is_mixed_type:
@@ -182,13 +182,10 @@ def setter(item, v):
             elif isinstance(value, ABCDataFrame):
                 value = self._align_frame(indexer, value)
 
-            if isinstance(value, Panel):
+            if isinstance(value, ABCPanel):
                 value = self._align_panel(indexer, value)
 
-            # 2096
-            values = self.obj.values
-            if np.prod(values.shape):
-                values[indexer] = value
+            self.obj._data = self.obj._data.setitem(indexer,value)
 
     def _align_series(self, indexer, ser):
         # indexer to assign Series can be tuple or scalar