BUG: (GH4758) Bug in multi-indexing with a partial string selection as one part of a MultIndex

Author: jreback

doc/source/release.rst

@@ -322,6 +322,7 @@ See :ref:`Internal Refactoring<whatsnew_0130.refactoring>`
   - Bug in using ``iloc/loc`` with a cross-sectional and duplicate indicies (:issue:`4726`)
   - Bug with using ``QUOTE_NONE`` with ``to_csv`` causing ``Exception``. (:issue:`4328`)
   - Bug with Series indexing not raising an error when the right-hand-side has an incorrect length (:issue:`2702`)
+  - Bug in multi-indexing with a partial string selection as one part of a MultIndex (:issue:`4758`)
 
 pandas 0.12
 ===========

pandas/core/index.py

@@ -2596,10 +2596,15 @@ def _maybe_drop_levels(indexer, levels, drop_level):
             if not drop_level:
                 return self[indexer]
             # kludgearound
-            new_index = self[indexer]
+            orig_index = new_index = self[indexer]
             levels = [self._get_level_number(i) for i in levels]
             for i in sorted(levels, reverse=True):
-                new_index = new_index.droplevel(i)
+                try:
+                    new_index = new_index.droplevel(i)
+                except:
+
+                    # no dropping here
+                    return orig_index
             return new_index
 
         if isinstance(level, (tuple, list)):
@@ -2635,20 +2640,37 @@ def _maybe_drop_levels(indexer, levels, drop_level):
                 pass
 
             if not any(isinstance(k, slice) for k in key):
-                if len(key) == self.nlevels:
-                    if self.is_unique:
-                        return self._engine.get_loc(_values_from_object(key)), None
-                    else:
-                        indexer = slice(*self.slice_locs(key, key))
-                        return indexer, self[indexer]
-                else:
-                    # partial selection
+
+                # partial selection
+                def partial_selection(key):
                     indexer = slice(*self.slice_locs(key, key))
                     if indexer.start == indexer.stop:
                         raise KeyError(key)
                     ilevels = [i for i in range(len(key))
                                if key[i] != slice(None, None)]
                     return indexer, _maybe_drop_levels(indexer, ilevels, drop_level)
+
+                if len(key) == self.nlevels:
+
+                    if self.is_unique:
+
+                        # here we have a completely specified key, but are using some partial string matching here
+                        # GH4758
+                        can_index_exactly = any([ l.is_all_dates and not isinstance(k,compat.string_types) for k, l in zip(key, self.levels) ])
+                        if any([ l.is_all_dates for k, l in zip(key, self.levels) ]) and not can_index_exactly:
+                            indexer = slice(*self.slice_locs(key, key))
+
+                            # we have a multiple selection here
+                            if not indexer.stop-indexer.start == 1:
+                                return partial_selection(key)
+
+                            key = tuple(self[indexer].tolist()[0])
+
+                        return self._engine.get_loc(_values_from_object(key)), None
+                    else:
+                        return partial_selection(key)
+                else:
+                    return partial_selection(key)
             else:
                 indexer = None
                 for i, k in enumerate(key):

pandas/core/indexing.py

@@ -952,9 +952,15 @@ def _has_valid_type(self, key, axis):
             if not len(ax):
                 raise KeyError("The [%s] axis is empty" % self.obj._get_axis_name(axis))
 
-            if not key in ax:
+            try:
+                if not key in ax:
+                    raise KeyError("the label [%s] is not in the [%s]" % (key,self.obj._get_axis_name(axis)))
+            except (TypeError):
+
+                # if we have a weird type of key/ax
                 raise KeyError("the label [%s] is not in the [%s]" % (key,self.obj._get_axis_name(axis)))
 
+
         return True
 
     def _getitem_axis(self, key, axis=0):

pandas/tests/test_multilevel.py

@@ -1842,9 +1842,9 @@ def test_duplicate_mi(self):
                        columns=list('ABCD'))
         df = df.set_index(['A','B'])
         df = df.sortlevel(0)
-        result = df.loc[('foo','bar')]
         expected = DataFrame([['foo','bar',1.0,1],['foo','bar',2.0,2],['foo','bar',5.0,5]],
                              columns=list('ABCD')).set_index(['A','B'])
+        result = df.loc[('foo','bar')]
         assert_frame_equal(result,expected)
 
     def test_multiindex_set_index(self):

pandas/tools/tests/test_pivot.py

@@ -175,6 +175,7 @@ def _check_output(res, col, rows=['A', 'B'], cols=['C']):
             exp = self.data.groupby(rows)[col].mean()
             tm.assert_series_equal(cmarg, exp)
 
+            res.sortlevel(inplace=True)
             rmarg = res.xs(('All', ''))[:-1]
             exp = self.data.groupby(cols)[col].mean()
             tm.assert_series_equal(rmarg, exp)

pandas/tseries/tests/test_timeseries.py

@@ -2562,9 +2562,32 @@ def test_partial_slice_minutely(self):
         result = s['2005-1-1']
         assert_series_equal(result, s.ix[:60])
 
-        assert_series_equal(s['2005-1-1 23:59:00'],s.ix[0])
+        self.assert_(s[Timestamp('2005-1-1 23:59:00')] == s.ix[0])
         self.assertRaises(Exception, s.__getitem__, '2004-12-31 00:00:00')
 
+    def test_partial_slicing_with_multiindex(self):
+
+        # GH 4758
+        # partial string indexing with a multi-index buggy
+        df = DataFrame({'ACCOUNT':["ACCT1", "ACCT1", "ACCT1", "ACCT2"],
+                        'TICKER':["ABC", "MNP", "XYZ", "XYZ"],
+                        'val':[1,2,3,4]},
+                       index=date_range("2013-06-19 09:30:00", periods=4, freq='5T'))
+        df_multi = df.set_index(['ACCOUNT', 'TICKER'], append=True)
+
+        expected = DataFrame([[1]],index=Index(['ABC'],name='TICKER'),columns=['val'])
+        result = df_multi.loc[('2013-06-19 09:30:00', 'ACCT1')]
+        assert_frame_equal(result, expected)
+
+        expected = df_multi.loc[(pd.Timestamp('2013-06-19 09:30:00', tz=None), 'ACCT1', 'ABC')]
+        result = df_multi.loc[('2013-06-19 09:30:00', 'ACCT1', 'ABC')]
+        assert_series_equal(result, expected)
+
+        # this is a KeyError as we don't do partial string selection on multi-levels
+        def f():
+            df_multi.loc[('2013-06-19', 'ACCT1', 'ABC')]
+        self.assertRaises(KeyError, f)
+
     def test_date_range_normalize(self):
         snap = datetime.today()
         n = 50