BUG: Categorical concat should preserve levels (GH7864)

Author: jreback

doc/source/v0.15.0.txt

@@ -117,7 +117,7 @@ Categoricals in Series/DataFrame
 
 :class:`~pandas.Categorical` can now be included in `Series` and `DataFrames` and gained new
 methods to manipulate. Thanks to Jan Schultz for much of this API/implementation. (:issue:`3943`, :issue:`5313`, :issue:`5314`,
-:issue:`7444`, :issue:`7839`, :issue:`7848`).
+:issue:`7444`, :issue:`7839`, :issue:`7848`, :issue:`7864`).
 
 For full docs, see the :ref:`Categorical introduction <categorical>` and the :ref:`API documentation <api.categorical>`.
 

pandas/core/internals.py

@@ -451,9 +451,9 @@ def to_native_types(self, slicer=None, na_rep='', **kwargs):
         values[mask] = na_rep
         return values.tolist()
 
-    def _validate_merge(self, blocks):
-        """ validate that we can merge these blocks """
-        return True
+    def _concat_blocks(self, blocks, values):
+        """ return the block concatenation """
+        return self._holder(values[0])
 
     # block actions ####
     def copy(self, deep=True):
@@ -1639,15 +1639,19 @@ def _astype(self, dtype, copy=False, raise_on_error=True, values=None,
                           ndim=self.ndim,
                           placement=self.mgr_locs)
 
-    def _validate_merge(self, blocks):
-        """ validate that we can merge these blocks """
+    def _concat_blocks(self, blocks, values):
+        """
+        validate that we can merge these blocks
+
+        return the block concatenation
+        """
 
         levels = self.values.levels
         for b in blocks:
             if not levels.equals(b.values.levels):
                 raise ValueError("incompatible levels in categorical block merge")
 
-        return True
+        return self._holder(values[0], levels=levels)
 
     def to_native_types(self, slicer=None, na_rep='', **kwargs):
         """ convert to our native types format, slicing if desired """
@@ -4026,17 +4030,11 @@ def concatenate_join_units(join_units, concat_axis, copy):
     else:
         concat_values = com._concat_compat(to_concat, axis=concat_axis)
 
-    # FIXME: optimization potential: if len(join_units) == 1, single join unit
-    # is densified and sparsified back.
     if any(unit.needs_block_conversion for unit in join_units):
 
         # need to ask the join unit block to convert to the underlying repr for us
         blocks = [ unit.block for unit in join_units if unit.block is not None ]
-
-        # may need to validate this combination
-        blocks[0]._validate_merge(blocks)
-
-        return blocks[0]._holder(concat_values[0])
+        return blocks[0]._concat_blocks(blocks, concat_values)
     else:
         return concat_values
 

pandas/tests/test_categorical.py

@@ -1470,6 +1470,22 @@ def f():
             pd.concat([df,df_wrong_levels])
         self.assertRaises(ValueError, f)
 
+        # GH 7864
+        # make sure ordering is preserverd
+        df = pd.DataFrame({"id":[1,2,3,4,5,6], "raw_grade":['a', 'b', 'b', 'a', 'a', 'e']})
+        df["grade"] = pd.Categorical(df["raw_grade"])
+        df['grade'].cat.reorder_levels(['e', 'a', 'b'])
+
+        df1 = df[0:3]
+        df2 = df[3:]
+
+        self.assert_numpy_array_equal(df['grade'].cat.levels, df1['grade'].cat.levels)
+        self.assert_numpy_array_equal(df['grade'].cat.levels, df2['grade'].cat.levels)
+
+        dfx = pd.concat([df1, df2])
+        dfx['grade'].cat.levels
+        self.assert_numpy_array_equal(df['grade'].cat.levels, dfx['grade'].cat.levels)
+
     def test_append(self):
         cat = pd.Categorical(["a","b"], levels=["a","b"])
         vals = [1,2]