ENH: Categorical.unique can keep same dtype

Author: topper-123

doc/source/whatsnew/v1.2.0.rst

@@ -523,6 +523,7 @@ Categorical
 - :meth:`Categorical.fillna` will always return a copy, validate a passed fill value regardless of whether there are any NAs to fill, and disallow an ``NaT`` as a fill value for numeric categories (:issue:`36530`)
 - Bug in :meth:`Categorical.__setitem__` that incorrectly raised when trying to set a tuple value (:issue:`20439`)
 - Bug in :meth:`CategoricalIndex.equals` incorrectly casting non-category entries to ``np.nan`` (:issue:`37667`)
+- Bug in :meth:`Categorical.unique` where dtype was changed, it there were unused categories (:issue:`xxxxx`).
 - Bug in :meth:`CategoricalIndex.where` incorrectly setting non-category entries to ``np.nan`` instead of raising ``TypeError`` (:issue:`37977`)
 -
 

pandas/core/arrays/categorical.py

@@ -2078,16 +2078,8 @@ def unique(self):
         """
         # unlike np.unique, unique1d does not sort
         unique_codes = unique1d(self.codes)
-        cat = self.copy()
-
-        # keep nan in codes
-        cat._codes = unique_codes
-
-        # exclude nan from indexer for categories
-        take_codes = unique_codes[unique_codes != -1]
-        if self.ordered:
-            take_codes = np.sort(take_codes)
-        return cat.set_categories(cat.categories.take(take_codes))
+        cat = self._constructor(unique_codes, dtype=self.dtype, fastpath=True)
+        return cat
 
     def _values_for_factorize(self):
         return self._ndarray, -1

pandas/core/groupby/categorical.py

@@ -72,6 +72,11 @@ def recode_for_groupby(
 
     # sort=False should order groups in as-encountered order (GH-8868)
     cat = c.unique()
+    # exclude nan from indexer for categories
+    take_codes = cat.codes[cat.codes != -1]
+    if cat.ordered:
+        take_codes = np.sort(take_codes)
+    cat = cat.set_categories(cat.categories.take(take_codes))
 
     # But for groupby to work, all categories should be present,
     # including those missing from the data (GH-13179), which .unique()

pandas/tests/arrays/categorical/test_analytics.py

@@ -6,7 +6,7 @@
 
 from pandas.compat import PYPY
 
-from pandas import Categorical, Index, NaT, Series, date_range
+from pandas import Categorical, CategoricalDtype, Index, NaT, Series, date_range
 import pandas._testing as tm
 from pandas.api.types import is_scalar
 
@@ -188,84 +188,49 @@ def test_searchsorted(self, ordered):
         with pytest.raises(KeyError, match="cucumber"):
             ser.searchsorted(["bread", "cucumber"])
 
-    def test_unique(self):
+    def test_unique(self, ordered):
+        # GHXXXXX
+        dtype = CategoricalDtype(["a", "b", "c"], ordered=ordered)
+
         # categories are reordered based on value when ordered=False
-        cat = Categorical(["a", "b"])
-        exp = Index(["a", "b"])
+        cat = Categorical(["a", "b", "c"], dtype=dtype)
         res = cat.unique()
-        tm.assert_index_equal(res.categories, exp)
         tm.assert_categorical_equal(res, cat)
 
-        cat = Categorical(["a", "b", "a", "a"], categories=["a", "b", "c"])
+        cat = Categorical(["a", "b", "a", "a"], dtype=dtype)
         res = cat.unique()
-        tm.assert_index_equal(res.categories, exp)
-        tm.assert_categorical_equal(res, Categorical(exp))
+        tm.assert_categorical_equal(res, Categorical(["a", "b"], dtype=dtype))
 
-        cat = Categorical(["c", "a", "b", "a", "a"], categories=["a", "b", "c"])
-        exp = Index(["c", "a", "b"])
+        cat = Categorical(["c", "a", "b", "a", "a"], dtype=dtype)
         res = cat.unique()
-        tm.assert_index_equal(res.categories, exp)
-        exp_cat = Categorical(exp, categories=["c", "a", "b"])
+        exp_cat = Categorical(["c", "a", "b"], dtype=dtype)
         tm.assert_categorical_equal(res, exp_cat)
 
         # nan must be removed
-        cat = Categorical(["b", np.nan, "b", np.nan, "a"], categories=["a", "b", "c"])
-        res = cat.unique()
-        exp = Index(["b", "a"])
-        tm.assert_index_equal(res.categories, exp)
-        exp_cat = Categorical(["b", np.nan, "a"], categories=["b", "a"])
-        tm.assert_categorical_equal(res, exp_cat)
-
-    def test_unique_ordered(self):
-        # keep categories order when ordered=True
-        cat = Categorical(["b", "a", "b"], categories=["a", "b"], ordered=True)
+        cat = Categorical(["b", np.nan, "b", np.nan, "a"], dtype=dtype)
         res = cat.unique()
-        exp_cat = Categorical(["b", "a"], categories=["a", "b"], ordered=True)
+        exp_cat = Categorical(["b", np.nan, "a"], dtype=dtype)
         tm.assert_categorical_equal(res, exp_cat)
 
-        cat = Categorical(
-            ["c", "b", "a", "a"], categories=["a", "b", "c"], ordered=True
-        )
-        res = cat.unique()
-        exp_cat = Categorical(["c", "b", "a"], categories=["a", "b", "c"], ordered=True)
-        tm.assert_categorical_equal(res, exp_cat)
-
-        cat = Categorical(["b", "a", "a"], categories=["a", "b", "c"], ordered=True)
-        res = cat.unique()
-        exp_cat = Categorical(["b", "a"], categories=["a", "b"], ordered=True)
-        tm.assert_categorical_equal(res, exp_cat)
+    def test_unique_index_series(self, ordered):
+        # GHXXXXX
+        dtype = CategoricalDtype([3, 2, 1], ordered=ordered)
 
-        cat = Categorical(
-            ["b", "b", np.nan, "a"], categories=["a", "b", "c"], ordered=True
-        )
-        res = cat.unique()
-        exp_cat = Categorical(["b", np.nan, "a"], categories=["a", "b"], ordered=True)
-        tm.assert_categorical_equal(res, exp_cat)
-
-    def test_unique_index_series(self):
-        c = Categorical([3, 1, 2, 2, 1], categories=[3, 2, 1])
+        c = Categorical([3, 1, 2, 2, 1], dtype=dtype)
         # Categorical.unique sorts categories by appearance order
         # if ordered=False
-        exp = Categorical([3, 1, 2], categories=[3, 1, 2])
+        exp = Categorical([3, 1, 2], dtype=dtype)
         tm.assert_categorical_equal(c.unique(), exp)
 
         tm.assert_index_equal(Index(c).unique(), Index(exp))
         tm.assert_categorical_equal(Series(c).unique(), exp)
 
-        c = Categorical([1, 1, 2, 2], categories=[3, 2, 1])
-        exp = Categorical([1, 2], categories=[1, 2])
+        c = Categorical([1, 1, 2, 2], dtype=dtype)
+        exp = Categorical([1, 2], dtype=dtype)
         tm.assert_categorical_equal(c.unique(), exp)
         tm.assert_index_equal(Index(c).unique(), Index(exp))
         tm.assert_categorical_equal(Series(c).unique(), exp)
 
-        c = Categorical([3, 1, 2, 2, 1], categories=[3, 2, 1], ordered=True)
-        # Categorical.unique keeps categories order if ordered=True
-        exp = Categorical([3, 1, 2], categories=[3, 2, 1], ordered=True)
-        tm.assert_categorical_equal(c.unique(), exp)
-
-        tm.assert_index_equal(Index(c).unique(), Index(exp))
-        tm.assert_categorical_equal(Series(c).unique(), exp)
-
     def test_shift(self):
         # GH 9416
         cat = Categorical(["a", "b", "c", "d", "a"])