Allow for na values that are of same type as the data

Author: topper-123

doc/source/whatsnew/v1.2.0.rst

@@ -484,6 +484,7 @@ Deprecations
 - Deprecated :meth:`Index.asi8` for :class:`Index` subclasses other than :class:`.DatetimeIndex`, :class:`.TimedeltaIndex`, and :class:`PeriodIndex` (:issue:`37877`)
 - The ``inplace`` parameter of :meth:`Categorical.remove_unused_categories` is deprecated and will be removed in a future version (:issue:`37643`)
 - The ``null_counts`` parameter of :meth:`DataFrame.info` is deprecated and replaced by ``show_counts``. It will be removed in a future version (:issue:`37999`)
+- Deprecated membership checks for nan-likes in :class:`Categorical`. In the future the membership check will only return True if the nan-like is ``nan`` or of the same dtype as the underlying categories (:issue:`37867`)
 
 .. ---------------------------------------------------------------------------
 

pandas/core/arrays/base.py

@@ -362,10 +362,13 @@ def __contains__(self, item) -> bool:
         # GH37867
         # comparisons of any item to pd.NA always return pd.NA, so e.g. "a" in [pd.NA]
         # would raise a TypeError. The implementation below works around that.
-        if item is self.dtype.na_value:
-            return self.isna().any() if self._can_hold_na else False
-        elif is_scalar(item) and isna(item):
-            return False
+        if is_scalar(item) and isna(item):
+            if not self._can_hold_na:
+                return False
+            elif item is self.dtype.na_value or isinstance(item, self.dtype.type):
+                return self.isna().any()
+            else:
+                return False
         else:
             return (item == self).any()
 

pandas/core/arrays/categorical.py

@@ -1745,8 +1745,16 @@ def __contains__(self, key) -> bool:
         """
         Returns True if `key` is in this Categorical.
         """
+        # in pandas 2.0, remove this method.
+
         # if key is a NaN, check if any NaN is in self.
         if is_valid_nat_for_dtype(key, self.categories.dtype):
+            if key is not self.dtype.na_value and not isinstance(key, self.dtype.type):
+                warn(f"Membership check with {key} will return False in the future. "
+                     f"Consider using {self.dtype.na_value} instead",
+                     FutureWarning,
+                     stacklevel=2,
+                     )
             return self.isna().any()
 
         return contains(self, key, container=self._codes)

pandas/core/arrays/numpy_.py

@@ -51,6 +51,13 @@ def numpy_dtype(self) -> np.dtype:
         """
         return self._dtype
 
+    @property
+    def na_value(self) -> object:
+        if issubclass(self.type, np.floating):
+            return self.type("nan")
+        else:
+            return super().na_value
+
     @property
     def name(self) -> str:
         """

pandas/tests/extension/arrow/test_bool.py

@@ -50,6 +50,10 @@ def test_view(self, data):
         # __setitem__ does not work, so we only have a smoke-test
         data.view()
 
+    @pytest.mark.xfail(raises=AssertionError, reason="Not implemented yet")
+    def test_contains(self, data, data_missing):
+        super().test_contains(data, data_missing)
+
 
 class TestConstructors(BaseArrowTests, base.BaseConstructorsTests):
     def test_from_dtype(self, data):

pandas/tests/extension/decimal/array.py

@@ -155,6 +155,14 @@ def __setitem__(self, key, value):
     def __len__(self) -> int:
         return len(self._data)
 
+    def __contains__(self, item) -> bool:
+        if not isinstance(item, decimal.Decimal):
+            return False
+        elif item.is_nan():
+            return self.isna().any()
+        else:
+            return super().__contains__(item)
+
     @property
     def nbytes(self) -> int:
         n = len(self)

pandas/tests/extension/test_categorical.py

@@ -87,6 +87,32 @@ def test_memory_usage(self, data):
         # Is this deliberate?
         super().test_memory_usage(data)
 
+    def test_contains(self, data, data_missing):
+        # GH-37867
+        # na value handling in Categorical.__contains__ is deprecated.
+        # See base.BaseInterFaceTests.test_contains for more details.
+
+        na_value = data.dtype.na_value
+        # ensure data without missing values
+        data = data[~data.isna()]
+
+        # first elements are non-missing
+        assert data[0] in data
+        assert data_missing[0] in data_missing
+
+        # check the presence of na_value
+        assert na_value in data_missing
+        assert na_value not in data
+
+        # the data can never contain other nan-likes than na_value
+        for na_value_type in {None, np.nan, pd.NA, pd.NaT}:
+            if na_value_type is na_value:
+                continue
+
+            with tm.assert_produces_warning(FutureWarning, check_stacklevel=False):
+                assert na_value_type not in data
+                assert na_value_type in data_missing
+
 
 class TestConstructors(base.BaseConstructorsTests):
     pass