bpo-23882: unittest: Drop PEP 420 support from discovery. (GH-29745)

Author: methane

Doc/library/unittest.rst

@@ -266,8 +266,7 @@ Test Discovery
 
 Unittest supports simple test discovery. In order to be compatible with test
 discovery, all of the test files must be :ref:`modules <tut-modules>` or
-:ref:`packages <tut-packages>` (including :term:`namespace packages
-<namespace package>`) importable from the top-level directory of
+:ref:`packages <tut-packages>` importable from the top-level directory of
 the project (this means that their filenames must be valid :ref:`identifiers
 <identifiers>`).
 
@@ -340,6 +339,24 @@ the `load_tests protocol`_.
    directory too (e.g.
    ``python -m unittest discover -s root/namespace -t root``).
 
+.. versionchanged:: 3.11
+   Python 3.11 dropped the :term:`namespace packages <namespace package>`
+   support. It has been broken since Python 3.7. Start directory and
+   subdirectories containing tests must be regular package that have
+   ``__init__.py`` file.
+
+   Directories containing start directory still can be a namespace package.
+   In this case, you need to specify start directory as dotted package name,
+   and target directory explicitly. For example::
+
+      # proj/  <-- current directory
+      #   namespace/
+      #     mypkg/
+      #       __init__.py
+      #       test_mypkg.py
+
+      python -m unittest discover -s namespace.mypkg -t .
+
 
 .. _organizing-tests:
 
@@ -1858,6 +1875,10 @@ Loading and running tests
          whether their path matches *pattern*, because it is impossible for
          a package name to match the default pattern.
 
+      .. versionchanged:: 3.11
+         *start_dir* can not be a :term:`namespace packages <namespace package>`.
+         It has been broken since Python 3.7 and Python 3.11 officially remove it.
+
 
    The following attributes of a :class:`TestLoader` can be configured either by
    subclassing or assignment on an instance:

Doc/whatsnew/3.11.rst

@@ -542,6 +542,10 @@ Removed
 
   (Contributed by Hugo van Kemenade in :issue:`45320`.)
 
+* Remove namespace package support from unittest discovery. It was introduced in
+  Python 3.4 but has been broken since Python 3.7.
+  (Contributed by Inada Naoki in :issue:`23882`.)
+
 
 Porting to Python 3.11
 ======================

Lib/unittest/loader.py

@@ -264,8 +264,6 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None):
         self._top_level_dir = top_level_dir
 
         is_not_importable = False
-        is_namespace = False
-        tests = []
         if os.path.isdir(os.path.abspath(start_dir)):
             start_dir = os.path.abspath(start_dir)
             if start_dir != top_level_dir:
@@ -281,50 +279,25 @@ def discover(self, start_dir, pattern='test*.py', top_level_dir=None):
                 top_part = start_dir.split('.')[0]
                 try:
                     start_dir = os.path.abspath(
-                       os.path.dirname((the_module.__file__)))
+                        os.path.dirname((the_module.__file__)))
                 except AttributeError:
-                    # look for namespace packages
-                    try:
-                        spec = the_module.__spec__
-                    except AttributeError:
-                        spec = None
-
-                    if spec and spec.loader is None:
-                        if spec.submodule_search_locations is not None:
-                            is_namespace = True
-
-                            for path in the_module.__path__:
-                                if (not set_implicit_top and
-                                    not path.startswith(top_level_dir)):
-                                    continue
-                                self._top_level_dir = \
-                                    (path.split(the_module.__name__
-                                         .replace(".", os.path.sep))[0])
-                                tests.extend(self._find_tests(path,
-                                                              pattern,
-                                                              namespace=True))
-                    elif the_module.__name__ in sys.builtin_module_names:
+                    if the_module.__name__ in sys.builtin_module_names:
                         # builtin module
                         raise TypeError('Can not use builtin modules '
                                         'as dotted module names') from None
                     else:
                         raise TypeError(
-                            'don\'t know how to discover from {!r}'
-                            .format(the_module)) from None
+                            f"don't know how to discover from {the_module!r}"
+                            ) from None
 
                 if set_implicit_top:
-                    if not is_namespace:
-                        self._top_level_dir = \
-                           self._get_directory_containing_module(top_part)
-                        sys.path.remove(top_level_dir)
-                    else:
-                        sys.path.remove(top_level_dir)
+                    self._top_level_dir = self._get_directory_containing_module(top_part)
+                    sys.path.remove(top_level_dir)
 
         if is_not_importable:
             raise ImportError('Start directory is not importable: %r' % start_dir)
 
-        if not is_namespace:
-            tests = list(self._find_tests(start_dir, pattern))
+        tests = list(self._find_tests(start_dir, pattern))
         return self.suiteClass(tests)
 
     def _get_directory_containing_module(self, module_name):
@@ -359,7 +332,7 @@ def _match_path(self, path, full_path, pattern):
         # override this method to use alternative matching strategy
         return fnmatch(path, pattern)
 
-    def _find_tests(self, start_dir, pattern, namespace=False):
+    def _find_tests(self, start_dir, pattern):
         """Used by discovery. Yields test suites it loads."""
         # Handle the __init__ in this package
         name = self._get_name_from_path(start_dir)
@@ -368,8 +341,7 @@ def _find_tests(self, start_dir, pattern, namespace=False):
         if name != '.' and name not in self._loading_packages:
             # name is in self._loading_packages while we have called into
             # loadTestsFromModule with name.
-            tests, should_recurse = self._find_test_path(
-                start_dir, pattern, namespace)
+            tests, should_recurse = self._find_test_path(start_dir, pattern)
             if tests is not None:
                 yield tests
             if not should_recurse:
@@ -380,20 +352,19 @@ def _find_tests(self, start_dir, pattern, namespace=False):
         paths = sorted(os.listdir(start_dir))
         for path in paths:
             full_path = os.path.join(start_dir, path)
-            tests, should_recurse = self._find_test_path(
-                full_path, pattern, namespace)
+            tests, should_recurse = self._find_test_path(full_path, pattern)
             if tests is not None:
                 yield tests
             if should_recurse:
                 # we found a package that didn't use load_tests.
                 name = self._get_name_from_path(full_path)
                 self._loading_packages.add(name)
                 try:
-                    yield from self._find_tests(full_path, pattern, namespace)
+                    yield from self._find_tests(full_path, pattern)
                 finally:
                     self._loading_packages.discard(name)
 
-    def _find_test_path(self, full_path, pattern, namespace=False):
+    def _find_test_path(self, full_path, pattern):
         """Used by discovery.
 
         Loads tests from a single file, or a directories' __init__.py when
@@ -437,8 +408,7 @@ def _find_test_path(self, full_path, pattern, namespace=False):
                         msg % (mod_name, module_dir, expected_dir))
                 return self.loadTestsFromModule(module, pattern=pattern), False
         elif os.path.isdir(full_path):
-            if (not namespace and
-                not os.path.isfile(os.path.join(full_path, '__init__.py'))):
+            if not os.path.isfile(os.path.join(full_path, '__init__.py')):
                 return None, False
 
             load_tests = None

Lib/unittest/test/test_discovery.py

@@ -396,7 +396,7 @@ def restore_isdir():
         self.addCleanup(restore_isdir)
 
         _find_tests_args = []
-        def _find_tests(start_dir, pattern, namespace=None):
+        def _find_tests(start_dir, pattern):
             _find_tests_args.append((start_dir, pattern))
             return ['tests']
         loader._find_tests = _find_tests
@@ -792,7 +792,7 @@ def test_discovery_from_dotted_path(self):
         expectedPath = os.path.abspath(os.path.dirname(unittest.test.__file__))
 
         self.wasRun = False
-        def _find_tests(start_dir, pattern, namespace=None):
+        def _find_tests(start_dir, pattern):
             self.wasRun = True
             self.assertEqual(start_dir, expectedPath)
             return tests
@@ -825,37 +825,6 @@ def restore():
                          'Can not use builtin modules '
                          'as dotted module names')
 
-    def test_discovery_from_dotted_namespace_packages(self):
-        loader = unittest.TestLoader()
-
-        package = types.ModuleType('package')
-        package.__path__ = ['/a', '/b']
-        package.__spec__ = types.SimpleNamespace(
-           loader=None,
-           submodule_search_locations=['/a', '/b']
-        )
-
-        def _import(packagename, *args, **kwargs):
-            sys.modules[packagename] = package
-            return package
-
-        _find_tests_args = []
-        def _find_tests(start_dir, pattern, namespace=None):
-            _find_tests_args.append((start_dir, pattern))
-            return ['%s/tests' % start_dir]
-
-        loader._find_tests = _find_tests
-        loader.suiteClass = list
-
-        with unittest.mock.patch('builtins.__import__', _import):
-            # Since loader.discover() can modify sys.path, restore it when done.
-            with import_helper.DirsOnSysPath():
-                # Make sure to remove 'package' from sys.modules when done.
-                with test.test_importlib.util.uncache('package'):
-                    suite = loader.discover('package')
-
-        self.assertEqual(suite, ['/a/tests', '/b/tests'])
-
     def test_discovery_failed_discovery(self):
         loader = unittest.TestLoader()
         package = types.ModuleType('package')

Misc/NEWS.d/next/Library/2021-11-24-19-09-14.bpo-23882._tctCv.rst

@@ -0,0 +1,2 @@
+Remove namespace package (PEP 420) support from unittest discovery. It was
+introduced in Python 3.4 but has been broken since Python 3.7.