From a42ee8f0697439eac3a86644729750c472438b98 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Tue, 27 Mar 2018 20:32:34 -0700 Subject: [PATCH 1/2] Fix the stubgen test flake. The core problem is that `importlib.invalidate_caches()` is full of lies and does not actually invalidate all of the relevant caches. Python maintains sys.path_importer_cache, which maps from directory names on the sys.path to "importer" objects for that path entry. `importlib.invalidate_caches()` invalidates the caches inside each of the per-directory importer objects, but there is an additional negative cache that is *not* cleared: sys.path entries whose directory doesn't exist have their importer set to None, which prevents that path entry from ever being searched unless it is manually cleared. This failure comes up rarely, because it only occurs if the following events occur in sequence: 1. 'stubgen-test-path' is added to sys.path, but no new import is done while there is a 'stubgen-test-path' subdirectory in the working directory. This is done by all the stubgen tests that don't end with `_import`. 2. Some non-stubgen test does an import of a previously unimported module. This will cause sys.path_importer_cache['stubgen-test-path'] to be set to None when the directory doesn't exist during the module search. This can happen the *first* time that a `_python2` test is run since `parse.parse()` dynamically imports `mypy.fastparse2` when asked to parse python 2. 3. A stubgen test tries to use importlib to import a module in 'stubgen-test-path'. All of the `_import` tests do this. We fix this by clearing out the relevant entry from `sys.path_importer_cache` ourselves. --- mypy/stubgen.py | 32 -------------------------------- mypy/test/teststubgen.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 39 deletions(-) diff --git a/mypy/stubgen.py b/mypy/stubgen.py index c0afb4f380ece..5cb7a10c91457 100755 --- a/mypy/stubgen.py +++ b/mypy/stubgen.py @@ -153,13 +153,6 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int], try: mod = importlib.import_module(module) except Exception: - # Print some debugging output that might help diagnose problems. - print('=== debug dump follows ===') - traceback.print_exc() - dump_sys_path() - print('PYTHONPATH: %s' % os.getenv("PYTHONPATH")) - dump_dir(os.getcwd()) - print('=== end of debug dump ===') raise CantImport(module) if is_c_module(mod): return None @@ -175,31 +168,6 @@ def find_module_path_and_all(module: str, pyversion: Tuple[int, int], return module_path, module_all -def dump_sys_path() -> None: - print('sys.path:') - for entry in sys.path: - print(' %r' % entry) - if entry in sys.path_importer_cache: - x = sys.path_importer_cache[entry] - print(' path_importer: %r / %r' % (x, getattr(x, '__dict__', None))) - try: - print(' stat: %s ' % str(os.stat(entry))) - except Exception as e: - pass - - -def dump_dir(path: str) -> None: - for root, dirs, files in os.walk(os.getcwd()): - print('%s:' % root) - for d in dirs: - print(' %s/' % d) - for f in files: - path = os.path.join(root, f) - print(' %s (%d bytes)' % (f, os.path.getsize(path))) - if not dirs and not files: - print(' (empty)') - - def load_python_module_info(module: str, interpreter: str) -> Tuple[str, Optional[List[str]]]: """Return tuple (module path, module __all__) for a Python 2 module. diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 6d84e66ba7015..828169107e98b 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -1,5 +1,4 @@ import glob -import importlib import os.path import shutil import sys @@ -125,7 +124,7 @@ def test_stubgen(testcase: DataDrivenTestCase) -> None: handle.close() # Without this we may sometimes be unable to import the module below, as importlib # caches os.listdir() results in Python 3.3+ (Guido explained this to me). - reset_importlib_caches() + reset_importlib_cache('stubgen-test-path') try: if testcase.name.endswith('_import'): generate_stub_for_module(name, out_dir, quiet=True, @@ -145,11 +144,13 @@ def test_stubgen(testcase: DataDrivenTestCase) -> None: shutil.rmtree(out_dir) -def reset_importlib_caches() -> None: - try: - importlib.invalidate_caches() - except (ImportError, AttributeError): - pass +def reset_importlib_cache(entry: str) -> None: + # importlib.invalidate_caches() is insufficient, since it doesn't + # clear cache entries indicate that a directory on the path does + # not exist, which can cause failures. + # Just directly clear the sys.path_importer_cache entry ourselves. + if entry in sys.path_importer_cache: + del sys.path_importer_cache[entry] def load_output(dirname: str) -> List[str]: From 062b089a5a44f6f500ca5ddb15f28d939849bfc9 Mon Sep 17 00:00:00 2001 From: Michael Sullivan Date: Wed, 28 Mar 2018 10:13:21 -0700 Subject: [PATCH 2/2] some doc improvements --- mypy/test/teststubgen.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index 828169107e98b..66339bd420325 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -146,9 +146,12 @@ def test_stubgen(testcase: DataDrivenTestCase) -> None: def reset_importlib_cache(entry: str) -> None: # importlib.invalidate_caches() is insufficient, since it doesn't - # clear cache entries indicate that a directory on the path does - # not exist, which can cause failures. - # Just directly clear the sys.path_importer_cache entry ourselves. + # clear cache entries that indicate that a directory on the path + # does not exist, which can cause failures. Just directly clear + # the sys.path_importer_cache entry ourselves. Other possible + # workarounds include always using different paths in the sys.path + # (perhaps by using the full path name) or removing the entry from + # sys.path after each run. if entry in sys.path_importer_cache: del sys.path_importer_cache[entry]