From 6f8c9e42028c826f267a052f46738f98040e8efc Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Jul 2021 16:12:08 -0700 Subject: [PATCH 1/2] Fix mypy for new virtualenv --- mypy/modulefinder.py | 31 +++++++++++++++++++++++++++---- mypy/{sitepkgs.py => pyinfo.py} | 22 ++++++++++++++++------ setup.py | 2 +- test-requirements.txt | 2 +- 4 files changed, 45 insertions(+), 12 deletions(-) rename mypy/{sitepkgs.py => pyinfo.py} (62%) diff --git a/mypy/modulefinder.py b/mypy/modulefinder.py index 17f386e933c1..6cebd7db8bc0 100644 --- a/mypy/modulefinder.py +++ b/mypy/modulefinder.py @@ -18,7 +18,7 @@ from mypy.fscache import FileSystemCache from mypy.options import Options from mypy.stubinfo import is_legacy_bundled_package -from mypy import sitepkgs +from mypy import pyinfo # Paths to be searched in find_module(). SearchPaths = NamedTuple( @@ -582,6 +582,27 @@ def default_lib_path(data_dir: str, return path +@functools.lru_cache(maxsize=None) +def get_prefixes(python_executable: Optional[str]) -> Tuple[str, str]: + """Get the sys.base_prefix and sys.prefix for the given python. + + This runs a subprocess call to get the prefix paths of the given Python executable. + To avoid repeatedly calling a subprocess (which can be slow!) we + lru_cache the results. + """ + if python_executable is None: + return '', '' + elif python_executable == sys.executable: + # Use running Python's package dirs + return pyinfo.getprefixes() + else: + # Use subprocess to get the package directory of given Python + # executable + return ast.literal_eval( + subprocess.check_output([python_executable, pyinfo.__file__, 'getprefixes'], + stderr=subprocess.PIPE).decode()) + + @functools.lru_cache(maxsize=None) def get_site_packages_dirs(python_executable: Optional[str]) -> Tuple[List[str], List[str]]: """Find package directories for given python. @@ -595,12 +616,12 @@ def get_site_packages_dirs(python_executable: Optional[str]) -> Tuple[List[str], return [], [] elif python_executable == sys.executable: # Use running Python's package dirs - site_packages = sitepkgs.getsitepackages() + site_packages = pyinfo.getsitepackages() else: # Use subprocess to get the package directory of given Python # executable site_packages = ast.literal_eval( - subprocess.check_output([python_executable, sitepkgs.__file__], + subprocess.check_output([python_executable, pyinfo.__file__, 'getsitepackages'], stderr=subprocess.PIPE).decode()) return expand_site_packages(site_packages) @@ -736,6 +757,8 @@ def compute_search_paths(sources: List[BuildSource], mypypath = add_py2_mypypath_entries(mypypath) egg_dirs, site_packages = get_site_packages_dirs(options.python_executable) + base_prefix, prefix = get_prefixes(options.python_executable) + is_venv = base_prefix != prefix for site_dir in site_packages: assert site_dir not in lib_path if (site_dir in mypypath or @@ -745,7 +768,7 @@ def compute_search_paths(sources: List[BuildSource], print("See https://mypy.readthedocs.io/en/stable/running_mypy.html" "#how-mypy-handles-imports for more info", file=sys.stderr) sys.exit(1) - elif site_dir in python_path: + elif site_dir in python_path and (is_venv and not site_dir.startswith(prefix)): print("{} is in the PYTHONPATH. Please change directory" " so it is not.".format(site_dir), file=sys.stderr) diff --git a/mypy/sitepkgs.py b/mypy/pyinfo.py similarity index 62% rename from mypy/sitepkgs.py rename to mypy/pyinfo.py index 79dd5d80fcf6..7da94e0a1fb9 100644 --- a/mypy/sitepkgs.py +++ b/mypy/pyinfo.py @@ -1,21 +1,25 @@ from __future__ import print_function -"""This file is used to find the site packages of a Python executable, which may be Python 2. +"""Utilities to find the site and prefix information of a Python executable, which may be Python 2. This file MUST remain compatible with Python 2. Since we cannot make any assumptions about the Python being executed, this module should not use *any* dependencies outside of the standard library found in Python 2. This file is run each mypy run, so it should be kept as fast as possible. """ +import site +import sys if __name__ == '__main__': - import sys sys.path = sys.path[1:] # we don't want to pick up mypy.types -import site - MYPY = False if MYPY: - from typing import List + from typing import List, Tuple + + +def getprefixes(): + # type: () -> Tuple[str, str] + return sys.base_prefix, sys.prefix def getsitepackages(): @@ -29,4 +33,10 @@ def getsitepackages(): if __name__ == '__main__': - print(repr(getsitepackages())) + if sys.argv[-1] == 'getsitepackages': + print(repr(getsitepackages())) + elif sys.argv[-1] == 'getprefixes': + print(repr(getprefixes())) + else: + print("ERROR: incorrect argument to pyinfo.py.", file=sys.stderr) + sys.exit(1) diff --git a/setup.py b/setup.py index 726229bea1fe..7bd5b20cbf2c 100644 --- a/setup.py +++ b/setup.py @@ -86,7 +86,7 @@ def run(self): MYPYC_BLACKLIST = tuple(os.path.join('mypy', x) for x in ( # Need to be runnable as scripts '__main__.py', - 'sitepkgs.py', + 'pyinfo.py', os.path.join('dmypy', '__main__.py'), # Uses __getattr__/__setattr__ diff --git a/test-requirements.txt b/test-requirements.txt index d80d127cc440..5879395983ef 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -12,6 +12,6 @@ pytest-forked>=1.3.0,<2.0.0 pytest-cov>=2.10.0,<3.0.0 typing>=3.5.2; python_version < '3.5' py>=1.5.2 -virtualenv<16.7.11 +virtualenv setuptools!=50 importlib-metadata==0.20 From 3074a8c2c197f75ab3b46094d4bd33eba4493559 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Wed, 21 Jul 2021 16:50:51 -0700 Subject: [PATCH 2/2] Update test-requirements to use newer importlib-metadata and remove old markers that are no longer needed --- test-requirements.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5879395983ef..93300baaaa8e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,16 +2,15 @@ -r build-requirements.txt attrs>=18.0 flake8>=3.8.1 -flake8-bugbear; python_version >= '3.5' -flake8-pyi>=20.5; python_version >= '3.6' +flake8-bugbear +flake8-pyi>=20.5 lxml>=4.4.0 psutil>=4.0 pytest>=6.2.0,<7.0.0 pytest-xdist>=1.34.0,<2.0.0 pytest-forked>=1.3.0,<2.0.0 pytest-cov>=2.10.0,<3.0.0 -typing>=3.5.2; python_version < '3.5' py>=1.5.2 -virtualenv +virtualenv>=20.6.0 setuptools!=50 -importlib-metadata==0.20 +importlib-metadata>=4.6.1,<5.0.0