From eee27705c7efd11d6e69e987229ccfc1940421b3 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 17 Sep 2023 15:21:04 +0200 Subject: [PATCH 01/15] WIP --- tests/mypy_test.py | 32 +++++++++++++++++++++++--------- tests/parse_metadata.py | 17 +++++++++++++++++ tests/stubtest_third_party.py | 8 +++++++- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 3d10ad689466..31baa6dadcfe 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -25,7 +25,7 @@ import tomli -from parse_metadata import PackageDependencies, get_recursive_requirements +from parse_metadata import PYTHON_VERSION, PackageDependencies, get_recursive_requirements, read_metadata from utils import ( VERSIONS_RE as VERSION_LINE_RE, VenvInfo, @@ -307,6 +307,7 @@ def add_third_party_files( class TestResults(NamedTuple): exit_code: int files_checked: int + packages_skipped: int = 0 def test_third_party_distribution( @@ -471,6 +472,7 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults: print("Testing third-party packages...") files_checked = 0 + packages_skipped = 0 gitignore_spec = get_gitignore_spec() distributions_to_check: dict[str, PackageDependencies] = {} @@ -480,6 +482,11 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe if spec_matches_path(gitignore_spec, distribution_path): continue + dist_metadata = read_metadata(distribution) + if dist_metadata.requires_python and not dist_metadata.requires_python.contains(PYTHON_VERSION): + packages_skipped += 1 + continue + if ( distribution_path in args.filter or Path("stubs") in args.filter @@ -497,30 +504,32 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items(): non_types_dependencies = venv_info.python_exe != sys.executable - this_code, checked = test_third_party_distribution( + this_code, checked, _ = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) code = max(code, this_code) files_checked += checked - return TestResults(code, files_checked) + return TestResults(code, files_checked, packages_skipped) def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: print(f"*** Testing Python {args.version} on {args.platform}") files_checked_this_version = 0 + packages_skipped_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - code, stdlib_files_checked = test_stdlib(code, args) + code, stdlib_files_checked, _ = test_stdlib(code, args) files_checked_this_version += stdlib_files_checked print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): - code, third_party_files_checked = test_third_party_stubs(code, args, tempdir) + code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir) files_checked_this_version += third_party_files_checked + packages_skipped_this_version = third_party_packages_skipped print() - return TestResults(code, files_checked_this_version) + return TestResults(code, files_checked_this_version, packages_skipped_this_version) def main() -> None: @@ -531,19 +540,24 @@ def main() -> None: exclude = args.exclude or [] code = 0 total_files_checked = 0 + total_packages_skipped = 0 with tempfile.TemporaryDirectory() as td: td_path = Path(td) for version, platform in product(versions, platforms): config = TestConfig(args.verbose, filter, exclude, version, platform) - code, files_checked_this_version = test_typeshed(code, args=config, tempdir=td_path) + code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path) total_files_checked += files_checked_this_version + total_packages_skipped += packages_skipped_this_version if code: print_error(f"--- exit status {code}, {total_files_checked} files checked ---") sys.exit(code) - if not total_files_checked: + if total_packages_skipped: + print(colored(f"--- {total_packages_skipped} packages skipped ---", "yellow")) + elif not total_files_checked: print_error("--- nothing to do; exit 1 ---") sys.exit(1) - print(colored(f"--- success, {total_files_checked} files checked ---", "green")) + if total_files_checked: + print(colored(f"--- success, {total_files_checked} files checked ---", "green")) if __name__ == "__main__": diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index f6148e6e399d..6fb8f6c48e41 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -6,6 +6,7 @@ import os import re +import sys import urllib.parse from collections.abc import Mapping from dataclasses import dataclass @@ -15,6 +16,7 @@ import tomli from packaging.requirements import Requirement +from packaging.specifiers import Specifier from packaging.version import Version from utils import cache @@ -30,6 +32,8 @@ "read_stubtest_settings", ] +OLDEST_SUPPORTED_PYTHON: Final = "3.7" +PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" _STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"} # Some older websites have a bad pattern of using query params for navigation. @@ -130,6 +134,7 @@ class StubMetadata: uploaded_to_pypi: Annotated[bool, "Whether or not a distribution is uploaded to PyPI"] partial_stub: Annotated[bool, "Whether this is a partial type stub package as per PEP 561."] stubtest_settings: StubtestSettings + requires_python: Annotated[Specifier | None, "Versions of Python supported by the stub package"] _KNOWN_METADATA_FIELDS: Final = frozenset( @@ -144,6 +149,7 @@ class StubMetadata: "upload", "tool", "partial_stub", + "requires_python", } ) _KNOWN_METADATA_TOOL_FIELDS: Final = { @@ -240,6 +246,16 @@ def read_metadata(distribution: str) -> StubMetadata: assert type(uploaded_to_pypi) is bool partial_stub: object = data.get("partial_stub", True) assert type(partial_stub) is bool + requires_python_str: object = data.get("requires_python") + if requires_python_str is None: + requires_python = None + else: + assert isinstance(requires_python_str, str) + requires_python = Specifier(requires_python_str) + # Check minimum Python version is not less than the oldest version of Python supported by typeshed + assert Specifier(f">={OLDEST_SUPPORTED_PYTHON}").contains( + requires_python.version + ), f"'requires_python' contains versions lower than the oldest supported Python {OLDEST_SUPPORTED_PYTHON}" empty_tools: dict[object, object] = {} tools_settings: object = data.get("tool", empty_tools) @@ -262,6 +278,7 @@ def read_metadata(distribution: str) -> StubMetadata: uploaded_to_pypi=uploaded_to_pypi, partial_stub=partial_stub, stubtest_settings=read_stubtest_settings(distribution), + requires_python=requires_python, ) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index af393c35f48e..a1738facbe16 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -12,7 +12,7 @@ from textwrap import dedent from typing import NoReturn -from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata +from parse_metadata import PYTHON_VERSION, NoSuchStubError, get_recursive_requirements, read_metadata from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg @@ -37,6 +37,12 @@ def run_stubtest( return True print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow")) + if metadata.requires_python: + if not metadata.requires_python.contains(PYTHON_VERSION): + print(colored(f"skipping (requires python {metadata.requires_python})", "yellow")) + return True + print(colored(f"Note: {dist_name} requires python {metadata.requires_python}.", "yellow")) + with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) try: From db9b3caa61be0f6fb3dabbca7fef86ef8841b3f4 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 17 Sep 2023 19:20:57 +0200 Subject: [PATCH 02/15] Also check in regr_test --- tests/regr_test.py | 7 ++++++- tests/stubtest_third_party.py | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 6746b2bc7ed8..0d6166aab6db 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -15,7 +15,7 @@ from pathlib import Path from typing_extensions import TypeAlias -from parse_metadata import get_recursive_requirements +from parse_metadata import PYTHON_VERSION, get_recursive_requirements, read_metadata from utils import ( PackageInfo, VenvInfo, @@ -254,6 +254,11 @@ def main() -> ReturnCode: code = 0 for testcase_dir in testcase_directories: + if not testcase_dir.is_stdlib: + metadata = read_metadata(testcase_dir.name) + if metadata.requires_python and not metadata.requires_python.contains(PYTHON_VERSION): + print(colored(f"skipping {testcase_dir.name!r} (requires Python {metadata.requires_python})", "yellow")) + continue with tempfile.TemporaryDirectory() as td: tempdir = Path(td) for platform, version in product(platforms_to_test, versions_to_test): diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index a1738facbe16..fa49c4b45716 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -39,9 +39,9 @@ def run_stubtest( if metadata.requires_python: if not metadata.requires_python.contains(PYTHON_VERSION): - print(colored(f"skipping (requires python {metadata.requires_python})", "yellow")) + print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow")) return True - print(colored(f"Note: {dist_name} requires python {metadata.requires_python}.", "yellow")) + print(colored(f"Note: {dist_name} requires Python {metadata.requires_python}.", "yellow")) with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) From 1e1c1dcde884eba07515d2627f6a04d9d1f16152 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 17 Sep 2023 19:26:08 +0200 Subject: [PATCH 03/15] mypy --- tests/regr_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/regr_test.py b/tests/regr_test.py index 0d6166aab6db..168d3ac2fdef 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -254,6 +254,7 @@ def main() -> ReturnCode: code = 0 for testcase_dir in testcase_directories: + assert isinstance(testcase_dir, PackageInfo) if not testcase_dir.is_stdlib: metadata = read_metadata(testcase_dir.name) if metadata.requires_python and not metadata.requires_python.contains(PYTHON_VERSION): From 32625b37ed87ac262ae0865ee2909787d82c283c Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 17 Sep 2023 19:39:35 +0200 Subject: [PATCH 04/15] check specifier operator --- tests/parse_metadata.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index 6fb8f6c48e41..3c443fb085d5 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -256,6 +256,7 @@ def read_metadata(distribution: str) -> StubMetadata: assert Specifier(f">={OLDEST_SUPPORTED_PYTHON}").contains( requires_python.version ), f"'requires_python' contains versions lower than the oldest supported Python {OLDEST_SUPPORTED_PYTHON}" + assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'" empty_tools: dict[object, object] = {} tools_settings: object = data.get("tool", empty_tools) From be228009a7f17b7370c576691c6d87dfe8af8ed2 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Wed, 20 Sep 2023 21:59:28 +0200 Subject: [PATCH 05/15] CR - move to pyproject.toml - add to __all__ - parens in message - mininum supported version is also not allowed - delete print - test against python runtime version and target versions (mypy and regr tests) - different messages for different reasons - default the specifier to >=oldest_supported --- pyproject.toml | 1 + tests/mypy_test.py | 11 +++++++++-- tests/parse_metadata.py | 15 ++++++++++----- tests/regr_test.py | 14 +++++++++++--- tests/stubtest_third_party.py | 8 +++----- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6cbb5256fa31..626d15639cf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,3 +98,4 @@ select = [ [tool.typeshed] pyright_version = "1.1.326" +oldest_supported_python = "3.7" diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 31baa6dadcfe..b30ecee72466 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -482,8 +482,15 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe if spec_matches_path(gitignore_spec, distribution_path): continue - dist_metadata = read_metadata(distribution) - if dist_metadata.requires_python and not dist_metadata.requires_python.contains(PYTHON_VERSION): + metadata = read_metadata(distribution) + if not metadata.requires_python.contains(PYTHON_VERSION): + msg = f"skipping {distribution!r} on Python {PYTHON_VERSION} (requires Python {metadata.requires_python})" + print(colored(msg, "yellow")) + packages_skipped += 1 + continue + if not metadata.requires_python.contains(args.version): + msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})" + print(colored(msg, "yellow")) packages_skipped += 1 continue diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index 3c443fb085d5..1783679a1b33 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -30,9 +30,12 @@ "read_dependencies", "read_metadata", "read_stubtest_settings", + "OLDEST_SUPPORTED_PYTHON", + "PYTHON_VERSION", ] -OLDEST_SUPPORTED_PYTHON: Final = "3.7" +with open("pyproject.toml", "rb") as config: + OLDEST_SUPPORTED_PYTHON: Final[str] = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"] PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" _STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"} @@ -134,7 +137,7 @@ class StubMetadata: uploaded_to_pypi: Annotated[bool, "Whether or not a distribution is uploaded to PyPI"] partial_stub: Annotated[bool, "Whether this is a partial type stub package as per PEP 561."] stubtest_settings: StubtestSettings - requires_python: Annotated[Specifier | None, "Versions of Python supported by the stub package"] + requires_python: Annotated[Specifier, "Versions of Python supported by the stub package"] _KNOWN_METADATA_FIELDS: Final = frozenset( @@ -247,15 +250,17 @@ def read_metadata(distribution: str) -> StubMetadata: partial_stub: object = data.get("partial_stub", True) assert type(partial_stub) is bool requires_python_str: object = data.get("requires_python") + oldest_supported_python_specifier = Specifier(f">={OLDEST_SUPPORTED_PYTHON}") if requires_python_str is None: - requires_python = None + requires_python = oldest_supported_python_specifier else: assert isinstance(requires_python_str, str) requires_python = Specifier(requires_python_str) + assert requires_python != oldest_supported_python_specifier, f'requires_python="{requires_python}" is redundant' # Check minimum Python version is not less than the oldest version of Python supported by typeshed - assert Specifier(f">={OLDEST_SUPPORTED_PYTHON}").contains( + assert oldest_supported_python_specifier.contains( requires_python.version - ), f"'requires_python' contains versions lower than the oldest supported Python {OLDEST_SUPPORTED_PYTHON}" + ), f"'requires_python' contains versions lower than the oldest supported Python ({OLDEST_SUPPORTED_PYTHON})" assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'" empty_tools: dict[object, object] = {} diff --git a/tests/regr_test.py b/tests/regr_test.py index 168d3ac2fdef..3ccdb2c1ab10 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -250,19 +250,27 @@ def main() -> ReturnCode: platforms_to_test, versions_to_test = SUPPORTED_PLATFORMS, SUPPORTED_VERSIONS else: platforms_to_test = args.platforms_to_test or [sys.platform] - versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"] + versions_to_test = args.versions_to_test or [PYTHON_VERSION] code = 0 for testcase_dir in testcase_directories: assert isinstance(testcase_dir, PackageInfo) + metadata = None if not testcase_dir.is_stdlib: metadata = read_metadata(testcase_dir.name) - if metadata.requires_python and not metadata.requires_python.contains(PYTHON_VERSION): - print(colored(f"skipping {testcase_dir.name!r} (requires Python {metadata.requires_python})", "yellow")) + if not metadata.requires_python.contains(PYTHON_VERSION): + msg = f"skipping {testcase_dir.name!r} on Python {PYTHON_VERSION} (requires Python {metadata.requires_python})" + print(colored(msg, "yellow")) continue with tempfile.TemporaryDirectory() as td: tempdir = Path(td) for platform, version in product(platforms_to_test, versions_to_test): + if not testcase_dir.is_stdlib: + assert metadata is not None + if not metadata.requires_python.contains(version): + msg = f"skipping {testcase_dir.name!r} for target Python {version} (requires Python {metadata.requires_python})" + print(colored(msg, "yellow")) + continue this_code = test_testcase_directory(testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) code = max(code, this_code) if code: diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index fa49c4b45716..d1538202c8f9 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -37,11 +37,9 @@ def run_stubtest( return True print(colored(f"Note: {dist_name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow")) - if metadata.requires_python: - if not metadata.requires_python.contains(PYTHON_VERSION): - print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow")) - return True - print(colored(f"Note: {dist_name} requires Python {metadata.requires_python}.", "yellow")) + if not metadata.requires_python.contains(PYTHON_VERSION): + print(colored(f"skipping (requires Python {metadata.requires_python})", "yellow")) + return True with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) From 549ff525a32102a32abb3234a8e543565c34d09b Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Wed, 20 Sep 2023 22:54:20 +0200 Subject: [PATCH 06/15] Alex's suggestion --- tests/parse_metadata.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index 1783679a1b33..863f0a2a76ee 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -30,12 +30,9 @@ "read_dependencies", "read_metadata", "read_stubtest_settings", - "OLDEST_SUPPORTED_PYTHON", "PYTHON_VERSION", ] -with open("pyproject.toml", "rb") as config: - OLDEST_SUPPORTED_PYTHON: Final[str] = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"] PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" _STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"} @@ -47,6 +44,14 @@ def _is_list_of_strings(obj: object) -> TypeGuard[list[str]]: return isinstance(obj, list) and all(isinstance(item, str) for item in obj) +@cache +def _get_oldest_supported_python() -> str: + with open("pyproject.toml", "rb") as config: + val = tomli.load(config)["tool"]["typeshed"]["oldest_supported_python"] + assert type(val) is str + return val + + @final @dataclass(frozen=True) class StubtestSettings: @@ -250,7 +255,8 @@ def read_metadata(distribution: str) -> StubMetadata: partial_stub: object = data.get("partial_stub", True) assert type(partial_stub) is bool requires_python_str: object = data.get("requires_python") - oldest_supported_python_specifier = Specifier(f">={OLDEST_SUPPORTED_PYTHON}") + oldest_supported_python = _get_oldest_supported_python() + oldest_supported_python_specifier = Specifier(f">={oldest_supported_python}") if requires_python_str is None: requires_python = oldest_supported_python_specifier else: @@ -260,7 +266,7 @@ def read_metadata(distribution: str) -> StubMetadata: # Check minimum Python version is not less than the oldest version of Python supported by typeshed assert oldest_supported_python_specifier.contains( requires_python.version - ), f"'requires_python' contains versions lower than the oldest supported Python ({OLDEST_SUPPORTED_PYTHON})" + ), f"'requires_python' contains versions lower than the oldest supported Python ({oldest_supported_python})" assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'" empty_tools: dict[object, object] = {} From ac949cd70ba8b93d855e8ba6927837f903b179a9 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Wed, 20 Sep 2023 23:00:39 +0200 Subject: [PATCH 07/15] Move to utils --- tests/mypy_test.py | 3 ++- tests/parse_metadata.py | 3 --- tests/regr_test.py | 3 ++- tests/stubtest_third_party.py | 4 ++-- tests/utils.py | 5 ++++- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index b30ecee72466..c2223cc3652d 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -25,8 +25,9 @@ import tomli -from parse_metadata import PYTHON_VERSION, PackageDependencies, get_recursive_requirements, read_metadata +from parse_metadata import PackageDependencies, get_recursive_requirements, read_metadata from utils import ( + PYTHON_VERSION, VERSIONS_RE as VERSION_LINE_RE, VenvInfo, colored, diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index 863f0a2a76ee..2dff8ae8bcc6 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -6,7 +6,6 @@ import os import re -import sys import urllib.parse from collections.abc import Mapping from dataclasses import dataclass @@ -30,10 +29,8 @@ "read_dependencies", "read_metadata", "read_stubtest_settings", - "PYTHON_VERSION", ] -PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" _STUBTEST_PLATFORM_MAPPING: Final = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"} # Some older websites have a bad pattern of using query params for navigation. diff --git a/tests/regr_test.py b/tests/regr_test.py index 3ccdb2c1ab10..c5ce22afd751 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -15,8 +15,9 @@ from pathlib import Path from typing_extensions import TypeAlias -from parse_metadata import PYTHON_VERSION, get_recursive_requirements, read_metadata +from parse_metadata import get_recursive_requirements, read_metadata from utils import ( + PYTHON_VERSION, PackageInfo, VenvInfo, colored, diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index d1538202c8f9..9b4e301d049e 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -12,8 +12,8 @@ from textwrap import dedent from typing import NoReturn -from parse_metadata import PYTHON_VERSION, NoSuchStubError, get_recursive_requirements, read_metadata -from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg +from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata +from utils import PYTHON_VERSION, colored, get_mypy_req, make_venv, print_error, print_success_msg def run_stubtest( diff --git a/tests/utils.py b/tests/utils.py index dd5243a5905c..2fd376314057 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,7 +9,7 @@ import venv from functools import lru_cache from pathlib import Path -from typing import Any, NamedTuple +from typing import Any, Final, NamedTuple from typing_extensions import Annotated import pathspec @@ -22,6 +22,9 @@ def colored(text: str, color: str | None = None, **kwargs: Any) -> str: # type: return text +PYTHON_VERSION: Final = f"{sys.version_info.major}.{sys.version_info.minor}" + + # A backport of functools.cache for Python <3.9 # This module is imported by mypy_test.py, which needs to run on 3.8 in CI cache = lru_cache(None) From ef749c42624c40b8955b02f05e4ce152d3b75662 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Wed, 20 Sep 2023 23:07:04 +0200 Subject: [PATCH 08/15] Apply suggestions from code review Co-authored-by: Alex Waygood --- tests/parse_metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parse_metadata.py b/tests/parse_metadata.py index 2dff8ae8bcc6..dfb266ad08fb 100644 --- a/tests/parse_metadata.py +++ b/tests/parse_metadata.py @@ -257,13 +257,13 @@ def read_metadata(distribution: str) -> StubMetadata: if requires_python_str is None: requires_python = oldest_supported_python_specifier else: - assert isinstance(requires_python_str, str) + assert type(requires_python_str) is str requires_python = Specifier(requires_python_str) assert requires_python != oldest_supported_python_specifier, f'requires_python="{requires_python}" is redundant' # Check minimum Python version is not less than the oldest version of Python supported by typeshed assert oldest_supported_python_specifier.contains( requires_python.version - ), f"'requires_python' contains versions lower than the oldest supported Python ({oldest_supported_python})" + ), f"'requires_python' contains versions lower than typeshed's oldest supported Python ({oldest_supported_python})" assert requires_python.operator == ">=", "'requires_python' should be a minimum version specifier, use '>=3.x'" empty_tools: dict[object, object] = {} From 7b1043e11e7b0ac867d9aa77b478e32c7ff4a746 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 23 Sep 2023 10:17:02 +0200 Subject: [PATCH 09/15] Fix assertion error, actually skip mypy test when needed, improve messages --- tests/mypy_test.py | 33 ++++++++++++++++++++++++++------- tests/regr_test.py | 12 ++++++++++-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index c2223cc3652d..ddb280da9fba 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -485,7 +485,10 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe metadata = read_metadata(distribution) if not metadata.requires_python.contains(PYTHON_VERSION): - msg = f"skipping {distribution!r} on Python {PYTHON_VERSION} (requires Python {metadata.requires_python})" + msg = ( + f"skipping {distribution!r} (requires Python {metadata.requires_python}; " + f"test is being run using Python {PYTHON_VERSION})" + ) print(colored(msg, "yellow")) packages_skipped += 1 continue @@ -508,9 +511,21 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe if not _DISTRIBUTION_TO_VENV_MAPPING: setup_virtual_environments(distributions_to_check, args, tempdir) - assert _DISTRIBUTION_TO_VENV_MAPPING.keys() == distributions_to_check.keys() - - for distribution, venv_info in _DISTRIBUTION_TO_VENV_MAPPING.items(): + # Some distributions may have been skipped earlier and therefore don't have a venv. + distributions_without_venv = { + distribution: requirements + for distribution, requirements in distributions_to_check.items() + if distribution not in _DISTRIBUTION_TO_VENV_MAPPING + } + if distributions_without_venv: + setup_virtual_environments(distributions_without_venv, args, tempdir) + + # Check that there is a venv for every distribution we're testing. + # Some venvs may exist from previous runs but are skipped in this run. + assert _DISTRIBUTION_TO_VENV_MAPPING.keys() >= distributions_to_check.keys() + + for distribution in distributions_to_check: + venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution] non_types_dependencies = venv_info.python_exe != sys.executable this_code, checked, _ = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies @@ -557,15 +572,19 @@ def main() -> None: total_files_checked += files_checked_this_version total_packages_skipped += packages_skipped_this_version if code: - print_error(f"--- exit status {code}, {total_files_checked} files checked ---") + plural = "" if total_files_checked == 1 else "s" + print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---") sys.exit(code) if total_packages_skipped: - print(colored(f"--- {total_packages_skipped} packages skipped ---", "yellow")) + {1: ""}.get(total_packages_skipped, "s") + plural = "" if total_packages_skipped == 1 else "s" + print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow")) elif not total_files_checked: print_error("--- nothing to do; exit 1 ---") sys.exit(1) if total_files_checked: - print(colored(f"--- success, {total_files_checked} files checked ---", "green")) + plural = "" if total_files_checked == 1 else "s" + print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green")) if __name__ == "__main__": diff --git a/tests/regr_test.py b/tests/regr_test.py index c5ce22afd751..015336ea54da 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -254,13 +254,17 @@ def main() -> ReturnCode: versions_to_test = args.versions_to_test or [PYTHON_VERSION] code = 0 + completed_tests = 0 for testcase_dir in testcase_directories: assert isinstance(testcase_dir, PackageInfo) metadata = None if not testcase_dir.is_stdlib: metadata = read_metadata(testcase_dir.name) if not metadata.requires_python.contains(PYTHON_VERSION): - msg = f"skipping {testcase_dir.name!r} on Python {PYTHON_VERSION} (requires Python {metadata.requires_python})" + msg = ( + f"skipping {testcase_dir.name!r} (requires Python {metadata.requires_python}; " + f"test is being run using Python {PYTHON_VERSION})" + ) print(colored(msg, "yellow")) continue with tempfile.TemporaryDirectory() as td: @@ -272,12 +276,16 @@ def main() -> ReturnCode: msg = f"skipping {testcase_dir.name!r} for target Python {version} (requires Python {metadata.requires_python})" print(colored(msg, "yellow")) continue + completed_tests += 1 this_code = test_testcase_directory(testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) code = max(code, this_code) if code: print_error("\nTest completed with errors") + elif completed_tests: + plural = "" if completed_tests == 1 else "s" + print(colored(f"\n{completed_tests} test{plural} completed successfully!", "green")) else: - print(colored("\nTest completed successfully!", "green")) + print(colored("\nAll tests were skipped!", "yellow")) return code From 585fc7c0a8802f4fda9ff7d22522bc5e819c0f83 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 23 Sep 2023 16:32:51 +0200 Subject: [PATCH 10/15] type ignore mypy happiness --- tests/regr_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 015336ea54da..95442437a61b 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -256,9 +256,8 @@ def main() -> ReturnCode: code = 0 completed_tests = 0 for testcase_dir in testcase_directories: - assert isinstance(testcase_dir, PackageInfo) metadata = None - if not testcase_dir.is_stdlib: + if not testcase_dir.is_stdlib: # type: ignore[misc] metadata = read_metadata(testcase_dir.name) if not metadata.requires_python.contains(PYTHON_VERSION): msg = ( @@ -270,7 +269,7 @@ def main() -> ReturnCode: with tempfile.TemporaryDirectory() as td: tempdir = Path(td) for platform, version in product(platforms_to_test, versions_to_test): - if not testcase_dir.is_stdlib: + if not testcase_dir.is_stdlib: # type: ignore[misc] assert metadata is not None if not metadata.requires_python.contains(version): msg = f"skipping {testcase_dir.name!r} for target Python {version} (requires Python {metadata.requires_python})" From e61af2c172ecc1cefe6adaee3f4ca472527c772b Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 23 Sep 2023 16:35:45 +0200 Subject: [PATCH 11/15] Apply suggestions from code review Co-authored-by: Alex Waygood --- tests/regr_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 95442437a61b..7ca9be62ec19 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -257,7 +257,7 @@ def main() -> ReturnCode: completed_tests = 0 for testcase_dir in testcase_directories: metadata = None - if not testcase_dir.is_stdlib: # type: ignore[misc] + if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master metadata = read_metadata(testcase_dir.name) if not metadata.requires_python.contains(PYTHON_VERSION): msg = ( @@ -269,7 +269,7 @@ def main() -> ReturnCode: with tempfile.TemporaryDirectory() as td: tempdir = Path(td) for platform, version in product(platforms_to_test, versions_to_test): - if not testcase_dir.is_stdlib: # type: ignore[misc] + if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master assert metadata is not None if not metadata.requires_python.contains(version): msg = f"skipping {testcase_dir.name!r} for target Python {version} (requires Python {metadata.requires_python})" From 515fe800aa11aab1e015290a5c72ef1d1d9165cb Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 23 Sep 2023 19:28:12 +0200 Subject: [PATCH 12/15] CR --- tests/mypy_test.py | 7 +++--- tests/regr_test.py | 55 +++++++++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index ddb280da9fba..602b747fc27e 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -576,15 +576,14 @@ def main() -> None: print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---") sys.exit(code) if total_packages_skipped: - {1: ""}.get(total_packages_skipped, "s") plural = "" if total_packages_skipped == 1 else "s" print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow")) - elif not total_files_checked: - print_error("--- nothing to do; exit 1 ---") - sys.exit(1) if total_files_checked: plural = "" if total_files_checked == 1 else "s" print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green")) + else: + print_error("--- nothing to do; exit 1 ---") + sys.exit(1) if __name__ == "__main__": diff --git a/tests/regr_test.py b/tests/regr_test.py index 3fb7a8656d3a..81c8a5f38e8c 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -13,9 +13,11 @@ import sys import tempfile import threading +from collections.abc import Callable from contextlib import ExitStack, suppress from dataclasses import dataclass from enum import IntEnum +from functools import partial from pathlib import Path from typing_extensions import TypeAlias @@ -274,6 +276,29 @@ def concurrently_run_testcases( packageinfo_to_tempdir = { package_info: Path(stack.enter_context(tempfile.TemporaryDirectory())) for package_info in testcase_directories } + to_do: list[Callable[[], Result]] = [] + for testcase_dir, tempdir in packageinfo_to_tempdir.items(): + pkg = testcase_dir.name + requires_python = None + if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master + requires_python = read_metadata(pkg).requires_python + if not requires_python.contains(PYTHON_VERSION): + msg = f"skipping {pkg!r} (requires Python {requires_python}; test is being run using Python {PYTHON_VERSION})" + print(colored(msg, "yellow")) + continue + for version in versions_to_test: + if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master + assert requires_python is not None + if not requires_python.contains(version): + msg = f"skipping {pkg!r} for target Python {version} (requires Python {requires_python})" + print(colored(msg, "yellow")) + continue + to_do.extend( + [ + partial(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) + for platform in platforms_to_test + ] + ) event = threading.Event() printer_thread = threading.Thread(target=print_queued_messages, args=(event,)) @@ -289,31 +314,7 @@ def concurrently_run_testcases( ] concurrent.futures.wait(testcase_futures) - mypy_futures: list[concurrent.futures.Future[Result]] = [] - for testcase_dir, tempdir in packageinfo_to_tempdir.items(): - metadata = None - if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master - metadata = read_metadata(testcase_dir.name) - if not metadata.requires_python.contains(PYTHON_VERSION): - msg = ( - f"skipping {testcase_dir.name!r} (requires Python {metadata.requires_python}; " - f"test is being run using Python {PYTHON_VERSION})" - ) - _PRINT_QUEUE.put(colored(msg, "yellow")) - continue - for version in versions_to_test: - if not testcase_dir.is_stdlib: # type: ignore[misc] # mypy bug, already fixed on master - assert metadata is not None - if not metadata.requires_python.contains(version): - msg = f"skipping {testcase_dir.name!r} for target Python {version} (requires Python {metadata.requires_python})" - _PRINT_QUEUE.put(colored(msg, "yellow")) - continue - for platform in platforms_to_test: - mypy_futures.append( - executor.submit( - test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir - ) - ) + mypy_futures = [executor.submit(task) for task in to_do] results = [future.result() for future in mypy_futures] event.set() @@ -345,8 +346,8 @@ def main() -> ReturnCode: assert results is not None if not results: - print(colored("All tests were skipped!", "yellow")) - return 0 + print_error("All tests were skipped!") + return 1 print() From 9ae52399ac9bd6142bd1c345581ba80ebbd2f350 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 23 Sep 2023 19:42:07 +0200 Subject: [PATCH 13/15] Apply suggestions from code review Co-authored-by: Alex Waygood --- tests/regr_test.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 81c8a5f38e8c..82004780ad00 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -294,10 +294,8 @@ def concurrently_run_testcases( print(colored(msg, "yellow")) continue to_do.extend( - [ - partial(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) - for platform in platforms_to_test - ] + partial(test_testcase_directory, testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir) + for platform in platforms_to_test ) event = threading.Event() @@ -337,8 +335,6 @@ def main() -> ReturnCode: platforms_to_test = args.platforms_to_test or [sys.platform] versions_to_test = args.versions_to_test or [PYTHON_VERSION] - code = 0 - results: list[Result] | None = None with ExitStack() as stack: From 4a14f429c4d41dd024b6055636c256989a4195cd Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 23 Sep 2023 20:10:10 +0200 Subject: [PATCH 14/15] Simplify --- tests/mypy_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 602b747fc27e..0ff0cd66c1dc 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -395,6 +395,9 @@ def install_requirements_for_venv(venv_info: VenvInfo, args: TestConfig, externa def setup_virtual_environments(distributions: dict[str, PackageDependencies], args: TestConfig, tempdir: Path) -> None: """Logic necessary for testing stubs with non-types dependencies in isolated environments.""" + if not distributions: + return # hooray! Nothing to do + # STAGE 1: Determine which (if any) stubs packages require virtual environments. # Group stubs packages according to their external-requirements sets @@ -505,20 +508,17 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe ): distributions_to_check[distribution] = get_recursive_requirements(distribution) - # If it's the first time test_third_party_stubs() has been called during this session, - # setup the necessary virtual environments for testing the third-party stubs. - # It should only be necessary to call setup_virtual_environments() once per session. - if not _DISTRIBUTION_TO_VENV_MAPPING: - setup_virtual_environments(distributions_to_check, args, tempdir) - - # Some distributions may have been skipped earlier and therefore don't have a venv. + # Setup the necessary virtual environments for testing the third-party stubs. + # Note that some stubs may not be tested on all Python versions + # (due to version incompatibilities), + # so we can't guarantee that setup_virtual_environments() + # will only be called once per session. distributions_without_venv = { distribution: requirements for distribution, requirements in distributions_to_check.items() if distribution not in _DISTRIBUTION_TO_VENV_MAPPING } - if distributions_without_venv: - setup_virtual_environments(distributions_without_venv, args, tempdir) + setup_virtual_environments(distributions_without_venv, args, tempdir) # Check that there is a venv for every distribution we're testing. # Some venvs may exist from previous runs but are skipped in this run. From b4b6e5c60df05ba3d76b2fce4041bb3c69e87038 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 24 Sep 2023 18:19:19 +0100 Subject: [PATCH 15/15] Update tests/regr_test.py --- tests/regr_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/regr_test.py b/tests/regr_test.py index 82004780ad00..aaface2fc0f5 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -298,6 +298,9 @@ def concurrently_run_testcases( for platform in platforms_to_test ) + if not to_do: + return [] + event = threading.Event() printer_thread = threading.Thread(target=print_queued_messages, args=(event,)) printer_thread.start()