From 3c5353ad47ddfdcad10517bdc4d2cf3a39849c99 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 30 Jan 2024 11:57:46 +0100 Subject: [PATCH 1/3] Limit regression test workers to 1 This should address #11220. --- tests/regr_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/regr_test.py b/tests/regr_test.py index 3d50aec406ea..ccd432f1d4ca 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -308,7 +308,11 @@ def concurrently_run_testcases( printer_thread = threading.Thread(target=print_queued_messages, args=(event,)) printer_thread.start() - with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor: + # Limit the number of workers to 1 for now. There are intermittent mypy + # crashes when running with multiple workers. See + # https://github.com/python/typeshed/issues/11220 for details. + max_workers = 1 # os.cpu_count() or 2 + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: # Each temporary directory may be used by multiple processes concurrently during the next step; # must make sure that they're all setup correctly before starting the next step, # in order to avoid race conditions From f084d5f0128abcd2e5ee93673ea06c314e756378 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 30 Jan 2024 12:20:34 +0100 Subject: [PATCH 2/3] Add --all-python-versions to regr_test --- tests/regr_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/regr_test.py b/tests/regr_test.py index ccd432f1d4ca..b931b5e0b998 100755 --- a/tests/regr_test.py +++ b/tests/regr_test.py @@ -82,6 +82,14 @@ class Verbosity(IntEnum): "Note that this cannot be specified if --platform and/or --python-version are specified." ), ) +parser.add_argument( + "--all-python-versions", + action="store_true", + help=( + 'Run tests on all available Python versions (defaults to "False"). ' + "Note that this cannot be specified if --all and/or --python-version are specified." + ), +) parser.add_argument( "--verbosity", choices=[member.name for member in Verbosity], @@ -338,9 +346,16 @@ def main() -> ReturnCode: if args.all: if args.platforms_to_test: parser.error("Cannot specify both --platform and --all") + if args.all_python_versions: + parser.error("Cannot specify both --all-python-versions and --all") if args.versions_to_test: parser.error("Cannot specify both --python-version and --all") platforms_to_test, versions_to_test = SUPPORTED_PLATFORMS, SUPPORTED_VERSIONS + elif args.all_python_versions: + if args.versions_to_test: + parser.error("Cannot specify both --python-version and --all-python-versions") + platforms_to_test = args.platforms_to_test or [sys.platform] + versions_to_test = SUPPORTED_VERSIONS else: platforms_to_test = args.platforms_to_test or [sys.platform] versions_to_test = args.versions_to_test or [PYTHON_VERSION] From 5992e20b56d9201f586291cf42caeb16548c0db8 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Tue, 30 Jan 2024 12:22:08 +0100 Subject: [PATCH 3/3] Shard regression tests by platform --- .github/workflows/tests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b0a7a6f961e2..3e37e8dba635 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -109,15 +109,19 @@ jobs: regression-tests: name: Run mypy on the test cases runs-on: ubuntu-latest + strategy: + matrix: + platform: ["linux", "darwin", "win32"] + fail-fast: false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" cache: pip cache-dependency-path: requirements-tests.txt - run: pip install -r requirements-tests.txt - - run: python ./tests/regr_test.py --all --verbosity QUIET + - run: python ./tests/regr_test.py --platform=${{ matrix.platform }} --all-python-versions --verbosity QUIET pyright: name: Test typeshed with pyright