From 736d5f26f9fcc86a15cc3a42a840b496eeafcb39 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sat, 12 Nov 2022 12:27:52 -0500 Subject: [PATCH 1/7] Allow running non-listed platforms locally --- tests/stubtest_third_party.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 49892435b795..ccb8416f225a 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -34,10 +34,24 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: print(colored("skipping", "yellow")) return True - platforms_to_test = stubtest_meta.get("platforms", ["linux"]) - if sys.platform not in platforms_to_test: - print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow")) - return True + platforms_to_test: list[str] | None = stubtest_meta.get("platforms") + if os.environ.get("CI"): + if platforms_to_test: + if sys.platform not in platforms_to_test: + print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow")) + return True + elif sys.platform != "linux": + print(colored(f"skipping, redundant platform: {sys.platform}", "yellow")) + return True + + elif platforms_to_test and sys.platform not in platforms_to_test: + print( + colored( + rf"warning: your platform {sys.platform}, is not in: {platforms_to_test}. " + + "It may be unsupported or deemed redundant to test on the CI", + "yellow", + ) + ) with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) From 7db8d55ba313cf07988d3261cd60e55f20dffb46 Mon Sep 17 00:00:00 2001 From: Avasam Date: Sat, 12 Nov 2022 12:46:17 -0500 Subject: [PATCH 2/7] redundant raw string --- tests/stubtest_third_party.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index ccb8416f225a..ec33089eaacc 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -47,7 +47,7 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: elif platforms_to_test and sys.platform not in platforms_to_test: print( colored( - rf"warning: your platform {sys.platform}, is not in: {platforms_to_test}. " + f"warning: your platform, {sys.platform}, is not in: {platforms_to_test}. " + "It may be unsupported or deemed redundant to test on the CI", "yellow", ) From 6b9a9f0e0d5364c548a83931d6a5b192a4904a1f Mon Sep 17 00:00:00 2001 From: Avasam Date: Sat, 12 Nov 2022 14:09:43 -0500 Subject: [PATCH 3/7] Add --ci-run flag --- .github/workflows/daily.yml | 4 ++-- .github/workflows/stubtest_third_party.yml | 6 +++--- tests/stubtest_third_party.py | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 04b97e54b0d8..715c7f6132fb 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -64,9 +64,9 @@ jobs: sudo apt update sudo apt install -y $(python tests/get_packages.py) - xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} + xvfb-run python tests/stubtest_third_party.py --ci-run --num-shards 4 --shard-index ${{ matrix.shard-index }} else - python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }} + python tests/stubtest_third_party.py --ci-run --num-shards 4 --shard-index ${{ matrix.shard-index }} fi stub-uploader: diff --git a/.github/workflows/stubtest_third_party.yml b/.github/workflows/stubtest_third_party.yml index 08ae44ab26ac..2dc98e194496 100644 --- a/.github/workflows/stubtest_third_party.yml +++ b/.github/workflows/stubtest_third_party.yml @@ -58,17 +58,17 @@ jobs: echo "Installing apt packages: $PACKAGES" sudo apt update && sudo apt install -y $PACKAGES fi - xvfb-run python tests/stubtest_third_party.py $STUBS + xvfb-run python tests/stubtest_third_party.py --ci-run $STUBS fi if [ "${{ matrix.os }}" = "macos-latest" ]; then # Could install brew packages here if we run into stubs that need it - python tests/stubtest_third_party.py $STUBS + python tests/stubtest_third_party.py --ci-run $STUBS fi if [ "${{ matrix.os }}" = "windows-latest" ]; then # Could install choco packages here if we run into stubs that need it - python tests/stubtest_third_party.py $STUBS + python tests/stubtest_third_party.py --ci-run $STUBS fi else echo "Nothing to test" diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index ec33089eaacc..efb6e73ce289 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -23,7 +23,7 @@ def get_mypy_req() -> str: return next(line.strip() for line in f if "mypy" in line) -def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: +def run_stubtest(dist: Path, *, verbose: bool = False, ci_run: bool = False) -> bool: with open(dist / "METADATA.toml", encoding="UTF-8") as f: metadata = dict(tomli.loads(f.read())) @@ -35,7 +35,7 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool: return True platforms_to_test: list[str] | None = stubtest_meta.get("platforms") - if os.environ.get("CI"): + if ci_run: if platforms_to_test: if sys.platform not in platforms_to_test: print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow")) @@ -181,6 +181,7 @@ def main() -> NoReturn: parser.add_argument("-v", "--verbose", action="store_true", help="verbose output") parser.add_argument("--num-shards", type=int, default=1) parser.add_argument("--shard-index", type=int, default=0) + parser.add_argument("--ci-run", action="store_true") parser.add_argument("dists", metavar="DISTRIBUTION", type=str, nargs=argparse.ZERO_OR_MORE) args = parser.parse_args() @@ -194,7 +195,7 @@ def main() -> NoReturn: for i, dist in enumerate(dists): if i % args.num_shards != args.shard_index: continue - if not run_stubtest(dist, verbose=args.verbose): + if not run_stubtest(dist, verbose=args.verbose, ci_run=args.ci_run): result = 1 sys.exit(result) From b5f21c9ee314ddc12bab697886896380e8d9076e Mon Sep 17 00:00:00 2001 From: Avasam Date: Mon, 14 Nov 2022 19:35:57 -0500 Subject: [PATCH 4/7] Update tests/stubtest_third_party.py Co-authored-by: Alex Waygood --- tests/stubtest_third_party.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index efb6e73ce289..e20532c4b5e8 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -34,21 +34,14 @@ def run_stubtest(dist: Path, *, verbose: bool = False, ci_run: bool = False) -> print(colored("skipping", "yellow")) return True - platforms_to_test: list[str] | None = stubtest_meta.get("platforms") - if ci_run: - if platforms_to_test: - if sys.platform not in platforms_to_test: - print(colored(f"skipping, unsupported platform: {sys.platform}, supported: {platforms_to_test}", "yellow")) - return True - elif sys.platform != "linux": - print(colored(f"skipping, redundant platform: {sys.platform}", "yellow")) + platforms_to_test = stubtest_meta.get("platforms", ["linux"]) + if sys.platform not in platforms_to_test: + if ci_run: + print(colored(f"skipping (unsupported platform)", "yellow")) return True - - elif platforms_to_test and sys.platform not in platforms_to_test: print( colored( - f"warning: your platform, {sys.platform}, is not in: {platforms_to_test}. " - + "It may be unsupported or deemed redundant to test on the CI", + f"Note: stubtest is not currently tested on {sys.platform} in typeshed's CI.", "yellow", ) ) From 4166bae0e33336ff5a754a7aa3895084c5a4957f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 15 Nov 2022 00:37:02 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks --- tests/stubtest_third_party.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index e20532c4b5e8..780b576b428e 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -39,12 +39,7 @@ def run_stubtest(dist: Path, *, verbose: bool = False, ci_run: bool = False) -> if ci_run: print(colored(f"skipping (unsupported platform)", "yellow")) return True - print( - colored( - f"Note: stubtest is not currently tested on {sys.platform} in typeshed's CI.", - "yellow", - ) - ) + print(colored(f"Note: stubtest is not currently tested on {sys.platform} in typeshed's CI.", "yellow")) with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) From 8c940df48fd9a81487aa6bdd094ce78139596e21 Mon Sep 17 00:00:00 2001 From: Avasam Date: Mon, 14 Nov 2022 20:14:49 -0500 Subject: [PATCH 6/7] PR comments --- .github/workflows/daily.yml | 4 ++-- .github/workflows/stubtest_third_party.yml | 6 +++--- tests/stubtest_third_party.py | 16 ++++++++++------ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 715c7f6132fb..5eed93f02b7b 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -64,9 +64,9 @@ jobs: sudo apt update sudo apt install -y $(python tests/get_packages.py) - xvfb-run python tests/stubtest_third_party.py --ci-run --num-shards 4 --shard-index ${{ matrix.shard-index }} + xvfb-run python tests/stubtest_third_party.py --specified-stubs-only --num-shards 4 --shard-index ${{ matrix.shard-index }} else - python tests/stubtest_third_party.py --ci-run --num-shards 4 --shard-index ${{ matrix.shard-index }} + python tests/stubtest_third_party.py --specified-stubs-only --num-shards 4 --shard-index ${{ matrix.shard-index }} fi stub-uploader: diff --git a/.github/workflows/stubtest_third_party.yml b/.github/workflows/stubtest_third_party.yml index 2dc98e194496..5f82c3ceefed 100644 --- a/.github/workflows/stubtest_third_party.yml +++ b/.github/workflows/stubtest_third_party.yml @@ -58,17 +58,17 @@ jobs: echo "Installing apt packages: $PACKAGES" sudo apt update && sudo apt install -y $PACKAGES fi - xvfb-run python tests/stubtest_third_party.py --ci-run $STUBS + xvfb-run python tests/stubtest_third_party.py --specified-stubs-only $STUBS fi if [ "${{ matrix.os }}" = "macos-latest" ]; then # Could install brew packages here if we run into stubs that need it - python tests/stubtest_third_party.py --ci-run $STUBS + python tests/stubtest_third_party.py --specified-stubs-only $STUBS fi if [ "${{ matrix.os }}" = "windows-latest" ]; then # Could install choco packages here if we run into stubs that need it - python tests/stubtest_third_party.py --ci-run $STUBS + python tests/stubtest_third_party.py --specified-stubs-only $STUBS fi else echo "Nothing to test" diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 780b576b428e..7d82ad02f324 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -23,7 +23,7 @@ def get_mypy_req() -> str: return next(line.strip() for line in f if "mypy" in line) -def run_stubtest(dist: Path, *, verbose: bool = False, ci_run: bool = False) -> bool: +def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: bool = False) -> bool: with open(dist / "METADATA.toml", encoding="UTF-8") as f: metadata = dict(tomli.loads(f.read())) @@ -36,10 +36,10 @@ def run_stubtest(dist: Path, *, verbose: bool = False, ci_run: bool = False) -> platforms_to_test = stubtest_meta.get("platforms", ["linux"]) if sys.platform not in platforms_to_test: - if ci_run: - print(colored(f"skipping (unsupported platform)", "yellow")) + if specified_stubs_only: + print(colored("skipping (unspecified platform)", "yellow")) return True - print(colored(f"Note: stubtest is not currently tested on {sys.platform} in typeshed's CI.", "yellow")) + print(colored(f"Note: {dist.name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow")) with tempfile.TemporaryDirectory() as tmp: venv_dir = Path(tmp) @@ -169,7 +169,11 @@ def main() -> NoReturn: parser.add_argument("-v", "--verbose", action="store_true", help="verbose output") parser.add_argument("--num-shards", type=int, default=1) parser.add_argument("--shard-index", type=int, default=0) - parser.add_argument("--ci-run", action="store_true") + parser.add_argument( + "--specified-stubs-only", + action="store_true", + help="skip the test if the current platform is not specified in METADATA.toml/tool.stubtest.platforms", + ) parser.add_argument("dists", metavar="DISTRIBUTION", type=str, nargs=argparse.ZERO_OR_MORE) args = parser.parse_args() @@ -183,7 +187,7 @@ def main() -> NoReturn: for i, dist in enumerate(dists): if i % args.num_shards != args.shard_index: continue - if not run_stubtest(dist, verbose=args.verbose, ci_run=args.ci_run): + if not run_stubtest(dist, verbose=args.verbose, specified_stubs_only=args.specified_stubs_only): result = 1 sys.exit(result) From c0b60cf415944b01b8be541a1e9bb6172f5a1048 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 15 Nov 2022 01:24:41 +0000 Subject: [PATCH 7/7] Minor wordsmithing --- tests/stubtest_third_party.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 7d82ad02f324..61ba07e71777 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -37,7 +37,7 @@ def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: boo platforms_to_test = stubtest_meta.get("platforms", ["linux"]) if sys.platform not in platforms_to_test: if specified_stubs_only: - print(colored("skipping (unspecified platform)", "yellow")) + print(colored("skipping (platform not specified in METADATA.toml)", "yellow")) return True print(colored(f"Note: {dist.name} is not currently tested on {sys.platform} in typeshed's CI.", "yellow"))