From b481b337630d1d7eb92c6dbab76df4ef44483fcb Mon Sep 17 00:00:00 2001 From: Ilya Priven Date: Tue, 20 Jun 2023 08:58:12 -0400 Subject: [PATCH 1/5] meta tests: refactor run_pytest --- mypy/test/meta/_pytest.py | 85 ++++++++++++++++++++++++++++++ mypy/test/meta/test_parse_data.py | 46 ++++------------ mypy/test/meta/test_update_data.py | 45 ++++------------ 3 files changed, 104 insertions(+), 72 deletions(-) create mode 100644 mypy/test/meta/_pytest.py diff --git a/mypy/test/meta/_pytest.py b/mypy/test/meta/_pytest.py new file mode 100644 index 000000000000..b17f03235114 --- /dev/null +++ b/mypy/test/meta/_pytest.py @@ -0,0 +1,85 @@ +import shlex +import subprocess +import sys +import textwrap +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +from mypy.test.config import test_data_prefix + + +@dataclass +class PytestResult: + source: str + source_updated: str + stdout: str + stderr: str + + +def strip_source(s: str) -> str: + return textwrap.dedent(s).lstrip() + + +def run_pytest( + data_suite: str, + *, + data_file_prefix: str, + node_prefix: str, + extra_args: Iterable[str], + max_attempts: int, +) -> PytestResult: + """ + Runs a suite of data test cases through pytest until either tests pass + or until a maximum number of attempts (needed for incremental tests). + """ + p_test_data = Path(test_data_prefix) + p_root = p_test_data.parent.parent + p = p_test_data / f"{data_file_prefix}-meta-{uuid.uuid4()}.test" + assert not p.exists() + data_suite = strip_source(data_suite) + try: + p.write_text(data_suite) + + test_nodeid = f"{node_prefix}::{p.name}" + extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid] + if sys.version_info >= (3, 8): + cmd = shlex.join(extra_args) + else: + cmd = " ".join(extra_args) + for i in range(max_attempts - 1, -1, -1): + print(f">> {cmd}") + proc = subprocess.run(extra_args, capture_output=True, check=False, cwd=p_root) + if proc.returncode == 0: + break + prefix = "NESTED PYTEST STDOUT" + for line in proc.stdout.decode().splitlines(): + print(f"{prefix}: {line}") + prefix = " " * len(prefix) + prefix = "NESTED PYTEST STDERR" + for line in proc.stderr.decode().splitlines(): + print(f"{prefix}: {line}") + prefix = " " * len(prefix) + print(f"Exit code {proc.returncode} ({i} attempts remaining)") + + return PytestResult( + source=data_suite, + source_updated=p.read_text(), + stdout=proc.stdout.decode(), + stderr=proc.stderr.decode(), + ) + finally: + p.unlink() + + +def run_type_check_suite( + data_suite: str, *, extra_args: Iterable[str], max_attempts: int +) -> PytestResult: + return run_pytest( + data_suite, + data_file_prefix="check", + node_prefix="mypy/test/testcheck.py::TypeCheckSuite", + extra_args=extra_args, + max_attempts=max_attempts, + ) diff --git a/mypy/test/meta/test_parse_data.py b/mypy/test/meta/test_parse_data.py index cc1b4ff6eeed..54d4b0db2b09 100644 --- a/mypy/test/meta/test_parse_data.py +++ b/mypy/test/meta/test_parse_data.py @@ -2,37 +2,17 @@ A "meta test" which tests the parsing of .test files. This is not meant to become exhaustive but to ensure we maintain a basic level of ergonomics for mypy contributors. """ -import subprocess -import sys -import textwrap -import uuid -from pathlib import Path - -from mypy.test.config import test_data_prefix from mypy.test.helpers import Suite +from mypy.test.meta._pytest import PytestResult, run_type_check_suite class ParseTestDataSuite(Suite): - def _dedent(self, s: str) -> str: - return textwrap.dedent(s).lstrip() - - def _run_pytest(self, data_suite: str) -> str: - p_test_data = Path(test_data_prefix) - p_root = p_test_data.parent.parent - p = p_test_data / f"check-meta-{uuid.uuid4()}.test" - assert not p.exists() - try: - p.write_text(data_suite) - test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}" - args = [sys.executable, "-m", "pytest", "-n", "0", "-s", test_nodeid] - proc = subprocess.run(args, cwd=p_root, capture_output=True, check=False) - return proc.stdout.decode() - finally: - p.unlink() + def _run_pytest(self, data_suite: str) -> PytestResult: + return run_type_check_suite(data_suite, extra_args=[], max_attempts=1) def test_parse_invalid_case(self) -> None: - # Arrange - data = self._dedent( + # Act + result = self._run_pytest( """ [case abc] s: str @@ -41,15 +21,12 @@ def test_parse_invalid_case(self) -> None: """ ) - # Act - actual = self._run_pytest(data) - # Assert - assert "Invalid testcase id 'foo-XFAIL'" in actual + assert "Invalid testcase id 'foo-XFAIL'" in result.stdout def test_parse_invalid_section(self) -> None: - # Arrange - data = self._dedent( + # Act + result = self._run_pytest( """ [case abc] s: str @@ -58,12 +35,9 @@ def test_parse_invalid_section(self) -> None: """ ) - # Act - actual = self._run_pytest(data) - # Assert - expected_lineno = data.splitlines().index("[unknownsection]") + 1 + expected_lineno = result.source.splitlines().index("[unknownsection]") + 1 expected = ( f".test:{expected_lineno}: Invalid section header [unknownsection] in case 'abc'" ) - assert expected in actual + assert expected in result.stdout diff --git a/mypy/test/meta/test_update_data.py b/mypy/test/meta/test_update_data.py index 67f9a7f56ebd..3fb2e85a4daf 100644 --- a/mypy/test/meta/test_update_data.py +++ b/mypy/test/meta/test_update_data.py @@ -3,50 +3,22 @@ Updating the expected output, especially when it's in the form of inline (comment) assertions, can be brittle, which is why we're "meta-testing" here. """ -import shlex -import subprocess -import sys -import textwrap -import uuid -from pathlib import Path - -from mypy.test.config import test_data_prefix from mypy.test.helpers import Suite +from mypy.test.meta._pytest import PytestResult, run_type_check_suite, strip_source class UpdateDataSuite(Suite): - def _run_pytest_update_data(self, data_suite: str, *, max_attempts: int) -> str: + def _run_pytest_update_data(self, data_suite: str) -> PytestResult: """ Runs a suite of data test cases through 'pytest --update-data' until either tests pass or until a maximum number of attempts (needed for incremental tests). """ - p_test_data = Path(test_data_prefix) - p_root = p_test_data.parent.parent - p = p_test_data / f"check-meta-{uuid.uuid4()}.test" - assert not p.exists() - try: - p.write_text(textwrap.dedent(data_suite).lstrip()) - - test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}" - args = [sys.executable, "-m", "pytest", "-n", "0", "-s", "--update-data", test_nodeid] - if sys.version_info >= (3, 8): - cmd = shlex.join(args) - else: - cmd = " ".join(args) - for i in range(max_attempts - 1, -1, -1): - res = subprocess.run(args, cwd=p_root) - if res.returncode == 0: - break - print(f"`{cmd}` returned {res.returncode}: {i} attempts remaining") - - return p.read_text() - finally: - p.unlink() + return run_type_check_suite(data_suite, extra_args=["--update-data"], max_attempts=3) def test_update_data(self) -> None: # Note: We test multiple testcases rather than 'test case per test case' # so we could also exercise rewriting multiple testcases at once. - actual = self._run_pytest_update_data( + result = self._run_pytest_update_data( """ [case testCorrect] s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") @@ -100,12 +72,12 @@ def test_update_data(self) -> None: [file b.py] s2: str = 43 # E: baz [builtins fixtures/list.pyi] - """, - max_attempts=3, + """ ) # Assert - expected = """ + expected = strip_source( + """ [case testCorrect] s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") @@ -157,4 +129,5 @@ def test_update_data(self) -> None: s2: str = 43 # E: Incompatible types in assignment (expression has type "int", variable has type "str") [builtins fixtures/list.pyi] """ - assert actual == textwrap.dedent(expected).lstrip() + ) + assert result.source_updated == expected From 34f8f43189ee8dd93c4fbff1ce7858fec1eee164 Mon Sep 17 00:00:00 2001 From: Ilya Priven Date: Tue, 11 Jul 2023 21:56:55 -0700 Subject: [PATCH 2/5] cleaner --- mypy/test/meta/_pytest.py | 28 +++++++++------------------- mypy/test/meta/test_parse_data.py | 13 +++++++------ mypy/test/meta/test_update_data.py | 21 +++++++++++---------- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/mypy/test/meta/_pytest.py b/mypy/test/meta/_pytest.py index b17f03235114..d1ad985a6ffa 100644 --- a/mypy/test/meta/_pytest.py +++ b/mypy/test/meta/_pytest.py @@ -13,36 +13,38 @@ @dataclass class PytestResult: source: str - source_updated: str + source_updated: str # any updates made by --update-data stdout: str stderr: str -def strip_source(s: str) -> str: +def dedent_docstring(s: str) -> str: return textwrap.dedent(s).lstrip() -def run_pytest( +def run_pytest_data_suite( data_suite: str, *, - data_file_prefix: str, - node_prefix: str, + data_file_prefix: str = "check", + pytest_node_prefix: str = "mypy/test/testcheck.py::TypeCheckSuite", extra_args: Iterable[str], max_attempts: int, ) -> PytestResult: """ Runs a suite of data test cases through pytest until either tests pass or until a maximum number of attempts (needed for incremental tests). + + :param data_suite: the actual "suite" i.e. the contents of a .test file """ p_test_data = Path(test_data_prefix) p_root = p_test_data.parent.parent p = p_test_data / f"{data_file_prefix}-meta-{uuid.uuid4()}.test" assert not p.exists() - data_suite = strip_source(data_suite) + data_suite = dedent_docstring(data_suite) try: p.write_text(data_suite) - test_nodeid = f"{node_prefix}::{p.name}" + test_nodeid = f"{pytest_node_prefix}::{p.name}" extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid] if sys.version_info >= (3, 8): cmd = shlex.join(extra_args) @@ -71,15 +73,3 @@ def run_pytest( ) finally: p.unlink() - - -def run_type_check_suite( - data_suite: str, *, extra_args: Iterable[str], max_attempts: int -) -> PytestResult: - return run_pytest( - data_suite, - data_file_prefix="check", - node_prefix="mypy/test/testcheck.py::TypeCheckSuite", - extra_args=extra_args, - max_attempts=max_attempts, - ) diff --git a/mypy/test/meta/test_parse_data.py b/mypy/test/meta/test_parse_data.py index 54d4b0db2b09..927bcd4dcf26 100644 --- a/mypy/test/meta/test_parse_data.py +++ b/mypy/test/meta/test_parse_data.py @@ -3,16 +3,17 @@ but to ensure we maintain a basic level of ergonomics for mypy contributors. """ from mypy.test.helpers import Suite -from mypy.test.meta._pytest import PytestResult, run_type_check_suite +from mypy.test.meta._pytest import PytestResult, run_pytest_data_suite -class ParseTestDataSuite(Suite): - def _run_pytest(self, data_suite: str) -> PytestResult: - return run_type_check_suite(data_suite, extra_args=[], max_attempts=1) +def _run_pytest(data_suite: str) -> PytestResult: + return run_pytest_data_suite(data_suite, extra_args=[], max_attempts=1) + +class ParseTestDataSuite(Suite): def test_parse_invalid_case(self) -> None: # Act - result = self._run_pytest( + result = _run_pytest( """ [case abc] s: str @@ -26,7 +27,7 @@ def test_parse_invalid_case(self) -> None: def test_parse_invalid_section(self) -> None: # Act - result = self._run_pytest( + result = _run_pytest( """ [case abc] s: str diff --git a/mypy/test/meta/test_update_data.py b/mypy/test/meta/test_update_data.py index 3fb2e85a4daf..6abf7fe03fcf 100644 --- a/mypy/test/meta/test_update_data.py +++ b/mypy/test/meta/test_update_data.py @@ -4,21 +4,22 @@ can be brittle, which is why we're "meta-testing" here. """ from mypy.test.helpers import Suite -from mypy.test.meta._pytest import PytestResult, run_type_check_suite, strip_source +from mypy.test.meta._pytest import PytestResult, dedent_docstring, run_pytest_data_suite -class UpdateDataSuite(Suite): - def _run_pytest_update_data(self, data_suite: str) -> PytestResult: - """ - Runs a suite of data test cases through 'pytest --update-data' until either tests pass - or until a maximum number of attempts (needed for incremental tests). - """ - return run_type_check_suite(data_suite, extra_args=["--update-data"], max_attempts=3) +def _run_pytest_update_data(data_suite: str) -> PytestResult: + """ + Runs a suite of data test cases through 'pytest --update-data' until either tests pass + or until a maximum number of attempts (needed for incremental tests). + """ + return run_pytest_data_suite(data_suite, extra_args=["--update-data"], max_attempts=3) + +class UpdateDataSuite(Suite): def test_update_data(self) -> None: # Note: We test multiple testcases rather than 'test case per test case' # so we could also exercise rewriting multiple testcases at once. - result = self._run_pytest_update_data( + result = _run_pytest_update_data( """ [case testCorrect] s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") @@ -76,7 +77,7 @@ def test_update_data(self) -> None: ) # Assert - expected = strip_source( + expected = dedent_docstring( """ [case testCorrect] s: str = 42 # E: Incompatible types in assignment (expression has type "int", variable has type "str") From f00a623d50621470faa83030a66a1e8acc2f4af0 Mon Sep 17 00:00:00 2001 From: Ilya Priven Date: Sat, 12 Aug 2023 22:51:29 -0400 Subject: [PATCH 3/5] fix --- mypy/test/meta/test_parse_data.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/mypy/test/meta/test_parse_data.py b/mypy/test/meta/test_parse_data.py index 1be98d9f5e89..c78e56a4193f 100644 --- a/mypy/test/meta/test_parse_data.py +++ b/mypy/test/meta/test_parse_data.py @@ -44,8 +44,8 @@ def test_parse_invalid_section(self) -> None: assert expected in result.stdout def test_bad_ge_version_check(self) -> None: - # Arrange - data = self._dedent( + # Act + actual = _run_pytest( """ [case abc] s: str @@ -54,15 +54,12 @@ def test_bad_ge_version_check(self) -> None: """ ) - # Act - actual = self._run_pytest(data) - # Assert - assert "version>=3.8 always true since minimum runtime version is (3, 8)" in actual + assert "version>=3.8 always true since minimum runtime version is (3, 8)" in actual.stdout def test_bad_eq_version_check(self) -> None: - # Arrange - data = self._dedent( + # Act + actual = _run_pytest( """ [case abc] s: str @@ -71,8 +68,5 @@ def test_bad_eq_version_check(self) -> None: """ ) - # Act - actual = self._run_pytest(data) - # Assert - assert "version==3.7 always false since minimum runtime version is (3, 8)" in actual + assert "version==3.7 always false since minimum runtime version is (3, 8)" in actual.stdout From c54e24ebda4d1ecf90ffab0019cc8f1bd5453715 Mon Sep 17 00:00:00 2001 From: Ilya Priven Date: Sat, 12 Aug 2023 22:53:40 -0400 Subject: [PATCH 4/5] naming --- mypy/test/meta/_pytest.py | 8 ++++---- mypy/test/meta/test_parse_data.py | 2 +- mypy/test/meta/test_update_data.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mypy/test/meta/_pytest.py b/mypy/test/meta/_pytest.py index d1ad985a6ffa..362f108ed299 100644 --- a/mypy/test/meta/_pytest.py +++ b/mypy/test/meta/_pytest.py @@ -12,8 +12,8 @@ @dataclass class PytestResult: - source: str - source_updated: str # any updates made by --update-data + input: str + input_updated: str # any updates made by --update-data stdout: str stderr: str @@ -66,8 +66,8 @@ def run_pytest_data_suite( print(f"Exit code {proc.returncode} ({i} attempts remaining)") return PytestResult( - source=data_suite, - source_updated=p.read_text(), + input=data_suite, + input_updated=p.read_text(), stdout=proc.stdout.decode(), stderr=proc.stderr.decode(), ) diff --git a/mypy/test/meta/test_parse_data.py b/mypy/test/meta/test_parse_data.py index c78e56a4193f..797fdd7b2c8c 100644 --- a/mypy/test/meta/test_parse_data.py +++ b/mypy/test/meta/test_parse_data.py @@ -37,7 +37,7 @@ def test_parse_invalid_section(self) -> None: ) # Assert - expected_lineno = result.source.splitlines().index("[unknownsection]") + 1 + expected_lineno = result.input.splitlines().index("[unknownsection]") + 1 expected = ( f".test:{expected_lineno}: Invalid section header [unknownsection] in case 'abc'" ) diff --git a/mypy/test/meta/test_update_data.py b/mypy/test/meta/test_update_data.py index 6abf7fe03fcf..40b70157a0e3 100644 --- a/mypy/test/meta/test_update_data.py +++ b/mypy/test/meta/test_update_data.py @@ -131,4 +131,4 @@ def test_update_data(self) -> None: [builtins fixtures/list.pyi] """ ) - assert result.source_updated == expected + assert result.input_updated == expected From b0083b03d1dd12a90e6dc8cae6558f450a3b3527 Mon Sep 17 00:00:00 2001 From: Ilya Priven Date: Sat, 2 Sep 2023 09:10:10 -0400 Subject: [PATCH 5/5] remove pre-py38 code --- mypy/test/meta/_pytest.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mypy/test/meta/_pytest.py b/mypy/test/meta/_pytest.py index 362f108ed299..b8648f033143 100644 --- a/mypy/test/meta/_pytest.py +++ b/mypy/test/meta/_pytest.py @@ -46,10 +46,7 @@ def run_pytest_data_suite( test_nodeid = f"{pytest_node_prefix}::{p.name}" extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid] - if sys.version_info >= (3, 8): - cmd = shlex.join(extra_args) - else: - cmd = " ".join(extra_args) + cmd = shlex.join(extra_args) for i in range(max_attempts - 1, -1, -1): print(f">> {cmd}") proc = subprocess.run(extra_args, capture_output=True, check=False, cwd=p_root)