meta tests: refactor run_pytest

Author: ikonst

mypy/test/meta/_pytest.py

@@ -0,0 +1,85 @@
+import shlex
+import subprocess
+import sys
+import textwrap
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterable
+
+from mypy.test.config import test_data_prefix
+
+
+@dataclass
+class PytestResult:
+    source: str
+    source_updated: str
+    stdout: str
+    stderr: str
+
+
+def strip_source(s: str) -> str:
+    return textwrap.dedent(s).lstrip()
+
+
+def run_pytest(
+    data_suite: str,
+    *,
+    data_file_prefix: str,
+    node_prefix: str,
+    extra_args: Iterable[str],
+    max_attempts: int,
+) -> PytestResult:
+    """
+    Runs a suite of data test cases through pytest until either tests pass
+    or until a maximum number of attempts (needed for incremental tests).
+    """
+    p_test_data = Path(test_data_prefix)
+    p_root = p_test_data.parent.parent
+    p = p_test_data / f"{data_file_prefix}-meta-{uuid.uuid4()}.test"
+    assert not p.exists()
+    data_suite = strip_source(data_suite)
+    try:
+        p.write_text(data_suite)
+
+        test_nodeid = f"{node_prefix}::{p.name}"
+        extra_args = [sys.executable, "-m", "pytest", "-n", "0", "-s", *extra_args, test_nodeid]
+        if sys.version_info >= (3, 8):
+            cmd = shlex.join(extra_args)
+        else:
+            cmd = " ".join(extra_args)
+        for i in range(max_attempts - 1, -1, -1):
+            print(f">> {cmd}")
+            proc = subprocess.run(extra_args, capture_output=True, check=False, cwd=p_root)
+            if proc.returncode == 0:
+                break
+            prefix = "NESTED PYTEST STDOUT"
+            for line in proc.stdout.decode().splitlines():
+                print(f"{prefix}: {line}")
+                prefix = " " * len(prefix)
+            prefix = "NESTED PYTEST STDERR"
+            for line in proc.stderr.decode().splitlines():
+                print(f"{prefix}: {line}")
+                prefix = " " * len(prefix)
+            print(f"Exit code {proc.returncode} ({i} attempts remaining)")
+
+        return PytestResult(
+            source=data_suite,
+            source_updated=p.read_text(),
+            stdout=proc.stdout.decode(),
+            stderr=proc.stderr.decode(),
+        )
+    finally:
+        p.unlink()
+
+
+def run_type_check_suite(
+    data_suite: str, *, extra_args: Iterable[str], max_attempts: int
+) -> PytestResult:
+    run_pytest(
+        data_suite,
+        data_file_prefix="check",
+        node_prefix="mypy/test/testcheck.py::TypeCheckSuite",
+        extra_args=extra_args,
+        max_attempts=max_attempts,
+    )

mypy/test/meta/test_parse_data.py

@@ -2,37 +2,17 @@
 A "meta test" which tests the parsing of .test files. This is not meant to become exhaustive
 but to ensure we maintain a basic level of ergonomics for mypy contributors.
 """
-import subprocess
-import sys
-import textwrap
-import uuid
-from pathlib import Path
-
-from mypy.test.config import test_data_prefix
 from mypy.test.helpers import Suite
+from mypy.test.meta._pytest import PytestResult, run_type_check_suite
 
 
 class ParseTestDataSuite(Suite):
-    def _dedent(self, s: str) -> str:
-        return textwrap.dedent(s).lstrip()
-
-    def _run_pytest(self, data_suite: str) -> str:
-        p_test_data = Path(test_data_prefix)
-        p_root = p_test_data.parent.parent
-        p = p_test_data / f"check-meta-{uuid.uuid4()}.test"
-        assert not p.exists()
-        try:
-            p.write_text(data_suite)
-            test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}"
-            args = [sys.executable, "-m", "pytest", "-n", "0", "-s", test_nodeid]
-            proc = subprocess.run(args, cwd=p_root, capture_output=True, check=False)
-            return proc.stdout.decode()
-        finally:
-            p.unlink()
+    def _run_pytest(self, data_suite: str) -> PytestResult:
+        return run_type_check_suite(data_suite, extra_args=[], max_attempts=1)
 
     def test_parse_invalid_case(self) -> None:
-        # Arrange
-        data = self._dedent(
+        # Act
+        result = self._run_pytest(
             """
             [case abc]
             s: str
@@ -41,15 +21,12 @@ def test_parse_invalid_case(self) -> None:
             """
         )
 
-        # Act
-        actual = self._run_pytest(data)
-
         # Assert
-        assert "Invalid testcase id 'foo-XFAIL'" in actual
+        assert "Invalid testcase id 'foo-XFAIL'" in result.stdout
 
     def test_parse_invalid_section(self) -> None:
-        # Arrange
-        data = self._dedent(
+        # Act
+        result = self._run_pytest(
             """
             [case abc]
             s: str
@@ -58,12 +35,9 @@ def test_parse_invalid_section(self) -> None:
             """
         )
 
-        # Act
-        actual = self._run_pytest(data)
-
         # Assert
-        expected_lineno = data.splitlines().index("[unknownsection]") + 1
+        expected_lineno = result.source.splitlines().index("[unknownsection]") + 1
         expected = (
             f".test:{expected_lineno}: Invalid section header [unknownsection] in case 'abc'"
         )
-        assert expected in actual
+        assert expected in result.stdout

mypy/test/meta/test_update_data.py

@@ -3,50 +3,22 @@
 Updating the expected output, especially when it's in the form of inline (comment) assertions,
 can be brittle, which is why we're "meta-testing" here.
 """
-import shlex
-import subprocess
-import sys
-import textwrap
-import uuid
-from pathlib import Path
-
-from mypy.test.config import test_data_prefix
 from mypy.test.helpers import Suite
+from mypy.test.meta._pytest import PytestResult, run_type_check_suite, strip_source
 
 
 class UpdateDataSuite(Suite):
-    def _run_pytest_update_data(self, data_suite: str, *, max_attempts: int) -> str:
+    def _run_pytest_update_data(self, data_suite: str) -> PytestResult:
         """
         Runs a suite of data test cases through 'pytest --update-data' until either tests pass
         or until a maximum number of attempts (needed for incremental tests).
         """
-        p_test_data = Path(test_data_prefix)
-        p_root = p_test_data.parent.parent
-        p = p_test_data / f"check-meta-{uuid.uuid4()}.test"
-        assert not p.exists()
-        try:
-            p.write_text(textwrap.dedent(data_suite).lstrip())
-
-            test_nodeid = f"mypy/test/testcheck.py::TypeCheckSuite::{p.name}"
-            args = [sys.executable, "-m", "pytest", "-n", "0", "-s", "--update-data", test_nodeid]
-            if sys.version_info >= (3, 8):
-                cmd = shlex.join(args)
-            else:
-                cmd = " ".join(args)
-            for i in range(max_attempts - 1, -1, -1):
-                res = subprocess.run(args, cwd=p_root)
-                if res.returncode == 0:
-                    break
-                print(f"`{cmd}` returned {res.returncode}: {i} attempts remaining")
-
-            return p.read_text()
-        finally:
-            p.unlink()
+        return run_type_check_suite(data_suite, extra_args=["--update-data"], max_attempts=3)
 
     def test_update_data(self) -> None:
         # Note: We test multiple testcases rather than 'test case per test case'
         #       so we could also exercise rewriting multiple testcases at once.
-        actual = self._run_pytest_update_data(
+        result = self._run_pytest_update_data(
             """
             [case testCorrect]
             s: str = 42  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
@@ -100,12 +72,12 @@ def test_update_data(self) -> None:
             [file b.py]
             s2: str = 43  # E: baz
             [builtins fixtures/list.pyi]
-            """,
-            max_attempts=3,
+            """
         )
 
         # Assert
-        expected = """
+        expected = strip_source(
+            """
         [case testCorrect]
         s: str = 42  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
 
@@ -157,4 +129,5 @@ def test_update_data(self) -> None:
         s2: str = 43  # E: Incompatible types in assignment (expression has type "int", variable has type "str")
         [builtins fixtures/list.pyi]
         """
-        assert actual == textwrap.dedent(expected).lstrip()
+        )
+        assert result.source_updated == expected