From 2023fa7de83b22997b55ede52f9e81b3f41f9ae0 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Fri, 1 Dec 2023 17:45:50 +0100 Subject: [PATCH 1/9] draft implementation of RaisesGroup --- src/_pytest/python_api.py | 157 +++++++++++++ testing/python/expected_exception_group.py | 250 +++++++++++++++++++++ 2 files changed, 407 insertions(+) create mode 100644 testing/python/expected_exception_group.py diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index 07db0f234d4..d7594cbf940 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -1,5 +1,7 @@ import math import pprint +import re +import sys from collections.abc import Collection from collections.abc import Sized from decimal import Decimal @@ -10,6 +12,8 @@ from typing import cast from typing import ContextManager from typing import final +from typing import Generic +from typing import Iterable from typing import List from typing import Mapping from typing import Optional @@ -28,6 +32,10 @@ if TYPE_CHECKING: from numpy import ndarray + from typing_extensions import TypeGuard + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup def _non_numeric_type_error(value, at: Optional[str]) -> TypeError: @@ -987,6 +995,155 @@ def raises( # noqa: F811 raises.Exception = fail.Exception # type: ignore +class Matcher(Generic[E]): + def __init__( + self, + exception_type: Optional[Type[E]] = None, + match: Optional[Union[str, Pattern[str]]] = None, + check: Optional[Callable[[E], bool]] = None, + ): + if exception_type is None and match is None and check is None: + raise ValueError("You must specify at least one parameter to match on.") + self.exception_type = exception_type + self.match = match + self.check = check + + def matches(self, exception: E) -> "TypeGuard[E]": + if self.exception_type is not None and not isinstance( + exception, self.exception_type + ): + return False + if self.match is not None and not re.search(self.match, str(exception)): + return False + if self.check is not None and not self.check(exception): + return False + return True + + +if TYPE_CHECKING: + SuperClass = BaseExceptionGroup +else: + SuperClass = Generic + + +@final +class RaisesGroup( + ContextManager[_pytest._code.ExceptionInfo[BaseExceptionGroup[E]]], SuperClass[E] +): + # My_T = TypeVar("My_T", bound=Union[Type[E], Matcher[E], "RaisesGroup[E]"]) + def __init__( + self, + exceptions: Union[Type[E], Matcher[E], E], + *args: Union[Type[E], Matcher[E], E], + strict: bool = True, + match: Optional[Union[str, Pattern[str]]] = None, + ): + # could add parameter `notes: Optional[Tuple[str, Pattern[str]]] = None` + self.expected_exceptions = (exceptions, *args) + self.strict = strict + self.match_expr = match + self.message = f"DID NOT RAISE ExceptionGroup{repr(self.expected_exceptions)}" # type: ignore[misc] + + for exc in self.expected_exceptions: + if not isinstance(exc, (Matcher, RaisesGroup)) and not ( + isinstance(exc, type) and issubclass(exc, BaseException) + ): + raise ValueError( + "Invalid argument {exc} must be exception type, Matcher, or RaisesGroup." + ) + if isinstance(exc, RaisesGroup) and not strict: # type: ignore[unreachable] + raise ValueError( + "You cannot specify a nested structure inside a RaisesGroup with strict=False" + ) + + def __enter__(self) -> _pytest._code.ExceptionInfo[BaseExceptionGroup[E]]: + self.excinfo: _pytest._code.ExceptionInfo[ + BaseExceptionGroup[E] + ] = _pytest._code.ExceptionInfo.for_later() + return self.excinfo + + def _unroll_exceptions( + self, exceptions: Iterable[BaseException] + ) -> Iterable[BaseException]: + res: list[BaseException] = [] + for exc in exceptions: + if isinstance(exc, BaseExceptionGroup): + res.extend(self._unroll_exceptions(exc.exceptions)) + + else: + res.append(exc) + return res + + def matches( + self, + exc_val: Optional[BaseException], + ) -> "TypeGuard[BaseExceptionGroup[E]]": + if exc_val is None: + return False + if not isinstance(exc_val, BaseExceptionGroup): + return False + if not len(exc_val.exceptions) == len(self.expected_exceptions): + return False + remaining_exceptions = list(self.expected_exceptions) + actual_exceptions: Iterable[BaseException] = exc_val.exceptions + if not self.strict: + actual_exceptions = self._unroll_exceptions(actual_exceptions) + + # it should be possible to get RaisesGroup.matches typed so as not to + # need these type: ignores, but I'm not sure that's possible while also having it + # transparent for the end user. + for e in actual_exceptions: + for rem_e in remaining_exceptions: + # TODO: how to print string diff on mismatch? + # Probably accumulate them, and then if fail, print them + # Further QoL would be to print how the exception structure differs on non-match + if ( + (isinstance(rem_e, type) and isinstance(e, rem_e)) + or ( + isinstance(e, BaseExceptionGroup) + and isinstance(rem_e, RaisesGroup) + and rem_e.matches(e) + ) + or ( + isinstance(rem_e, Matcher) + and rem_e.matches(e) # type: ignore[arg-type] + ) + ): + remaining_exceptions.remove(rem_e) # type: ignore[arg-type] + break + else: + return False + return True + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> bool: + __tracebackhide__ = True + if exc_type is None: + fail(self.message) + assert self.excinfo is not None + + if not self.matches(exc_val): + return False + + # Cast to narrow the exception type now that it's verified. + exc_info = cast( + Tuple[Type[BaseExceptionGroup[E]], BaseExceptionGroup[E], TracebackType], + (exc_type, exc_val, exc_tb), + ) + self.excinfo.fill_unfilled(exc_info) + if self.match_expr is not None: + self.excinfo.match(self.match_expr) + return True + + def __repr__(self) -> str: + # TODO: [Base]ExceptionGroup + return f"ExceptionGroup{self.expected_exceptions}" + + @final class RaisesContext(ContextManager[_pytest._code.ExceptionInfo[E]]): def __init__( diff --git a/testing/python/expected_exception_group.py b/testing/python/expected_exception_group.py new file mode 100644 index 00000000000..bea04acfc5e --- /dev/null +++ b/testing/python/expected_exception_group.py @@ -0,0 +1,250 @@ +import sys +from typing import TYPE_CHECKING + +import pytest +from _pytest.python_api import Matcher +from _pytest.python_api import RaisesGroup + +# TODO: make a public export + +if TYPE_CHECKING: + from typing_extensions import assert_type + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + + +class TestRaisesGroup: + def test_raises_group(self) -> None: + with pytest.raises( + ValueError, + match="^Invalid argument {exc} must be exception type, Matcher, or RaisesGroup.$", + ): + RaisesGroup(ValueError()) + + with RaisesGroup(ValueError): + raise ExceptionGroup("foo", (ValueError(),)) + + with RaisesGroup(SyntaxError): + with RaisesGroup(ValueError): + raise ExceptionGroup("foo", (SyntaxError(),)) + + # multiple exceptions + with RaisesGroup(ValueError, SyntaxError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # order doesn't matter + with RaisesGroup(SyntaxError, ValueError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # nested exceptions + with RaisesGroup(RaisesGroup(ValueError)): + raise ExceptionGroup("foo", (ExceptionGroup("bar", (ValueError(),)),)) + + with RaisesGroup( + SyntaxError, + RaisesGroup(ValueError), + RaisesGroup(RuntimeError), + ): + raise ExceptionGroup( + "foo", + ( + SyntaxError(), + ExceptionGroup("bar", (ValueError(),)), + ExceptionGroup("", (RuntimeError(),)), + ), + ) + + # will error if there's excess exceptions + with pytest.raises(ExceptionGroup): + with RaisesGroup(ValueError): + raise ExceptionGroup("", (ValueError(), ValueError())) + + with pytest.raises(ExceptionGroup): + with RaisesGroup(ValueError): + raise ExceptionGroup("", (RuntimeError(), ValueError())) + + # will error if there's missing exceptions + with pytest.raises(ExceptionGroup): + with RaisesGroup(ValueError, ValueError): + raise ExceptionGroup("", (ValueError(),)) + + with pytest.raises(ExceptionGroup): + with RaisesGroup(ValueError, SyntaxError): + raise ExceptionGroup("", (ValueError(),)) + + # loose semantics, as with expect* + with RaisesGroup(ValueError, strict=False): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # mixed loose is possible if you want it to be at least N deep + with RaisesGroup(RaisesGroup(ValueError, strict=True)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + with RaisesGroup(RaisesGroup(ValueError, strict=False)): + raise ExceptionGroup( + "", (ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)),) + ) + + # but not the other way around + with pytest.raises( + ValueError, + match="^You cannot specify a nested structure inside a RaisesGroup with strict=False$", + ): + RaisesGroup(RaisesGroup(ValueError), strict=False) + + # currently not fully identical in behaviour to expect*, which would also catch an unwrapped exception + with pytest.raises(ValueError): + with RaisesGroup(ValueError, strict=False): + raise ValueError + + def test_match(self) -> None: + # supports match string + with RaisesGroup(ValueError, match="bar"): + raise ExceptionGroup("bar", (ValueError(),)) + + try: + with RaisesGroup(ValueError, match="foo"): + raise ExceptionGroup("bar", (ValueError(),)) + except AssertionError as e: + assert str(e).startswith("Regex pattern did not match.") + else: + raise AssertionError("Expected pytest.raises.Exception") + + def test_RaisesGroup_matches(self) -> None: + eeg = RaisesGroup(ValueError) + assert not eeg.matches(None) + assert not eeg.matches(ValueError()) + assert eeg.matches(ExceptionGroup("", (ValueError(),))) + + def test_message(self) -> None: + try: + with RaisesGroup(ValueError): + ... + except pytest.fail.Exception as e: + assert e.msg == f"DID NOT RAISE ExceptionGroup({repr(ValueError)},)" + else: + assert False, "Expected pytest.raises.Exception" + try: + with RaisesGroup(RaisesGroup(ValueError)): + ... + except pytest.fail.Exception as e: + assert ( + e.msg + == f"DID NOT RAISE ExceptionGroup(ExceptionGroup({repr(ValueError)},),)" + ) + else: + assert False, "Expected pytest.raises.Exception" + + def test_matcher(self) -> None: + with pytest.raises( + ValueError, match="^You must specify at least one parameter to match on.$" + ): + Matcher() + + with RaisesGroup(Matcher(ValueError)): + raise ExceptionGroup("", (ValueError(),)) + try: + with RaisesGroup(Matcher(TypeError)): + raise ExceptionGroup("", (ValueError(),)) + except ExceptionGroup: + pass + else: + assert False, "Expected pytest.raises.Exception" + + def test_matcher_match(self) -> None: + with RaisesGroup(Matcher(ValueError, "foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + try: + with RaisesGroup(Matcher(ValueError, "foo")): + raise ExceptionGroup("", (ValueError("bar"),)) + except ExceptionGroup: + pass + else: + assert False, "Expected pytest.raises.Exception" + + # Can be used without specifying the type + with RaisesGroup(Matcher(match="foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + try: + with RaisesGroup(Matcher(match="foo")): + raise ExceptionGroup("", (ValueError("bar"),)) + except ExceptionGroup: + pass + else: + assert False, "Expected pytest.raises.Exception" + + def test_Matcher_check(self) -> None: + def check_oserror_and_errno_is_5(e: BaseException) -> bool: + return isinstance(e, OSError) and e.errno == 5 + + with RaisesGroup(Matcher(check=check_oserror_and_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # specifying exception_type narrows the parameter type to the callable + def check_errno_is_5(e: OSError) -> bool: + return e.errno == 5 + + with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + try: + with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): + raise ExceptionGroup("", (OSError(6, ""),)) + except ExceptionGroup: + pass + else: + assert False, "Expected pytest.raises.Exception" + + if TYPE_CHECKING: + # getting the typing working satisfactory is very tricky + # but with RaisesGroup being seen as a subclass of BaseExceptionGroup + # most end-user cases of checking excinfo.value.foobar should work fine now. + def test_types_0(self) -> None: + _: BaseExceptionGroup[ValueError] = RaisesGroup(ValueError) + _ = RaisesGroup(RaisesGroup(ValueError)) # type: ignore[arg-type] + a: BaseExceptionGroup[BaseExceptionGroup[ValueError]] + a = RaisesGroup(RaisesGroup(ValueError)) + a = BaseExceptionGroup("", (BaseExceptionGroup("", (ValueError(),)),)) + assert a + + def test_types_1(self) -> None: + with RaisesGroup(ValueError) as e: + raise ExceptionGroup("foo", (ValueError(),)) + assert_type(e.value, BaseExceptionGroup[ValueError]) + # assert_type(e.value, RaisesGroup[ValueError]) + + def test_types_2(self) -> None: + exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup( + "", (ValueError(),) + ) + if RaisesGroup(ValueError).matches(exc): + assert_type(exc, BaseExceptionGroup[ValueError]) + + def test_types_3(self) -> None: + e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( + "", (KeyboardInterrupt(),) + ) + if RaisesGroup(ValueError).matches(e): + assert_type(e, BaseExceptionGroup[ValueError]) + + def test_types_4(self) -> None: + with RaisesGroup(Matcher(ValueError)) as e: + ... + _: BaseExceptionGroup[ValueError] = e.value + assert_type(e.value, BaseExceptionGroup[ValueError]) + + def test_types_5(self) -> None: + with RaisesGroup(RaisesGroup(ValueError)) as excinfo: + raise ExceptionGroup("foo", (ValueError(),)) + _: BaseExceptionGroup[BaseExceptionGroup[ValueError]] = excinfo.value + assert_type( + excinfo.value, + BaseExceptionGroup[RaisesGroup[ValueError]], + ) + print(excinfo.value.exceptions[0].exceptions[0]) + + def test_types_6(self) -> None: + exc: ExceptionGroup[ExceptionGroup[ValueError]] = ... # type: ignore[assignment] + if RaisesGroup(RaisesGroup(ValueError)).matches(exc): # type: ignore[arg-type] + # ugly + assert_type(exc, BaseExceptionGroup[RaisesGroup[ValueError]]) From 43f62eb6b5e01320224d8f6241886b9bd209dc3e Mon Sep 17 00:00:00 2001 From: jakkdl Date: Sun, 21 Jan 2024 17:12:56 +0100 Subject: [PATCH 2/9] pare down implementation to mimimum viable, add assert_matches that has assertions with descriptive outputs for why a match failed --- src/_pytest/python_api.py | 167 ++++------ src/pytest/__init__.py | 2 + testing/python/expected_exception_group.py | 365 ++++++++------------- 3 files changed, 199 insertions(+), 335 deletions(-) diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index d7594cbf940..7231b28af6a 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -1,6 +1,5 @@ import math import pprint -import re import sys from collections.abc import Collection from collections.abc import Sized @@ -12,7 +11,6 @@ from typing import cast from typing import ContextManager from typing import final -from typing import Generic from typing import Iterable from typing import List from typing import Mapping @@ -995,66 +993,39 @@ def raises( # noqa: F811 raises.Exception = fail.Exception # type: ignore -class Matcher(Generic[E]): - def __init__( - self, - exception_type: Optional[Type[E]] = None, - match: Optional[Union[str, Pattern[str]]] = None, - check: Optional[Callable[[E], bool]] = None, - ): - if exception_type is None and match is None and check is None: - raise ValueError("You must specify at least one parameter to match on.") - self.exception_type = exception_type - self.match = match - self.check = check - - def matches(self, exception: E) -> "TypeGuard[E]": - if self.exception_type is not None and not isinstance( - exception, self.exception_type - ): - return False - if self.match is not None and not re.search(self.match, str(exception)): - return False - if self.check is not None and not self.check(exception): - return False - return True - - -if TYPE_CHECKING: - SuperClass = BaseExceptionGroup -else: - SuperClass = Generic - - @final -class RaisesGroup( - ContextManager[_pytest._code.ExceptionInfo[BaseExceptionGroup[E]]], SuperClass[E] -): - # My_T = TypeVar("My_T", bound=Union[Type[E], Matcher[E], "RaisesGroup[E]"]) +class RaisesGroup(ContextManager[_pytest._code.ExceptionInfo[BaseExceptionGroup[E]]]): + """Helper for catching exceptions wrapped in an ExceptionGroup. + + Similar to pytest.raises, except: + * It requires that the exception is inside an exceptiongroup + * It is only able to be used as a contextmanager + * Due to the above, is not split into a caller function and a cm class + Similar to trio.RaisesGroup, except: + * does not handle multiple levels of nested groups. + * does not have trio.Matcher, to add matching on the sub-exception + * does not handle multiple exceptions in the exceptiongroup. + + TODO: copy over docstring example usage from trio.RaisesGroup + """ + def __init__( self, - exceptions: Union[Type[E], Matcher[E], E], - *args: Union[Type[E], Matcher[E], E], - strict: bool = True, - match: Optional[Union[str, Pattern[str]]] = None, + exception: Type[E], + check: Optional[Callable[[BaseExceptionGroup[E]], bool]] = None, ): - # could add parameter `notes: Optional[Tuple[str, Pattern[str]]] = None` - self.expected_exceptions = (exceptions, *args) - self.strict = strict - self.match_expr = match - self.message = f"DID NOT RAISE ExceptionGroup{repr(self.expected_exceptions)}" # type: ignore[misc] - - for exc in self.expected_exceptions: - if not isinstance(exc, (Matcher, RaisesGroup)) and not ( - isinstance(exc, type) and issubclass(exc, BaseException) - ): - raise ValueError( - "Invalid argument {exc} must be exception type, Matcher, or RaisesGroup." - ) - if isinstance(exc, RaisesGroup) and not strict: # type: ignore[unreachable] - raise ValueError( - "You cannot specify a nested structure inside a RaisesGroup with strict=False" - ) + # copied from raises() above + if not isinstance(exception, type) or not issubclass(exception, BaseException): + msg = "expected exception must be a BaseException type, not {}" # type: ignore[unreachable] + not_a = ( + exception.__name__ + if isinstance(exception, type) + else type(exception).__name__ + ) + raise TypeError(msg.format(not_a)) + + self.exception = exception + self.check = check def __enter__(self) -> _pytest._code.ExceptionInfo[BaseExceptionGroup[E]]: self.excinfo: _pytest._code.ExceptionInfo[ @@ -1078,41 +1049,33 @@ def matches( self, exc_val: Optional[BaseException], ) -> "TypeGuard[BaseExceptionGroup[E]]": - if exc_val is None: - return False - if not isinstance(exc_val, BaseExceptionGroup): - return False - if not len(exc_val.exceptions) == len(self.expected_exceptions): - return False - remaining_exceptions = list(self.expected_exceptions) - actual_exceptions: Iterable[BaseException] = exc_val.exceptions - if not self.strict: - actual_exceptions = self._unroll_exceptions(actual_exceptions) - - # it should be possible to get RaisesGroup.matches typed so as not to - # need these type: ignores, but I'm not sure that's possible while also having it - # transparent for the end user. - for e in actual_exceptions: - for rem_e in remaining_exceptions: - # TODO: how to print string diff on mismatch? - # Probably accumulate them, and then if fail, print them - # Further QoL would be to print how the exception structure differs on non-match - if ( - (isinstance(rem_e, type) and isinstance(e, rem_e)) - or ( - isinstance(e, BaseExceptionGroup) - and isinstance(rem_e, RaisesGroup) - and rem_e.matches(e) - ) - or ( - isinstance(rem_e, Matcher) - and rem_e.matches(e) # type: ignore[arg-type] - ) - ): - remaining_exceptions.remove(rem_e) # type: ignore[arg-type] - break - else: - return False + return ( + exc_val is not None + and isinstance(exc_val, BaseExceptionGroup) + and len(exc_val.exceptions) == 1 + and isinstance(exc_val.exceptions[0], self.exception) + and (self.check is None or self.check(exc_val)) + ) + + def assert_matches( + self, + exc_val: Optional[BaseException], + ) -> "TypeGuard[BaseExceptionGroup[E]]": + assert ( + exc_val is not None + ), "Internal Error: exc_type is not None but exc_val is" + assert isinstance( + exc_val, BaseExceptionGroup + ), f"Expected an ExceptionGroup, not {type(exc_val)}" + assert ( + len(exc_val.exceptions) == 1 + ), f"Wrong number of exceptions: got {len(exc_val.exceptions)}, expected 1." + assert isinstance( + exc_val.exceptions[0], self.exception + ), f"Wrong type in group: got {type(exc_val.exceptions[0])}, expected {self.exception}" + if self.check is not None: + assert self.check(exc_val), f"Check failed on {repr(exc_val)}." + return True def __exit__( @@ -1123,11 +1086,10 @@ def __exit__( ) -> bool: __tracebackhide__ = True if exc_type is None: - fail(self.message) - assert self.excinfo is not None + fail("DID NOT RAISE ANY EXCEPTION, expected " + self.expected_type()) + assert self.excinfo is not None, "__exit__ without __enter__" - if not self.matches(exc_val): - return False + self.assert_matches(exc_val) # Cast to narrow the exception type now that it's verified. exc_info = cast( @@ -1135,13 +1097,14 @@ def __exit__( (exc_type, exc_val, exc_tb), ) self.excinfo.fill_unfilled(exc_info) - if self.match_expr is not None: - self.excinfo.match(self.match_expr) return True - def __repr__(self) -> str: - # TODO: [Base]ExceptionGroup - return f"ExceptionGroup{self.expected_exceptions}" + def expected_type(self) -> str: + if not issubclass(self.exception, Exception): + base = "Base" + else: + base = "" + return f"{base}ExceptionGroup({self.exception})" @final diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index 0aa496a2fa7..23829299271 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -57,6 +57,7 @@ from _pytest.python import Package from _pytest.python_api import approx from _pytest.python_api import raises +from _pytest.python_api import RaisesGroup from _pytest.recwarn import deprecated_call from _pytest.recwarn import WarningsRecorder from _pytest.recwarn import warns @@ -146,6 +147,7 @@ "PytestUnraisableExceptionWarning", "PytestWarning", "raises", + "RaisesGroup", "RecordedHookCall", "register_assert_rewrite", "RunResult", diff --git a/testing/python/expected_exception_group.py b/testing/python/expected_exception_group.py index bea04acfc5e..b6edf3d82ed 100644 --- a/testing/python/expected_exception_group.py +++ b/testing/python/expected_exception_group.py @@ -1,11 +1,10 @@ +import re import sys from typing import TYPE_CHECKING import pytest -from _pytest.python_api import Matcher -from _pytest.python_api import RaisesGroup - -# TODO: make a public export +from _pytest.outcomes import Failed +from pytest import RaisesGroup if TYPE_CHECKING: from typing_extensions import assert_type @@ -14,237 +13,137 @@ from exceptiongroup import ExceptionGroup -class TestRaisesGroup: - def test_raises_group(self) -> None: - with pytest.raises( - ValueError, - match="^Invalid argument {exc} must be exception type, Matcher, or RaisesGroup.$", - ): - RaisesGroup(ValueError()) +def test_raises_group() -> None: + # wrong type to constructor + with pytest.raises( + TypeError, + match="^expected exception must be a BaseException type, not ValueError$", + ): + RaisesGroup(ValueError()) # type: ignore[arg-type] + + # working example + with RaisesGroup(ValueError): + raise ExceptionGroup("foo", (ValueError(),)) + + with RaisesGroup(ValueError, check=lambda x: True): + raise ExceptionGroup("foo", (ValueError(),)) + # wrong subexception + with pytest.raises( + AssertionError, + match="Wrong type in group: got , expected ", + ): with RaisesGroup(ValueError): - raise ExceptionGroup("foo", (ValueError(),)) + raise ExceptionGroup("foo", (SyntaxError(),)) - with RaisesGroup(SyntaxError): - with RaisesGroup(ValueError): - raise ExceptionGroup("foo", (SyntaxError(),)) - - # multiple exceptions - with RaisesGroup(ValueError, SyntaxError): - raise ExceptionGroup("foo", (ValueError(), SyntaxError())) - - # order doesn't matter - with RaisesGroup(SyntaxError, ValueError): - raise ExceptionGroup("foo", (ValueError(), SyntaxError())) - - # nested exceptions - with RaisesGroup(RaisesGroup(ValueError)): - raise ExceptionGroup("foo", (ExceptionGroup("bar", (ValueError(),)),)) - - with RaisesGroup( - SyntaxError, - RaisesGroup(ValueError), - RaisesGroup(RuntimeError), - ): - raise ExceptionGroup( - "foo", - ( - SyntaxError(), - ExceptionGroup("bar", (ValueError(),)), - ExceptionGroup("", (RuntimeError(),)), - ), - ) - - # will error if there's excess exceptions - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError): - raise ExceptionGroup("", (ValueError(), ValueError())) - - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError): - raise ExceptionGroup("", (RuntimeError(), ValueError())) - - # will error if there's missing exceptions - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, ValueError): - raise ExceptionGroup("", (ValueError(),)) - - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, SyntaxError): - raise ExceptionGroup("", (ValueError(),)) - - # loose semantics, as with expect* - with RaisesGroup(ValueError, strict=False): - raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + # will error if there's excess exceptions + with pytest.raises( + AssertionError, match="Wrong number of exceptions: got 2, expected 1" + ): + with RaisesGroup(ValueError): + raise ExceptionGroup("", (ValueError(), ValueError())) - # mixed loose is possible if you want it to be at least N deep - with RaisesGroup(RaisesGroup(ValueError, strict=True)): + # double nested exceptions is not (currently) supported (contrary to expect*) + with pytest.raises( + AssertionError, + match="Wrong type in group: got , expected ", + ): + with RaisesGroup(ValueError): raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) - with RaisesGroup(RaisesGroup(ValueError, strict=False)): - raise ExceptionGroup( - "", (ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)),) - ) - - # but not the other way around - with pytest.raises( - ValueError, - match="^You cannot specify a nested structure inside a RaisesGroup with strict=False$", - ): - RaisesGroup(RaisesGroup(ValueError), strict=False) - - # currently not fully identical in behaviour to expect*, which would also catch an unwrapped exception - with pytest.raises(ValueError): - with RaisesGroup(ValueError, strict=False): - raise ValueError - - def test_match(self) -> None: - # supports match string - with RaisesGroup(ValueError, match="bar"): - raise ExceptionGroup("bar", (ValueError(),)) - - try: - with RaisesGroup(ValueError, match="foo"): - raise ExceptionGroup("bar", (ValueError(),)) - except AssertionError as e: - assert str(e).startswith("Regex pattern did not match.") - else: - raise AssertionError("Expected pytest.raises.Exception") - - def test_RaisesGroup_matches(self) -> None: - eeg = RaisesGroup(ValueError) - assert not eeg.matches(None) - assert not eeg.matches(ValueError()) - assert eeg.matches(ExceptionGroup("", (ValueError(),))) - - def test_message(self) -> None: - try: - with RaisesGroup(ValueError): - ... - except pytest.fail.Exception as e: - assert e.msg == f"DID NOT RAISE ExceptionGroup({repr(ValueError)},)" - else: - assert False, "Expected pytest.raises.Exception" - try: - with RaisesGroup(RaisesGroup(ValueError)): - ... - except pytest.fail.Exception as e: - assert ( - e.msg - == f"DID NOT RAISE ExceptionGroup(ExceptionGroup({repr(ValueError)},),)" - ) - else: - assert False, "Expected pytest.raises.Exception" - - def test_matcher(self) -> None: - with pytest.raises( - ValueError, match="^You must specify at least one parameter to match on.$" - ): - Matcher() - - with RaisesGroup(Matcher(ValueError)): - raise ExceptionGroup("", (ValueError(),)) - try: - with RaisesGroup(Matcher(TypeError)): - raise ExceptionGroup("", (ValueError(),)) - except ExceptionGroup: - pass - else: - assert False, "Expected pytest.raises.Exception" - - def test_matcher_match(self) -> None: - with RaisesGroup(Matcher(ValueError, "foo")): - raise ExceptionGroup("", (ValueError("foo"),)) - try: - with RaisesGroup(Matcher(ValueError, "foo")): - raise ExceptionGroup("", (ValueError("bar"),)) - except ExceptionGroup: - pass - else: - assert False, "Expected pytest.raises.Exception" - - # Can be used without specifying the type - with RaisesGroup(Matcher(match="foo")): - raise ExceptionGroup("", (ValueError("foo"),)) - try: - with RaisesGroup(Matcher(match="foo")): - raise ExceptionGroup("", (ValueError("bar"),)) - except ExceptionGroup: - pass - else: - assert False, "Expected pytest.raises.Exception" - - def test_Matcher_check(self) -> None: - def check_oserror_and_errno_is_5(e: BaseException) -> bool: - return isinstance(e, OSError) and e.errno == 5 - - with RaisesGroup(Matcher(check=check_oserror_and_errno_is_5)): - raise ExceptionGroup("", (OSError(5, ""),)) - - # specifying exception_type narrows the parameter type to the callable - def check_errno_is_5(e: OSError) -> bool: - return e.errno == 5 - - with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): - raise ExceptionGroup("", (OSError(5, ""),)) - - try: - with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): - raise ExceptionGroup("", (OSError(6, ""),)) - except ExceptionGroup: - pass - else: - assert False, "Expected pytest.raises.Exception" - - if TYPE_CHECKING: - # getting the typing working satisfactory is very tricky - # but with RaisesGroup being seen as a subclass of BaseExceptionGroup - # most end-user cases of checking excinfo.value.foobar should work fine now. - def test_types_0(self) -> None: - _: BaseExceptionGroup[ValueError] = RaisesGroup(ValueError) - _ = RaisesGroup(RaisesGroup(ValueError)) # type: ignore[arg-type] - a: BaseExceptionGroup[BaseExceptionGroup[ValueError]] - a = RaisesGroup(RaisesGroup(ValueError)) - a = BaseExceptionGroup("", (BaseExceptionGroup("", (ValueError(),)),)) - assert a - - def test_types_1(self) -> None: - with RaisesGroup(ValueError) as e: - raise ExceptionGroup("foo", (ValueError(),)) - assert_type(e.value, BaseExceptionGroup[ValueError]) - # assert_type(e.value, RaisesGroup[ValueError]) - - def test_types_2(self) -> None: - exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup( - "", (ValueError(),) - ) - if RaisesGroup(ValueError).matches(exc): - assert_type(exc, BaseExceptionGroup[ValueError]) - - def test_types_3(self) -> None: - e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( - "", (KeyboardInterrupt(),) - ) - if RaisesGroup(ValueError).matches(e): - assert_type(e, BaseExceptionGroup[ValueError]) - - def test_types_4(self) -> None: - with RaisesGroup(Matcher(ValueError)) as e: - ... - _: BaseExceptionGroup[ValueError] = e.value - assert_type(e.value, BaseExceptionGroup[ValueError]) - - def test_types_5(self) -> None: - with RaisesGroup(RaisesGroup(ValueError)) as excinfo: - raise ExceptionGroup("foo", (ValueError(),)) - _: BaseExceptionGroup[BaseExceptionGroup[ValueError]] = excinfo.value - assert_type( - excinfo.value, - BaseExceptionGroup[RaisesGroup[ValueError]], - ) - print(excinfo.value.exceptions[0].exceptions[0]) - - def test_types_6(self) -> None: - exc: ExceptionGroup[ExceptionGroup[ValueError]] = ... # type: ignore[assignment] - if RaisesGroup(RaisesGroup(ValueError)).matches(exc): # type: ignore[arg-type] - # ugly - assert_type(exc, BaseExceptionGroup[RaisesGroup[ValueError]]) + + # you'd need to write + with RaisesGroup(ExceptionGroup) as excinfo: + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + RaisesGroup(ValueError).assert_matches(excinfo.value.exceptions[0]) + + # unwrapped exceptions are not accepted (contrary to expect*) + with pytest.raises( + AssertionError, match="Expected an ExceptionGroup, not None: + eeg = RaisesGroup(ValueError) + # exc_val is None + assert not eeg.matches(None) + # exc_val is not an exceptiongroup + assert not eeg.matches(ValueError()) + # wrong length + assert not eeg.matches(ExceptionGroup("", (ValueError(), ValueError()))) + # wrong type + assert not eeg.matches(ExceptionGroup("", (TypeError(),))) + # check fails + assert not RaisesGroup(ValueError, check=lambda _: False).matches( + ExceptionGroup("", (ValueError(),)) + ) + # success + assert eeg.matches(ExceptionGroup("", (ValueError(),))) + + +def test_RaisesGroup_assert_matches() -> None: + """Check direct use of RaisesGroup.assert_matches, without a context manager""" + eeg = RaisesGroup(ValueError) + with pytest.raises(AssertionError): + eeg.assert_matches(None) + with pytest.raises(AssertionError): + eeg.assert_matches(ValueError()) + eeg.assert_matches(ExceptionGroup("", (ValueError(),))) + + +def test_message() -> None: + with pytest.raises( + Failed, + match=re.escape( + f"DID NOT RAISE ANY EXCEPTION, expected ExceptionGroup({repr(ValueError)})" + ), + ): + with RaisesGroup(ValueError): + ... + + with pytest.raises( + Failed, + match=re.escape( + f"DID NOT RAISE ANY EXCEPTION, expected BaseExceptionGroup({repr(KeyboardInterrupt)})" + ), + ): + with RaisesGroup(KeyboardInterrupt): + ... + + +if TYPE_CHECKING: + + def test_types_1() -> None: + with RaisesGroup(ValueError) as e: + raise ExceptionGroup("foo", (ValueError(),)) + assert_type(e.value, BaseExceptionGroup[ValueError]) + + def test_types_2() -> None: + exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup( + "", (ValueError(),) + ) + if RaisesGroup(ValueError).assert_matches(exc): + assert_type(exc, BaseExceptionGroup[ValueError]) + + def test_types_3() -> None: + e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( + "", (KeyboardInterrupt(),) + ) + if RaisesGroup(ValueError).matches(e): + assert_type(e, BaseExceptionGroup[ValueError]) + + def test_types_4() -> None: + e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( + "", (KeyboardInterrupt(),) + ) + # not currently possible: https://github.com/python/typing/issues/930 + RaisesGroup(ValueError).assert_matches(e) + assert_type(e, BaseExceptionGroup[ValueError]) # type: ignore[assert-type] From 620f19b70c696bfbf102dffa47f0c1665f25f4d5 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Sun, 21 Jan 2024 17:22:51 +0100 Subject: [PATCH 3/9] rename test file --- testing/python/{expected_exception_group.py => raises_group.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename testing/python/{expected_exception_group.py => raises_group.py} (100%) diff --git a/testing/python/expected_exception_group.py b/testing/python/raises_group.py similarity index 100% rename from testing/python/expected_exception_group.py rename to testing/python/raises_group.py From 2a12ed97ce10462389ebbf25fd4a5d94a3628232 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Sun, 21 Jan 2024 17:32:55 +0100 Subject: [PATCH 4/9] remove unused _unroll_exceptions --- src/_pytest/python_api.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index 7231b28af6a..7b3b71db34e 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -11,7 +11,6 @@ from typing import cast from typing import ContextManager from typing import final -from typing import Iterable from typing import List from typing import Mapping from typing import Optional @@ -1033,18 +1032,6 @@ def __enter__(self) -> _pytest._code.ExceptionInfo[BaseExceptionGroup[E]]: ] = _pytest._code.ExceptionInfo.for_later() return self.excinfo - def _unroll_exceptions( - self, exceptions: Iterable[BaseException] - ) -> Iterable[BaseException]: - res: list[BaseException] = [] - for exc in exceptions: - if isinstance(exc, BaseExceptionGroup): - res.extend(self._unroll_exceptions(exc.exceptions)) - - else: - res.append(exc) - return res - def matches( self, exc_val: Optional[BaseException], From 80441bd7e9c3a6be239143db6b21644bb160553b Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 29 Jan 2025 15:18:45 +0100 Subject: [PATCH 5/9] copy over code from trio with minimal changes --- src/_pytest/_raises_group.py | 947 ++++++++++++++++++++ src/_pytest/python_api.py | 105 --- src/pytest/__init__.py | 3 +- testing/python/raises_group.py | 1148 ++++++++++++++++++++++--- testing/python/typing_raises_group.py | 234 +++++ 5 files changed, 2235 insertions(+), 202 deletions(-) create mode 100644 src/_pytest/_raises_group.py create mode 100644 testing/python/typing_raises_group.py diff --git a/src/_pytest/_raises_group.py b/src/_pytest/_raises_group.py new file mode 100644 index 00000000000..04106c6ec3c --- /dev/null +++ b/src/_pytest/_raises_group.py @@ -0,0 +1,947 @@ +from __future__ import annotations + +from abc import ABC +from abc import abstractmethod +import re +from re import Pattern +import sys +from textwrap import indent +from typing import cast +from typing import final +from typing import Generic +from typing import Literal +from typing import overload +from typing import TYPE_CHECKING + +from _pytest._code import ExceptionInfo + + +if TYPE_CHECKING: + # sphinx will *only* work if we use types.TracebackType, and import + # *inside* TYPE_CHECKING. No other combination works..... + from collections.abc import Callable + from collections.abc import Sequence + import types + + from typing_extensions import TypeGuard + from typing_extensions import TypeVar + + # this conditional definition is because we want to allow a TypeVar default + MatchE = TypeVar( + "MatchE", + bound=BaseException, + default=BaseException, + covariant=True, + ) +else: + from typing import TypeVar + + MatchE = TypeVar("MatchE", bound=BaseException, covariant=True) + +# RaisesGroup doesn't work with a default. +BaseExcT_co = TypeVar("BaseExcT_co", bound=BaseException, covariant=True) +BaseExcT_1 = TypeVar("BaseExcT_1", bound=BaseException) +BaseExcT_2 = TypeVar("BaseExcT_2", bound=BaseException) +ExcT_1 = TypeVar("ExcT_1", bound=Exception) +ExcT_2 = TypeVar("ExcT_2", bound=Exception) + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + + +# this differs slightly from pytest.ExceptionInfo +# we do `getattr(exc, "message", str(exc)`, they do str(exc) +# this is because we don't want '(x sub-exceptions)' when checking +# exception groups. +# Does it differ in behaviour on any other exceptions? Should we do an +# isinstance check instead? +def _stringify_exception(exc: BaseException) -> str: + return "\n".join( + [ + getattr(exc, "message", str(exc)), + *getattr(exc, "__notes__", []), + ], + ) + + +# String patterns default to including the unicode flag. +_REGEX_NO_FLAGS = re.compile(r"").flags + + +def _match_pattern(match: Pattern[str]) -> str | Pattern[str]: + """Helper function to remove redundant `re.compile` calls when printing regex""" + return match.pattern if match.flags == _REGEX_NO_FLAGS else match + + +def repr_callable(fun: Callable[[BaseExcT_1], bool]) -> str: + """Get the repr of a ``check`` parameter. + + Split out so it can be monkeypatched (e.g. by our hypothesis plugin) + """ + return repr(fun) + + +def _exception_type_name(e: type[BaseException]) -> str: + return repr(e.__name__) + + +def _check_raw_type( + expected_type: type[BaseException] | None, + exception: BaseException, +) -> str | None: + if expected_type is None: + return None + + if not isinstance( + exception, + expected_type, + ): + actual_type_str = _exception_type_name(type(exception)) + expected_type_str = _exception_type_name(expected_type) + if isinstance(exception, BaseExceptionGroup) and not issubclass( + expected_type, BaseExceptionGroup + ): + return f"Unexpected nested {actual_type_str}, expected {expected_type_str}" + return f"{actual_type_str} is not of type {expected_type_str}" + return None + + +class AbstractMatcher(ABC, Generic[BaseExcT_co]): + """ABC with common functionality shared between Matcher and RaisesGroup""" + + def __init__( + self, + match: str | Pattern[str] | None, + check: Callable[[BaseExcT_co], bool] | None, + ) -> None: + if isinstance(match, str): + self.match: Pattern[str] | None = re.compile(match) + else: + self.match = match + self.check = check + self._fail_reason: str | None = None + + # used to suppress repeated printing of `repr(self.check)` + self._nested: bool = False + + @property + def fail_reason(self) -> str | None: + """Set after a call to `matches` to give a human-readable + reason for why the match failed. + When used as a context manager the string will be given as the text of an + `AssertionError`""" + return self._fail_reason + + def _check_check( + self: AbstractMatcher[BaseExcT_1], + exception: BaseExcT_1, + ) -> bool: + if self.check is None: + return True + + if self.check(exception): + return True + + check_repr = "" if self._nested else " " + repr_callable(self.check) + self._fail_reason = f"check{check_repr} did not return True" + return False + + def _check_match(self, e: BaseException) -> bool: + if self.match is None or re.search( + self.match, + stringified_exception := _stringify_exception(e), + ): + return True + + maybe_specify_type = ( + f" of {_exception_type_name(type(e))}" + if isinstance(e, BaseExceptionGroup) + else "" + ) + self._fail_reason = f"Regex pattern {_match_pattern(self.match)!r} did not match {stringified_exception!r}{maybe_specify_type}" + if _match_pattern(self.match) == stringified_exception: + self._fail_reason += "\n Did you mean to `re.escape()` the regex?" + return False + + @abstractmethod + def matches( + self: AbstractMatcher[BaseExcT_1], exc_val: BaseException + ) -> TypeGuard[BaseExcT_1]: + """Check if an exception matches the requirements of this AbstractMatcher. + If it fails, `AbstractMatcher.fail_reason` should be set. + """ + + +@final +class Matcher(AbstractMatcher[MatchE]): + """Helper class to be used together with RaisesGroups when you want to specify requirements on sub-exceptions. Only specifying the type is redundant, and it's also unnecessary when the type is a nested `RaisesGroup` since it supports the same arguments. + The type is checked with `isinstance`, and does not need to be an exact match. If that is wanted you can use the ``check`` parameter. + :meth:`Matcher.matches` can also be used standalone to check individual exceptions. + + Examples:: + + with RaisesGroups(Matcher(ValueError, match="string")) + ... + with RaisesGroups(Matcher(check=lambda x: x.args == (3, "hello"))): + ... + with RaisesGroups(Matcher(check=lambda x: type(x) is ValueError)): + ... + + Tip: if you install ``hypothesis`` and import it in ``conftest.py`` you will get + readable ``repr``s of ``check`` callables in the output. + """ + + # At least one of the three parameters must be passed. + @overload + def __init__( + self: Matcher[MatchE], + exception_type: type[MatchE], + match: str | Pattern[str] = ..., + check: Callable[[MatchE], bool] = ..., + ) -> None: ... + + @overload + def __init__( + self: Matcher[BaseException], # Give E a value. + *, + match: str | Pattern[str], + # If exception_type is not provided, check() must do any typechecks itself. + check: Callable[[BaseException], bool] = ..., + ) -> None: ... + + @overload + def __init__(self, *, check: Callable[[BaseException], bool]) -> None: ... + + def __init__( + self, + exception_type: type[MatchE] | None = None, + match: str | Pattern[str] | None = None, + check: Callable[[MatchE], bool] | None = None, + ): + super().__init__(match, check) + if exception_type is None and match is None and check is None: + raise ValueError("You must specify at least one parameter to match on.") + if exception_type is not None and not issubclass(exception_type, BaseException): + raise ValueError( + f"exception_type {exception_type} must be a subclass of BaseException", + ) + self.exception_type = exception_type + + def matches( + self, + exception: BaseException, + ) -> TypeGuard[MatchE]: + """Check if an exception matches the requirements of this Matcher. + If it fails, `Matcher.fail_reason` will be set. + + Examples:: + + assert Matcher(ValueError).matches(my_exception): + # is equivalent to + assert isinstance(my_exception, ValueError) + + # this can be useful when checking e.g. the ``__cause__`` of an exception. + with pytest.raises(ValueError) as excinfo: + ... + assert Matcher(SyntaxError, match="foo").matches(excinfo.value.__cause__) + # above line is equivalent to + assert isinstance(excinfo.value.__cause__, SyntaxError) + assert re.search("foo", str(excinfo.value.__cause__) + + """ + if not self._check_type(exception): + return False + + if not self._check_match(exception): + return False + + return self._check_check(exception) + + def __repr__(self) -> str: + parameters = [] + if self.exception_type is not None: + parameters.append(self.exception_type.__name__) + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + parameters.append( + f"match={_match_pattern(self.match)!r}", + ) + if self.check is not None: + parameters.append(f"check={repr_callable(self.check)}") + return f"Matcher({', '.join(parameters)})" + + def _check_type(self, exception: BaseException) -> TypeGuard[MatchE]: + self._fail_reason = _check_raw_type(self.exception_type, exception) + return self._fail_reason is None + + +@final +class RaisesGroup(AbstractMatcher[BaseExceptionGroup[BaseExcT_co]]): + """Contextmanager for checking for an expected `ExceptionGroup`. + This works similar to ``pytest.raises``, but allows for specifying the structure of an `ExceptionGroup`. + `ExceptionInfo.group_contains` also tries to handle exception groups, but it is very bad at checking that you *didn't* get exceptions you didn't expect. + + + The catching behaviour differs from :ref:`except* ` in multiple different ways, being much stricter by default. By using ``allow_unwrapped=True`` and ``flatten_subgroups=True`` you can match ``except*`` fully when expecting a single exception. + + #. All specified exceptions must be present, *and no others*. + + * If you expect a variable number of exceptions you need to use ``pytest.raises(ExceptionGroup)`` and manually check the contained exceptions. Consider making use of :func:`Matcher.matches`. + + #. It will only catch exceptions wrapped in an exceptiongroup by default. + + * With ``allow_unwrapped=True`` you can specify a single expected exception or `Matcher` and it will match the exception even if it is not inside an `ExceptionGroup`. If you expect one of several different exception types you need to use a `Matcher` object. + + #. By default it cares about the full structure with nested `ExceptionGroup`'s. You can specify nested `ExceptionGroup`'s by passing `RaisesGroup` objects as expected exceptions. + + * With ``flatten_subgroups=True`` it will "flatten" the raised `ExceptionGroup`, extracting all exceptions inside any nested :class:`ExceptionGroup`, before matching. + + It does not care about the order of the exceptions, so ``RaisesGroups(ValueError, TypeError)`` is equivalent to ``RaisesGroups(TypeError, ValueError)``. + + Examples:: + + with RaisesGroups(ValueError): + raise ExceptionGroup("", (ValueError(),)) + with RaisesGroups( + ValueError, ValueError, Matcher(TypeError, match="expected int") + ): + ... + with RaisesGroups( + KeyboardInterrupt, + match="hello", + check=lambda x: type(x) is BaseExceptionGroup, + ): + ... + with RaisesGroups(RaisesGroups(ValueError)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # flatten_subgroups + with RaisesGroups(ValueError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # allow_unwrapped + with RaisesGroups(ValueError, allow_unwrapped=True): + raise ValueError + + + `RaisesGroup.matches` can also be used directly to check a standalone exception group. + + + The matching algorithm is greedy, which means cases such as this may fail:: + + with RaisesGroups(ValueError, Matcher(ValueError, match="hello")): + raise ExceptionGroup("", (ValueError("hello"), ValueError("goodbye"))) + + even though it generally does not care about the order of the exceptions in the group. + To avoid the above you should specify the first ValueError with a Matcher as well. + + Tip: if you install ``hypothesis`` and import it in ``conftest.py`` you will get + readable ``repr``s of ``check`` callables in the output. + """ + + # allow_unwrapped=True requires: singular exception, exception not being + # RaisesGroup instance, match is None, check is None + @overload + def __init__( + self, + exception: type[BaseExcT_co] | Matcher[BaseExcT_co], + *, + allow_unwrapped: Literal[True], + flatten_subgroups: bool = False, + ) -> None: ... + + # flatten_subgroups = True also requires no nested RaisesGroup + @overload + def __init__( + self, + exception: type[BaseExcT_co] | Matcher[BaseExcT_co], + *other_exceptions: type[BaseExcT_co] | Matcher[BaseExcT_co], + flatten_subgroups: Literal[True], + match: str | Pattern[str] | None = None, + check: Callable[[BaseExceptionGroup[BaseExcT_co]], bool] | None = None, + ) -> None: ... + + # simplify the typevars if possible (the following 3 are equivalent but go simpler->complicated) + # ... the first handles RaisesGroup[ValueError], the second RaisesGroup[ExceptionGroup[ValueError]], + # the third RaisesGroup[ValueError | ExceptionGroup[ValueError]]. + # ... otherwise, we will get results like RaisesGroup[ValueError | ExceptionGroup[Never]] (I think) + # (technically correct but misleading) + @overload + def __init__( + self: RaisesGroup[ExcT_1], + exception: type[ExcT_1] | Matcher[ExcT_1], + *other_exceptions: type[ExcT_1] | Matcher[ExcT_1], + match: str | Pattern[str] | None = None, + check: Callable[[ExceptionGroup[ExcT_1]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[ExceptionGroup[ExcT_2]], + exception: RaisesGroup[ExcT_2], + *other_exceptions: RaisesGroup[ExcT_2], + match: str | Pattern[str] | None = None, + check: Callable[[ExceptionGroup[ExceptionGroup[ExcT_2]]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[ExcT_1 | ExceptionGroup[ExcT_2]], + exception: type[ExcT_1] | Matcher[ExcT_1] | RaisesGroup[ExcT_2], + *other_exceptions: type[ExcT_1] | Matcher[ExcT_1] | RaisesGroup[ExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[[ExceptionGroup[ExcT_1 | ExceptionGroup[ExcT_2]]], bool] | None + ) = None, + ) -> None: ... + + # same as the above 3 but handling BaseException + @overload + def __init__( + self: RaisesGroup[BaseExcT_1], + exception: type[BaseExcT_1] | Matcher[BaseExcT_1], + *other_exceptions: type[BaseExcT_1] | Matcher[BaseExcT_1], + match: str | Pattern[str] | None = None, + check: Callable[[BaseExceptionGroup[BaseExcT_1]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[BaseExceptionGroup[BaseExcT_2]], + exception: RaisesGroup[BaseExcT_2], + *other_exceptions: RaisesGroup[BaseExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[[BaseExceptionGroup[BaseExceptionGroup[BaseExcT_2]]], bool] | None + ) = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]], + exception: type[BaseExcT_1] | Matcher[BaseExcT_1] | RaisesGroup[BaseExcT_2], + *other_exceptions: type[BaseExcT_1] + | Matcher[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[ + [BaseExceptionGroup[BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]]], + bool, + ] + | None + ) = None, + ) -> None: ... + + def __init__( + self: RaisesGroup[ExcT_1 | BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]], + exception: type[BaseExcT_1] | Matcher[BaseExcT_1] | RaisesGroup[BaseExcT_2], + *other_exceptions: type[BaseExcT_1] + | Matcher[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + allow_unwrapped: bool = False, + flatten_subgroups: bool = False, + match: str | Pattern[str] | None = None, + check: ( + Callable[[BaseExceptionGroup[BaseExcT_1]], bool] + | Callable[[ExceptionGroup[ExcT_1]], bool] + | None + ) = None, + ): + # The type hint on the `self` and `check` parameters uses different formats + # that are *very* hard to reconcile while adhering to the overloads, so we cast + # it to avoid an error when passing it to super().__init__ + check = cast( + "Callable[[" + "BaseExceptionGroup[ExcT_1|BaseExcT_1|BaseExceptionGroup[BaseExcT_2]]" + "], bool]", + check, + ) + super().__init__(match, check) + self.expected_exceptions: tuple[ + type[BaseExcT_co] | Matcher[BaseExcT_co] | RaisesGroup[BaseException], ... + ] = ( + exception, + *other_exceptions, + ) + self.allow_unwrapped = allow_unwrapped + self.flatten_subgroups: bool = flatten_subgroups + self.is_baseexceptiongroup = False + + if allow_unwrapped and other_exceptions: + raise ValueError( + "You cannot specify multiple exceptions with `allow_unwrapped=True.`" + " If you want to match one of multiple possible exceptions you should" + " use a `Matcher`." + " E.g. `Matcher(check=lambda e: isinstance(e, (...)))`", + ) + if allow_unwrapped and isinstance(exception, RaisesGroup): + raise ValueError( + "`allow_unwrapped=True` has no effect when expecting a `RaisesGroup`." + " You might want it in the expected `RaisesGroup`, or" + " `flatten_subgroups=True` if you don't care about the structure.", + ) + if allow_unwrapped and (match is not None or check is not None): + raise ValueError( + "`allow_unwrapped=True` bypasses the `match` and `check` parameters" + " if the exception is unwrapped. If you intended to match/check the" + " exception you should use a `Matcher` object. If you want to match/check" + " the exceptiongroup when the exception *is* wrapped you need to" + " do e.g. `if isinstance(exc.value, ExceptionGroup):" + " assert RaisesGroup(...).matches(exc.value)` afterwards.", + ) + + # verify `expected_exceptions` and set `self.is_baseexceptiongroup` + for exc in self.expected_exceptions: + if isinstance(exc, RaisesGroup): + if self.flatten_subgroups: + raise ValueError( + "You cannot specify a nested structure inside a RaisesGroup with" + " `flatten_subgroups=True`. The parameter will flatten subgroups" + " in the raised exceptiongroup before matching, which would never" + " match a nested structure.", + ) + self.is_baseexceptiongroup |= exc.is_baseexceptiongroup + exc._nested = True + elif isinstance(exc, Matcher): + if exc.exception_type is not None: + # Matcher __init__ assures it's a subclass of BaseException + self.is_baseexceptiongroup |= not issubclass( + exc.exception_type, + Exception, + ) + exc._nested = True + elif isinstance(exc, type) and issubclass(exc, BaseException): + self.is_baseexceptiongroup |= not issubclass(exc, Exception) + else: + raise ValueError( + f'Invalid argument "{exc!r}" must be exception type, Matcher, or' + " RaisesGroup.", + ) + + @overload + def __enter__( + self: RaisesGroup[ExcT_1], + ) -> ExceptionInfo[ExceptionGroup[ExcT_1]]: ... + @overload + def __enter__( + self: RaisesGroup[BaseExcT_1], + ) -> ExceptionInfo[BaseExceptionGroup[BaseExcT_1]]: ... + + def __enter__(self) -> ExceptionInfo[BaseExceptionGroup[BaseException]]: + self.excinfo: ExceptionInfo[BaseExceptionGroup[BaseExcT_co]] = ( + ExceptionInfo.for_later() + ) + return self.excinfo + + def __repr__(self) -> str: + reqs = [ + e.__name__ if isinstance(e, type) else repr(e) + for e in self.expected_exceptions + ] + if self.allow_unwrapped: + reqs.append(f"allow_unwrapped={self.allow_unwrapped}") + if self.flatten_subgroups: + reqs.append(f"flatten_subgroups={self.flatten_subgroups}") + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + reqs.append(f"match={_match_pattern(self.match)!r}") + if self.check is not None: + reqs.append(f"check={repr_callable(self.check)}") + return f"RaisesGroup({', '.join(reqs)})" + + def _unroll_exceptions( + self, + exceptions: Sequence[BaseException], + ) -> Sequence[BaseException]: + """Used if `flatten_subgroups=True`.""" + res: list[BaseException] = [] + for exc in exceptions: + if isinstance(exc, BaseExceptionGroup): + res.extend(self._unroll_exceptions(exc.exceptions)) + + else: + res.append(exc) + return res + + @overload + def matches( + self: RaisesGroup[ExcT_1], + exc_val: BaseException | None, + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def matches( + self: RaisesGroup[BaseExcT_1], + exc_val: BaseException | None, + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def matches( + self, + exc_val: BaseException | None, + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_co]]: + """Check if an exception matches the requirements of this RaisesGroup. + If it fails, `RaisesGroup.fail_reason` will be set. + + Example:: + + with pytest.raises(TypeError) as excinfo: + ... + assert RaisesGroups(ValueError).matches(excinfo.value.__cause__) + # the above line is equivalent to + myexc = excinfo.value.__cause + assert isinstance(myexc, BaseExceptionGroup) + assert len(myexc.exceptions) == 1 + assert isinstance(myexc.exceptions[0], ValueError) + """ + self._fail_reason = None + if exc_val is None: + self._fail_reason = "exception is None" + return False + if not isinstance(exc_val, BaseExceptionGroup): + # we opt to only print type of the exception here, as the repr would + # likely be quite long + not_group_msg = f"{type(exc_val).__name__!r} is not an exception group" + if len(self.expected_exceptions) > 1: + self._fail_reason = not_group_msg + return False + # if we have 1 expected exception, check if it would work even if + # allow_unwrapped is not set + res = self._check_expected(self.expected_exceptions[0], exc_val) + if res is None and self.allow_unwrapped: + return True + + if res is None: + self._fail_reason = ( + f"{not_group_msg}, but would match with `allow_unwrapped=True`" + ) + elif self.allow_unwrapped: + self._fail_reason = res + else: + self._fail_reason = not_group_msg + return False + + actual_exceptions: Sequence[BaseException] = exc_val.exceptions + if self.flatten_subgroups: + actual_exceptions = self._unroll_exceptions(actual_exceptions) + + if not self._check_match(exc_val): + self._fail_reason = cast(str, self._fail_reason) + old_reason = self._fail_reason + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + and isinstance(actual := actual_exceptions[0], expected) + and self._check_match(actual) + ): + assert self.match is not None, "can't be None if _check_match failed" + assert self._fail_reason is old_reason is not None + self._fail_reason += f", but matched the expected {self._repr_expected(expected)}. You might want RaisesGroup(Matcher({expected.__name__}, match={_match_pattern(self.match)!r}))" + else: + self._fail_reason = old_reason + return False + + # do the full check on expected exceptions + if not self._check_exceptions( + exc_val, + actual_exceptions, + ): + self._fail_reason = cast(str, self._fail_reason) + assert self._fail_reason is not None + old_reason = self._fail_reason + # if we're not expecting a nested structure, and there is one, do a second + # pass where we try flattening it + if ( + not self.flatten_subgroups + and not any( + isinstance(e, RaisesGroup) for e in self.expected_exceptions + ) + and any(isinstance(e, BaseExceptionGroup) for e in actual_exceptions) + and self._check_exceptions( + exc_val, + self._unroll_exceptions(exc_val.exceptions), + ) + ): + # only indent if it's a single-line reason. In a multi-line there's already + # indented lines that this does not belong to. + indent = " " if "\n" not in self._fail_reason else "" + self._fail_reason = ( + old_reason + + f"\n{indent}Did you mean to use `flatten_subgroups=True`?" + ) + else: + self._fail_reason = old_reason + return False + + # Only run `self.check` once we know `exc_val` is of the correct type. + # TODO: if this fails, we should say the *group* did not match + return self._check_check(exc_val) + + @staticmethod + def _check_expected( + expected_type: ( + type[BaseException] | Matcher[BaseException] | RaisesGroup[BaseException] + ), + exception: BaseException, + ) -> str | None: + """Helper method for `RaisesGroup.matches` and `RaisesGroup._check_exceptions` + to check one of potentially several expected exceptions.""" + if isinstance(expected_type, type): + return _check_raw_type(expected_type, exception) + res = expected_type.matches(exception) + if res: + return None + assert expected_type.fail_reason is not None + if expected_type.fail_reason.startswith("\n"): + return f"\n{expected_type!r}: {indent(expected_type.fail_reason, ' ')}" + return f"{expected_type!r}: {expected_type.fail_reason}" + + @staticmethod + def _repr_expected(e: type[BaseException] | AbstractMatcher[BaseException]) -> str: + """Get the repr of an expected type/Matcher/RaisesGroup, but we only want + the name if it's a type""" + if isinstance(e, type): + return _exception_type_name(e) + return repr(e) + + @overload + def _check_exceptions( + self: RaisesGroup[ExcT_1], + _exc_val: Exception, + actual_exceptions: Sequence[Exception], + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def _check_exceptions( + self: RaisesGroup[BaseExcT_1], + _exc_val: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def _check_exceptions( + self, + _exc_val: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_co]]: + """Helper method for RaisesGroup.matches that attempts to pair up expected and actual exceptions""" + # full table with all results + results = ResultHolder(self.expected_exceptions, actual_exceptions) + + # (indexes of) raised exceptions that haven't (yet) found an expected + remaining_actual = list(range(len(actual_exceptions))) + # (indexes of) expected exceptions that haven't found a matching raised + failed_expected: list[int] = [] + # successful greedy matches + matches: dict[int, int] = {} + + # loop over expected exceptions first to get a more predictable result + for i_exp, expected in enumerate(self.expected_exceptions): + for i_rem in remaining_actual: + res = self._check_expected(expected, actual_exceptions[i_rem]) + results.set_result(i_exp, i_rem, res) + if res is None: + remaining_actual.remove(i_rem) + matches[i_exp] = i_rem + break + else: + failed_expected.append(i_exp) + + # All exceptions matched up successfully + if not remaining_actual and not failed_expected: + return True + + # in case of a single expected and single raised we simplify the output + if 1 == len(actual_exceptions) == len(self.expected_exceptions): + assert not matches + self._fail_reason = res + return False + + # The test case is failing, so we can do a slow and exhaustive check to find + # duplicate matches etc that will be helpful in debugging + for i_exp, expected in enumerate(self.expected_exceptions): + for i_actual, actual in enumerate(actual_exceptions): + if results.has_result(i_exp, i_actual): + continue + results.set_result( + i_exp, i_actual, self._check_expected(expected, actual) + ) + + successful_str = ( + f"{len(matches)} matched exception{'s' if len(matches) > 1 else ''}. " + if matches + else "" + ) + + # all expected were found + if not failed_expected and results.no_match_for_actual(remaining_actual): + self._fail_reason = f"{successful_str}Unexpected exception(s): {[actual_exceptions[i] for i in remaining_actual]!r}" + return False + # all raised exceptions were expected + if not remaining_actual and results.no_match_for_expected(failed_expected): + self._fail_reason = f"{successful_str}Too few exceptions raised, found no match for: [{', '.join(self._repr_expected(self.expected_exceptions[i]) for i in failed_expected)}]" + return False + + # if there's only one remaining and one failed, and the unmatched didn't match anything else, + # we elect to only print why the remaining and the failed didn't match. + if ( + 1 == len(remaining_actual) == len(failed_expected) + and results.no_match_for_actual(remaining_actual) + and results.no_match_for_expected(failed_expected) + ): + self._fail_reason = f"{successful_str}{results.get_result(failed_expected[0], remaining_actual[0])}" + return False + + # there's both expected and raised exceptions without matches + s = "" + if matches: + s += f"\n{successful_str}" + indent_1 = " " * 2 + indent_2 = " " * 4 + + if not remaining_actual: + s += "\nToo few exceptions raised!" + elif not failed_expected: + s += "\nUnexpected exception(s)!" + + if failed_expected: + s += "\nThe following expected exceptions did not find a match:" + rev_matches = {v: k for k, v in matches.items()} + for i_failed in failed_expected: + s += ( + f"\n{indent_1}{self._repr_expected(self.expected_exceptions[i_failed])}" + ) + for i_actual, actual in enumerate(actual_exceptions): + if results.get_result(i_exp, i_actual) is None: + # we print full repr of match target + s += f"\n{indent_2}It matches {actual!r} which was paired with {self._repr_expected(self.expected_exceptions[rev_matches[i_actual]])}" + + if remaining_actual: + s += "\nThe following raised exceptions did not find a match" + for i_actual in remaining_actual: + s += f"\n{indent_1}{actual_exceptions[i_actual]!r}:" + for i_exp, expected in enumerate(self.expected_exceptions): + res = results.get_result(i_exp, i_actual) + if i_exp in failed_expected: + assert res is not None + if res[0] != "\n": + s += "\n" + s += indent(res, indent_2) + if res is None: + # we print full repr of match target + s += f"\n{indent_2}It matches {self._repr_expected(expected)} which was paired with {actual_exceptions[matches[i_exp]]!r}" + + if len(self.expected_exceptions) == len(actual_exceptions) and possible_match( + results + ): + s += "\nThere exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `Matcher` etc so the greedy algorithm can function." + self._fail_reason = s + return False + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> bool: + __tracebackhide__ = True + # assert vs fail ... probably a thing everywhere? + assert exc_type is not None, ( + f"DID NOT RAISE any exception, expected {self.expected_type()}" + ) + assert self.excinfo is not None, ( + "Internal error - should have been constructed in __enter__" + ) + + group_str = ( + "(group)" + if self.allow_unwrapped and not issubclass(exc_type, BaseExceptionGroup) + else "group" + ) + + assert self.matches( + exc_val, + ), f"Raised exception {group_str} did not match: {self._fail_reason}" + + # Cast to narrow the exception type now that it's verified. + exc_info = cast( + "tuple[type[BaseExceptionGroup[BaseExcT_co]], BaseExceptionGroup[BaseExcT_co], types.TracebackType]", + (exc_type, exc_val, exc_tb), + ) + self.excinfo.fill_unfilled(exc_info) + return True + + def expected_type(self) -> str: + subexcs = [] + for e in self.expected_exceptions: + if isinstance(e, Matcher): + subexcs.append(str(e)) + elif isinstance(e, RaisesGroup): + subexcs.append(e.expected_type()) + elif isinstance(e, type): + subexcs.append(e.__name__) + else: # pragma: no cover + raise AssertionError("unknown type") + group_type = "Base" if self.is_baseexceptiongroup else "" + return f"{group_type}ExceptionGroup({', '.join(subexcs)})" + + +@final +class NotChecked: + """Singleton for null values in ResultHolder""" + + +class ResultHolder: + """Container for results of checking exceptions. + Used in RaisesGroup._check_exceptions and possible_match. + """ + + def __init__( + self, + expected_exceptions: tuple[ + type[BaseException] | AbstractMatcher[BaseException], ... + ], + actual_exceptions: Sequence[BaseException], + ) -> None: + self.results: list[list[str | type[NotChecked] | None]] = [ + [NotChecked for _ in expected_exceptions] for _ in actual_exceptions + ] + + def set_result(self, expected: int, actual: int, result: str | None) -> None: + self.results[actual][expected] = result + + def get_result(self, expected: int, actual: int) -> str | None: + res = self.results[actual][expected] + assert res is not NotChecked + # mypy doesn't support identity checking against anything but None + return res # type: ignore[return-value] + + def has_result(self, expected: int, actual: int) -> bool: + return self.results[actual][expected] is not NotChecked + + def no_match_for_expected(self, expected: list[int]) -> bool: + for i in expected: + for actual_results in self.results: + assert actual_results[i] is not NotChecked + if actual_results[i] is None: + return False + return True + + def no_match_for_actual(self, actual: list[int]) -> bool: + for i in actual: + for res in self.results[i]: + assert res is not NotChecked + if res is None: + return False + return True + + +def possible_match(results: ResultHolder, used: set[int] | None = None) -> bool: + if used is None: + used = set() + curr_row = len(used) + if curr_row == len(results.results): + return True + + for i, val in enumerate(results.results[curr_row]): + if val is None and i not in used and possible_match(results, used | {i}): + return True + return False diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index c52757a62b7..58ce0056d2c 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -33,7 +33,6 @@ if TYPE_CHECKING: from numpy import ndarray - from typing_extensions import TypeGuard if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup @@ -1030,110 +1029,6 @@ def validate_exc(exc: type[E]) -> type[E]: raises.Exception = fail.Exception # type: ignore -@final -class RaisesGroup( - AbstractContextManager[_pytest._code.ExceptionInfo[BaseExceptionGroup[E]]] -): - """Helper for catching exceptions wrapped in an ExceptionGroup. - - Similar to pytest.raises, except: - * It requires that the exception is inside an exceptiongroup - * It is only able to be used as a contextmanager - * Due to the above, is not split into a caller function and a cm class - Similar to trio.RaisesGroup, except: - * does not handle multiple levels of nested groups. - * does not have trio.Matcher, to add matching on the sub-exception - * does not handle multiple exceptions in the exceptiongroup. - - TODO: copy over docstring example usage from trio.RaisesGroup - """ - - def __init__( - self, - exception: type[E], - check: Callable[[BaseExceptionGroup[E]], bool] | None = None, - ): - # copied from raises() above - if not isinstance(exception, type) or not issubclass(exception, BaseException): - msg = "expected exception must be a BaseException type, not {}" # type: ignore[unreachable] - not_a = ( - exception.__name__ - if isinstance(exception, type) - else type(exception).__name__ - ) - raise TypeError(msg.format(not_a)) - - self.exception = exception - self.check = check - - def __enter__(self) -> _pytest._code.ExceptionInfo[BaseExceptionGroup[E]]: - self.excinfo: _pytest._code.ExceptionInfo[BaseExceptionGroup[E]] = ( - _pytest._code.ExceptionInfo.for_later() - ) - return self.excinfo - - def matches( - self, - exc_val: BaseException | None, - ) -> TypeGuard[BaseExceptionGroup[E]]: - return ( - exc_val is not None - and isinstance(exc_val, BaseExceptionGroup) - and len(exc_val.exceptions) == 1 - and isinstance(exc_val.exceptions[0], self.exception) - and (self.check is None or self.check(exc_val)) - ) - - def assert_matches( - self, - exc_val: BaseException | None, - ) -> TypeGuard[BaseExceptionGroup[E]]: - assert exc_val is not None, ( - "Internal Error: exc_type is not None but exc_val is" - ) - assert isinstance(exc_val, BaseExceptionGroup), ( - f"Expected an ExceptionGroup, not {type(exc_val)}" - ) - assert len(exc_val.exceptions) == 1, ( - f"Wrong number of exceptions: got {len(exc_val.exceptions)}, expected 1." - ) - assert isinstance(exc_val.exceptions[0], self.exception), ( - f"Wrong type in group: got {type(exc_val.exceptions[0])}, expected {self.exception}" - ) - if self.check is not None: - assert self.check(exc_val), f"Check failed on {exc_val!r}." - - return True - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - __tracebackhide__ = True - if exc_type is None: - fail("DID NOT RAISE ANY EXCEPTION, expected " + self.expected_type()) - assert self.excinfo is not None, "__exit__ without __enter__" - - self.assert_matches(exc_val) - - # Cast to narrow the exception type now that it's verified. - exc_info = cast( - tuple[type[BaseExceptionGroup[E]], BaseExceptionGroup[E], TracebackType], - (exc_type, exc_val, exc_tb), - ) - self.excinfo.fill_unfilled(exc_info) - return True - - def expected_type(self) -> str: - if not issubclass(self.exception, Exception): - base = "Base" - else: - base = "" - return f"{base}ExceptionGroup({self.exception})" - - @final class RaisesContext(AbstractContextManager[_pytest._code.ExceptionInfo[E]]): def __init__( diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index d84fd0cbb1f..e5ef2e440c2 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -6,6 +6,8 @@ from _pytest import __version__ from _pytest import version_tuple from _pytest._code import ExceptionInfo +from _pytest._raises_group import Matcher +from _pytest._raises_group import RaisesGroup from _pytest.assertion import register_assert_rewrite from _pytest.cacheprovider import Cache from _pytest.capture import CaptureFixture @@ -60,7 +62,6 @@ from _pytest.python import Package from _pytest.python_api import approx from _pytest.python_api import raises -from _pytest.python_api import RaisesGroup from _pytest.recwarn import deprecated_call from _pytest.recwarn import WarningsRecorder from _pytest.recwarn import warns diff --git a/testing/python/raises_group.py b/testing/python/raises_group.py index b6edf3d82ed..7d4aa6e471d 100644 --- a/testing/python/raises_group.py +++ b/testing/python/raises_group.py @@ -1,149 +1,1105 @@ +from __future__ import annotations + import re import sys from typing import TYPE_CHECKING +from _pytest._raises_group import Matcher +from _pytest._raises_group import RaisesGroup +from _pytest._raises_group import repr_callable import pytest -from _pytest.outcomes import Failed -from pytest import RaisesGroup -if TYPE_CHECKING: - from typing_extensions import assert_type if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup from exceptiongroup import ExceptionGroup +if TYPE_CHECKING: + from _pytest.python_api import RaisesContext + + +def wrap_escape(s: str) -> str: + return "^" + re.escape(s) + "$" + + +def fails_raises_group( + msg: str, add_prefix: bool = True +) -> RaisesContext[AssertionError]: + assert msg[-1] != "\n", ( + "developer error, expected string should not end with newline" + ) + prefix = "Raised exception group did not match: " if add_prefix else "" + return pytest.raises(AssertionError, match=wrap_escape(prefix + msg)) + def test_raises_group() -> None: - # wrong type to constructor with pytest.raises( - TypeError, - match="^expected exception must be a BaseException type, not ValueError$", + ValueError, + match=wrap_escape( + f'Invalid argument "{TypeError()!r}" must be exception type, Matcher, or RaisesGroup.', + ), ): - RaisesGroup(ValueError()) # type: ignore[arg-type] - - # working example + RaisesGroup(TypeError()) # type: ignore[call-overload] with RaisesGroup(ValueError): raise ExceptionGroup("foo", (ValueError(),)) - with RaisesGroup(ValueError, check=lambda x: True): - raise ExceptionGroup("foo", (ValueError(),)) + with ( + fails_raises_group("'SyntaxError' is not of type 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("foo", (SyntaxError(),)) - # wrong subexception - with pytest.raises( - AssertionError, - match="Wrong type in group: got , expected ", + # multiple exceptions + with RaisesGroup(ValueError, SyntaxError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # order doesn't matter + with RaisesGroup(SyntaxError, ValueError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # nested exceptions + with RaisesGroup(RaisesGroup(ValueError)): + raise ExceptionGroup("foo", (ExceptionGroup("bar", (ValueError(),)),)) + + with RaisesGroup( + SyntaxError, + RaisesGroup(ValueError), + RaisesGroup(RuntimeError), ): - with RaisesGroup(ValueError): - raise ExceptionGroup("foo", (SyntaxError(),)) + raise ExceptionGroup( + "foo", + ( + SyntaxError(), + ExceptionGroup("bar", (ValueError(),)), + ExceptionGroup("", (RuntimeError(),)), + ), + ) - # will error if there's excess exceptions - with pytest.raises( - AssertionError, match="Wrong number of exceptions: got 2, expected 1" + +def test_incorrect_number_exceptions() -> None: + # We previously gave an error saying the number of exceptions was wrong, + # but we now instead indicate excess/missing exceptions + with ( + fails_raises_group( + "1 matched exception. Unexpected exception(s): [RuntimeError()]" + ), + RaisesGroup(ValueError), ): - with RaisesGroup(ValueError): - raise ExceptionGroup("", (ValueError(), ValueError())) + raise ExceptionGroup("", (RuntimeError(), ValueError())) - # double nested exceptions is not (currently) supported (contrary to expect*) - with pytest.raises( - AssertionError, - match="Wrong type in group: got , expected ", + # will error if there's missing exceptions + with ( + fails_raises_group( + "1 matched exception. Too few exceptions raised, found no match for: ['SyntaxError']" + ), + RaisesGroup(ValueError, SyntaxError), ): - with RaisesGroup(ValueError): - raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + raise ExceptionGroup("", (ValueError(),)) - # you'd need to write - with RaisesGroup(ExceptionGroup) as excinfo: + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Too few exceptions raised!\n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Unexpected exception(s)!\n" + "The following raised exceptions did not find a match\n" + " ValueError():\n" + " It matches 'ValueError' which was paired with ValueError()" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (ValueError(), ValueError())) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " SyntaxError():\n" + " 'SyntaxError' is not of type 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), SyntaxError()]) + + +def test_flatten_subgroups() -> None: + # loose semantics, as with expect* + with RaisesGroup(ValueError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + with RaisesGroup(ValueError, TypeError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(), TypeError())),)) + with RaisesGroup(ValueError, TypeError, flatten_subgroups=True): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()]), TypeError()]) + + # mixed loose is possible if you want it to be at least N deep + with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) - RaisesGroup(ValueError).assert_matches(excinfo.value.exceptions[0]) + with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): + raise ExceptionGroup( + "", + (ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)),), + ) - # unwrapped exceptions are not accepted (contrary to expect*) + # but not the other way around with pytest.raises( - AssertionError, match="Expected an ExceptionGroup, not None: + # Catches lone exceptions with strict=False + # just as except* would + with RaisesGroup(ValueError, allow_unwrapped=True): + raise ValueError + # expecting multiple unwrapped exceptions is not possible with pytest.raises( - AssertionError, - match=re.escape("Check failed on ExceptionGroup('foo', (ValueError(),))."), + ValueError, + match=r"^You cannot specify multiple exceptions with", ): - with RaisesGroup(ValueError, check=lambda x: False): - raise ExceptionGroup("foo", (ValueError(),)) + RaisesGroup(SyntaxError, ValueError, allow_unwrapped=True) # type: ignore[call-overload] + # if users want one of several exception types they need to use a Matcher + # (which the error message suggests) + with RaisesGroup( + Matcher(check=lambda e: isinstance(e, (SyntaxError, ValueError))), + allow_unwrapped=True, + ): + raise ValueError + # Unwrapped nested `RaisesGroup` is likely a user error, so we raise an error. + with pytest.raises(ValueError, match="has no effect when expecting"): + RaisesGroup(RaisesGroup(ValueError), allow_unwrapped=True) # type: ignore[call-overload] -def test_RaisesGroup_matches() -> None: - eeg = RaisesGroup(ValueError) - # exc_val is None - assert not eeg.matches(None) - # exc_val is not an exceptiongroup - assert not eeg.matches(ValueError()) - # wrong length - assert not eeg.matches(ExceptionGroup("", (ValueError(), ValueError()))) - # wrong type - assert not eeg.matches(ExceptionGroup("", (TypeError(),))) - # check fails - assert not RaisesGroup(ValueError, check=lambda _: False).matches( - ExceptionGroup("", (ValueError(),)) + # But it *can* be used to check for nesting level +- 1 if they move it to + # the nested RaisesGroup. Users should probably use `Matcher`s instead though. + with RaisesGroup(RaisesGroup(ValueError, allow_unwrapped=True)): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + with RaisesGroup(RaisesGroup(ValueError, allow_unwrapped=True)): + raise ExceptionGroup("", [ValueError()]) + + # with allow_unwrapped=False (default) it will not be caught + with ( + fails_raises_group( + "'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError), + ): + raise ValueError("value error text") + + # allow_unwrapped on its own won't match against nested groups + with ( + fails_raises_group( + "Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise ExceptionGroup("foo", [ExceptionGroup("bar", [ValueError()])]) + + # you need both allow_unwrapped and flatten_subgroups to fully emulate except* + with RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + + # code coverage + with ( + fails_raises_group( + "Raised exception (group) did not match: 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise TypeError("this text doesn't show up in the error message") + with ( + fails_raises_group( + "Raised exception (group) did not match: Matcher(ValueError): 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup(Matcher(ValueError), allow_unwrapped=True), + ): + raise TypeError + + # check we don't suggest unwrapping with nested RaisesGroup + with ( + fails_raises_group("'ValueError' is not an exception group"), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ValueError + + +def test_match() -> None: + # supports match string + with RaisesGroup(ValueError, match="bar"): + raise ExceptionGroup("bar", (ValueError(),)) + + # now also works with ^$ + with RaisesGroup(ValueError, match="^bar$"): + raise ExceptionGroup("bar", (ValueError(),)) + + # it also includes notes + with RaisesGroup(ValueError, match="my note"): + e = ExceptionGroup("bar", (ValueError(),)) + e.add_note("my note") + raise e + + # and technically you can match it all with ^$ + # but you're probably better off using a Matcher at that point + with RaisesGroup(ValueError, match="^bar\nmy note$"): + e = ExceptionGroup("bar", (ValueError(),)) + e.add_note("my note") + raise e + + with ( + fails_raises_group( + "Regex pattern 'foo' did not match 'bar' of 'ExceptionGroup'" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", (ValueError(),)) + + # Suggest a fix for easy pitfall of adding match to the RaisesGroup instead of + # using a Matcher. + # This requires a single expected & raised exception, the expected is a type, + # and `isinstance(raised, expected_type)`. + with ( + fails_raises_group( + "Regex pattern 'foo' did not match 'bar' of 'ExceptionGroup', but matched the expected 'ValueError'. You might want RaisesGroup(Matcher(ValueError, match='foo'))" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", [ValueError("foo")]) + + +def test_check() -> None: + exc = ExceptionGroup("", (ValueError(),)) + + def is_exc(e: ExceptionGroup[ValueError]) -> bool: + return e is exc + + is_exc_repr = repr_callable(is_exc) + with RaisesGroup(ValueError, check=is_exc): + raise exc + + with ( + fails_raises_group(f"check {is_exc_repr} did not return True"), + RaisesGroup(ValueError, check=is_exc), + ): + raise ExceptionGroup("", (ValueError(),)) + + +def test_unwrapped_match_check() -> None: + def my_check(e: object) -> bool: # pragma: no cover + return True + + msg = ( + "`allow_unwrapped=True` bypasses the `match` and `check` parameters" + " if the exception is unwrapped. If you intended to match/check the" + " exception you should use a `Matcher` object. If you want to match/check" + " the exceptiongroup when the exception *is* wrapped you need to" + " do e.g. `if isinstance(exc.value, ExceptionGroup):" + " assert RaisesGroup(...).matches(exc.value)` afterwards." ) - # success - assert eeg.matches(ExceptionGroup("", (ValueError(),))) + with pytest.raises(ValueError, match=re.escape(msg)): + RaisesGroup(ValueError, allow_unwrapped=True, match="foo") # type: ignore[call-overload] + with pytest.raises(ValueError, match=re.escape(msg)): + RaisesGroup(ValueError, allow_unwrapped=True, check=my_check) # type: ignore[call-overload] + + # Users should instead use a Matcher + rg = RaisesGroup(Matcher(ValueError, match="^foo$"), allow_unwrapped=True) + with rg: + raise ValueError("foo") + with rg: + raise ExceptionGroup("", [ValueError("foo")]) + # or if they wanted to match/check the group, do a conditional `.matches()` + with RaisesGroup(ValueError, allow_unwrapped=True) as exc: + raise ExceptionGroup("bar", [ValueError("foo")]) + if isinstance(exc.value, ExceptionGroup): # pragma: no branch + assert RaisesGroup(ValueError, match="bar").matches(exc.value) -def test_RaisesGroup_assert_matches() -> None: - """Check direct use of RaisesGroup.assert_matches, without a context manager""" - eeg = RaisesGroup(ValueError) - with pytest.raises(AssertionError): - eeg.assert_matches(None) - with pytest.raises(AssertionError): - eeg.assert_matches(ValueError()) - eeg.assert_matches(ExceptionGroup("", (ValueError(),))) + +def test_RaisesGroup_matches() -> None: + rg = RaisesGroup(ValueError) + assert not rg.matches(None) + assert not rg.matches(ValueError()) + assert rg.matches(ExceptionGroup("", (ValueError(),))) def test_message() -> None: - with pytest.raises( - Failed, - match=re.escape( - f"DID NOT RAISE ANY EXCEPTION, expected ExceptionGroup({repr(ValueError)})" + def check_message( + message: str, + body: RaisesGroup[BaseException], + ) -> None: + with ( + pytest.raises( + AssertionError, + match=f"^DID NOT RAISE any exception, expected {re.escape(message)}$", + ), + body, + ): + ... + + # basic + check_message("ExceptionGroup(ValueError)", RaisesGroup(ValueError)) + # multiple exceptions + check_message( + "ExceptionGroup(ValueError, ValueError)", + RaisesGroup(ValueError, ValueError), + ) + # nested + check_message( + "ExceptionGroup(ExceptionGroup(ValueError))", + RaisesGroup(RaisesGroup(ValueError)), + ) + + # Matcher + check_message( + "ExceptionGroup(Matcher(ValueError, match='my_str'))", + RaisesGroup(Matcher(ValueError, "my_str")), + ) + check_message( + "ExceptionGroup(Matcher(match='my_str'))", + RaisesGroup(Matcher(match="my_str")), + ) + + # BaseExceptionGroup + check_message( + "BaseExceptionGroup(KeyboardInterrupt)", + RaisesGroup(KeyboardInterrupt), + ) + # BaseExceptionGroup with type inside Matcher + check_message( + "BaseExceptionGroup(Matcher(KeyboardInterrupt))", + RaisesGroup(Matcher(KeyboardInterrupt)), + ) + # Base-ness transfers to parent containers + check_message( + "BaseExceptionGroup(BaseExceptionGroup(KeyboardInterrupt))", + RaisesGroup(RaisesGroup(KeyboardInterrupt)), + ) + # but not to child containers + check_message( + "BaseExceptionGroup(BaseExceptionGroup(KeyboardInterrupt), ExceptionGroup(ValueError))", + RaisesGroup(RaisesGroup(KeyboardInterrupt), RaisesGroup(ValueError)), + ) + + +def test_assert_message() -> None: + # the message does not need to list all parameters to RaisesGroup, nor all exceptions + # in the exception group, as those are both visible in the traceback. + # first fails to match + with ( + fails_raises_group("'TypeError' is not of type 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("a", [TypeError()]) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(ValueError, match='a')\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [RuntimeError()]):\n" + " RaisesGroup(ValueError): 'RuntimeError' is not of type 'ValueError'\n" + " RaisesGroup(ValueError, match='a'): Regex pattern 'a' did not match '' of 'ExceptionGroup'\n" + " RuntimeError():\n" + " RaisesGroup(ValueError): 'RuntimeError' is not an exception group\n" + " RaisesGroup(ValueError, match='a'): 'RuntimeError' is not an exception group", + add_prefix=False, # to see the full structure ), + RaisesGroup(RaisesGroup(ValueError), RaisesGroup(ValueError, match="a")), ): - with RaisesGroup(ValueError): - ... + raise ExceptionGroup( + "", + [ExceptionGroup("", [RuntimeError()]), RuntimeError()], + ) - with pytest.raises( - Failed, - match=re.escape( - f"DID NOT RAISE ANY EXCEPTION, expected BaseExceptionGroup({repr(KeyboardInterrupt)})" + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "2 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(RuntimeError)\n" + " RaisesGroup(ValueError)\n" + "The following raised exceptions did not find a match\n" + " RuntimeError():\n" + # " 'RuntimeError' is not of type 'ValueError'\n" + # " Matcher(TypeError): 'RuntimeError' is not of type 'TypeError'\n" + " RaisesGroup(RuntimeError): 'RuntimeError' is not an exception group, but would match with `allow_unwrapped=True`\n" + " RaisesGroup(ValueError): 'RuntimeError' is not an exception group\n" + " ValueError('bar'):\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " RaisesGroup(RuntimeError): 'ValueError' is not an exception group\n" + " RaisesGroup(ValueError): 'ValueError' is not an exception group, but would match with `allow_unwrapped=True`", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + ValueError, + Matcher(TypeError), + RaisesGroup(RuntimeError), + RaisesGroup(ValueError), ), ): - with RaisesGroup(KeyboardInterrupt): - ... + raise ExceptionGroup( + "a", + [RuntimeError(), TypeError(), ValueError("foo"), ValueError("bar")], + ) + with ( + fails_raises_group( + "1 matched exception. 'AssertionError' is not of type 'TypeError'" + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("a", [ValueError(), AssertionError()]) -if TYPE_CHECKING: + with ( + fails_raises_group( + "Matcher(ValueError): 'TypeError' is not of type 'ValueError'" + ), + RaisesGroup(Matcher(ValueError)), + ): + raise ExceptionGroup("a", [TypeError()]) + + # suggest escaping + with ( + fails_raises_group( + "Raised exception group did not match: Regex pattern 'h(ell)o' did not match 'h(ell)o' of 'ExceptionGroup'\n" + " Did you mean to `re.escape()` the regex?", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, match="h(ell)o"), + ): + raise ExceptionGroup("h(ell)o", [ValueError()]) + with ( + fails_raises_group( + "Matcher(match='h(ell)o'): Regex pattern 'h(ell)o' did not match 'h(ell)o'\n" + " Did you mean to `re.escape()` the regex?", + ), + RaisesGroup(Matcher(match="h(ell)o")), + ): + raise ExceptionGroup("", [ValueError("h(ell)o")]) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " 'ValueError'\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, ValueError, ValueError, ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) - def test_types_1() -> None: - with RaisesGroup(ValueError) as e: - raise ExceptionGroup("foo", (ValueError(),)) - assert_type(e.value, BaseExceptionGroup[ValueError]) - def test_types_2() -> None: - exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup( - "", (ValueError(),) +def test_message_indent() -> None: + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [TypeError(), RuntimeError()]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'\n" + " 'TypeError' is not of type 'ValueError'\n" + " RuntimeError():\n" + " 'RuntimeError' is not of type 'ValueError'\n" + " 'RuntimeError' is not of type 'ValueError'\n" + # TODO: this line is not great, should maybe follow the same format as the other and say + # 'ValueError': Unexpected nested 'ExceptionGroup' (?) + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " TypeError():\n" + " RaisesGroup(ValueError, ValueError): 'TypeError' is not an exception group\n" + " 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ValueError, + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + TypeError(), + ], ) - if RaisesGroup(ValueError).assert_matches(exc): - assert_type(exc, BaseExceptionGroup[ValueError]) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'\n" + " 'TypeError' is not of type 'ValueError'\n" + " RuntimeError():\n" + " 'RuntimeError' is not of type 'ValueError'\n" + " 'RuntimeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + ], + ) + + +def test_suggestion_on_nested_and_brief_error() -> None: + # Make sure "Did you mean" suggestion gets indented iff it follows a single-line error + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError): Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'", + ), + RaisesGroup(RaisesGroup(ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + # if indented here it would look like another raised exception + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'" + ), + RaisesGroup(RaisesGroup(ValueError, ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ValueError(), ExceptionGroup("", [ValueError()])])], + ) + + # re.escape always comes after single-line errors + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(Exception, match='^hello')\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('^hello', [Exception()]):\n" + " RaisesGroup(Exception, match='^hello'): Regex pattern '^hello' did not match '^hello' of 'ExceptionGroup'\n" + " Did you mean to `re.escape()` the regex?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'" + ), + RaisesGroup(RaisesGroup(Exception, match="^hello"), ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("^hello", [Exception()])]) + - def test_types_3() -> None: - e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( - "", (KeyboardInterrupt(),) +def test_assert_message_nested() -> None: + # we only get one instance of aaaaaaaaaa... and bbbbbb..., but we do get multiple instances of ccccc... and dddddd.. + # but I think this now only prints the full repr when that is necessary to disambiguate exceptions + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(RaisesGroup(ValueError))\n" + " RaisesGroup(Matcher(TypeError, match='foo'))\n" + " RaisesGroup(TypeError, ValueError)\n" + "The following raised exceptions did not find a match\n" + " TypeError('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(RaisesGroup(ValueError)): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): 'TypeError' is not an exception group\n" + " RaisesGroup(TypeError, ValueError): 'TypeError' is not an exception group\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')]):\n" + " RaisesGroup(ValueError): 'TypeError' is not of type 'ValueError'\n" + " RaisesGroup(RaisesGroup(ValueError)): RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'\n" + " RaisesGroup(TypeError, ValueError): 1 matched exception. Too few exceptions raised, found no match for: ['ValueError']\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('cccccccccccccccccccccccccccccccccccccccc'), TypeError('dddddddddddddddddddddddddddddddddddddddd')]):\n" + " RaisesGroup(ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " 'TypeError' is not of type 'ValueError'\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " 'TypeError' is not of type 'ValueError'\n" + " RaisesGroup(RaisesGroup(ValueError)): \n" + " The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): \n" + " The following expected exceptions did not find a match:\n" + " Matcher(TypeError, match='foo')\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'cccccccccccccccccccccccccccccccccccccccc'\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'dddddddddddddddddddddddddddddddddddddddd'\n" + " RaisesGroup(TypeError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " It matches 'TypeError' which was paired with TypeError('cccccccccccccccccccccccccccccccccccccccc')\n" + " 'TypeError' is not of type 'ValueError'", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + RaisesGroup(ValueError), + RaisesGroup(RaisesGroup(ValueError)), + RaisesGroup(Matcher(TypeError, match="foo")), + RaisesGroup(TypeError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + TypeError("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ExceptionGroup( + "Exceptions from Trio nursery", + [TypeError("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")], + ), + ExceptionGroup( + "Exceptions from Trio nursery", + [ + TypeError("cccccccccccccccccccccccccccccccccccccccc"), + TypeError("dddddddddddddddddddddddddddddddddddddddd"), + ], + ), + ], + ) + + +@pytest.mark.skipif( + "hypothesis" in sys.modules, + reason="hypothesis may have monkeypatched _check_repr", +) +def test_check_no_patched_repr() -> None: + # We make `_check_repr` monkeypatchable to avoid this very ugly and verbose + # repr. The other tests that use `check` make use of `_check_repr` so they'll + # continue passing in case it is patched - but we have this one test that + # demonstrates just how nasty it gets otherwise. + with ( + pytest.raises( + AssertionError, + match=( + r"^Raised exception group did not match: \n" + r"The following expected exceptions did not find a match:\n" + r" Matcher\(check=. at .*>\)\n" + r" 'TypeError'\n" + r"The following raised exceptions did not find a match\n" + r" ValueError\('foo'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'\n" + r" ValueError\('bar'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'$" + ), + ), + RaisesGroup(Matcher(check=lambda x: False), TypeError), + ): + raise ExceptionGroup("", [ValueError("foo"), ValueError("bar")]) + + +def test_misordering_example() -> None: + with ( + fails_raises_group( + "\n" + "3 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " Matcher(ValueError, match='foo')\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ValueError('bar'):\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " Matcher(ValueError, match='foo'): Regex pattern 'foo' did not match 'bar'\n" + "There exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `Matcher` etc so the greedy algorithm can function." + ), + RaisesGroup( + ValueError, ValueError, ValueError, Matcher(ValueError, match="foo") + ), + ): + raise ExceptionGroup( + "", + [ + ValueError("foo"), + ValueError("foo"), + ValueError("foo"), + ValueError("bar"), + ], ) - if RaisesGroup(ValueError).matches(e): - assert_type(e, BaseExceptionGroup[ValueError]) - def test_types_4() -> None: - e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( - "", (KeyboardInterrupt(),) + +def test_brief_error_on_one_fail() -> None: + """If only one raised and one expected fail to match up, we print a full table iff + the raised exception would match one of the expected that previously got matched""" + # no also-matched + with ( + fails_raises_group( + "1 matched exception. 'TypeError' is not of type 'RuntimeError'" + ), + RaisesGroup(ValueError, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # raised would match an expected + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'RuntimeError'\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " It matches 'Exception' which was paired with ValueError()\n" + " 'TypeError' is not of type 'RuntimeError'" + ), + RaisesGroup(Exception, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # expected would match a raised + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + +def test_identity_oopsies() -> None: + # it's both possible to have several instances of the same exception in the same group + # and to expect multiple of the same type + # this previously messed up the logic + + with ( + fails_raises_group( + "3 matched exceptions. 'RuntimeError' is not of type 'TypeError'" + ), + RaisesGroup(ValueError, ValueError, ValueError, TypeError), + ): + raise ExceptionGroup( + "", [ValueError(), ValueError(), ValueError(), RuntimeError()] ) - # not currently possible: https://github.com/python/typing/issues/930 - RaisesGroup(ValueError).assert_matches(e) - assert_type(e, BaseExceptionGroup[ValueError]) # type: ignore[assert-type] + + e = ValueError("foo") + m = Matcher(match="bar") + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " Matcher(match='bar')\n" + " Matcher(match='bar')\n" + " Matcher(match='bar')\n" + "The following raised exceptions did not find a match\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'" + ), + RaisesGroup(m, m, m), + ): + raise ExceptionGroup("", [e, e, e]) + + +def test_matcher() -> None: + with pytest.raises( + ValueError, + match=r"^You must specify at least one parameter to match on.$", + ): + Matcher() # type: ignore[call-overload] + with pytest.raises( + ValueError, + match=f"^exception_type {re.escape(repr(object))} must be a subclass of BaseException$", + ): + Matcher(object) # type: ignore[type-var] + + with RaisesGroup(Matcher(ValueError)): + raise ExceptionGroup("", (ValueError(),)) + with ( + fails_raises_group( + "Matcher(TypeError): 'ValueError' is not of type 'TypeError'" + ), + RaisesGroup(Matcher(TypeError)), + ): + raise ExceptionGroup("", (ValueError(),)) + + +def test_matcher_match() -> None: + with RaisesGroup(Matcher(ValueError, "foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + with ( + fails_raises_group( + "Matcher(ValueError, match='foo'): Regex pattern 'foo' did not match 'bar'" + ), + RaisesGroup(Matcher(ValueError, "foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) + + # Can be used without specifying the type + with RaisesGroup(Matcher(match="foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + with ( + fails_raises_group( + "Matcher(match='foo'): Regex pattern 'foo' did not match 'bar'" + ), + RaisesGroup(Matcher(match="foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) + + # check ^$ + with RaisesGroup(Matcher(ValueError, match="^bar$")): + raise ExceptionGroup("", [ValueError("bar")]) + with ( + fails_raises_group( + "Matcher(ValueError, match='^bar$'): Regex pattern '^bar$' did not match 'barr'" + ), + RaisesGroup(Matcher(ValueError, match="^bar$")), + ): + raise ExceptionGroup("", [ValueError("barr")]) + + +def test_Matcher_check() -> None: + def check_oserror_and_errno_is_5(e: BaseException) -> bool: + return isinstance(e, OSError) and e.errno == 5 + + with RaisesGroup(Matcher(check=check_oserror_and_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # specifying exception_type narrows the parameter type to the callable + def check_errno_is_5(e: OSError) -> bool: + return e.errno == 5 + + with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # avoid printing overly verbose repr multiple times + with ( + fails_raises_group( + f"Matcher(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(Matcher(OSError, check=check_errno_is_5)), + ): + raise ExceptionGroup("", (OSError(6, ""),)) + + # in nested cases you still get it multiple times though + # to address this you'd need logic in Matcher.__repr__ and RaisesGroup.__repr__ + with ( + fails_raises_group( + f"RaisesGroup(Matcher(OSError, check={check_errno_is_5!r})): Matcher(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(RaisesGroup(Matcher(OSError, check=check_errno_is_5))), + ): + raise ExceptionGroup("", [ExceptionGroup("", [OSError(6, "")])]) + + +def test_matcher_tostring() -> None: + assert str(Matcher(ValueError)) == "Matcher(ValueError)" + assert str(Matcher(match="[a-z]")) == "Matcher(match='[a-z]')" + pattern_no_flags = re.compile(r"noflag", 0) + assert str(Matcher(match=pattern_no_flags)) == "Matcher(match='noflag')" + pattern_flags = re.compile(r"noflag", re.IGNORECASE) + assert str(Matcher(match=pattern_flags)) == f"Matcher(match={pattern_flags!r})" + assert ( + str(Matcher(ValueError, match="re", check=bool)) + == f"Matcher(ValueError, match='re', check={bool!r})" + ) + + +def test_raisesgroup_tostring() -> None: + def check_str_and_repr(s: str) -> None: + evaled = eval(s) + assert s == str(evaled) == repr(evaled) + + check_str_and_repr("RaisesGroup(ValueError)") + check_str_and_repr("RaisesGroup(RaisesGroup(ValueError))") + check_str_and_repr("RaisesGroup(Matcher(ValueError))") + check_str_and_repr("RaisesGroup(ValueError, allow_unwrapped=True)") + check_str_and_repr("RaisesGroup(ValueError, match='aoeu')") + + assert ( + str(RaisesGroup(ValueError, match="[a-z]", check=bool)) + == f"RaisesGroup(ValueError, match='[a-z]', check={bool!r})" + ) diff --git a/testing/python/typing_raises_group.py b/testing/python/typing_raises_group.py new file mode 100644 index 00000000000..2dc35031dac --- /dev/null +++ b/testing/python/typing_raises_group.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import sys +from typing import Callable +from typing import Union + +from typing_extensions import assert_type + +from _pytest._raises_group import Matcher +from _pytest._raises_group import RaisesGroup + + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + +# split into functions to isolate the different scopes + + +def check_matcher_typevar_default(e: Matcher) -> None: + assert e.exception_type is not None + _exc: type[BaseException] = e.exception_type + # this would previously pass, as the type would be `Any` + e.exception_type().blah() # type: ignore + + +def check_basic_contextmanager() -> None: + with RaisesGroup(ValueError) as e: + raise ExceptionGroup("foo", (ValueError(),)) + assert_type(e.value, ExceptionGroup[ValueError]) + + +def check_basic_matches() -> None: + # check that matches gets rid of the naked ValueError in the union + exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup("", (ValueError(),)) + if RaisesGroup(ValueError).matches(exc): + assert_type(exc, ExceptionGroup[ValueError]) + + # also check that BaseExceptionGroup shows up for BaseExceptions + if RaisesGroup(KeyboardInterrupt).matches(exc): + assert_type(exc, BaseExceptionGroup[KeyboardInterrupt]) + + +def check_matches_with_different_exception_type() -> None: + e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( + "", + (KeyboardInterrupt(),), + ) + + # note: it might be tempting to have this warn. + # however, that isn't possible with current typing + if RaisesGroup(ValueError).matches(e): + assert_type(e, ExceptionGroup[ValueError]) + + +def check_matcher_init() -> None: + def check_exc(exc: BaseException) -> bool: + return isinstance(exc, ValueError) + + # Check various combinations of constructor signatures. + # At least 1 arg must be provided. + Matcher() # type: ignore + Matcher(ValueError) + Matcher(ValueError, "regex") + Matcher(ValueError, "regex", check_exc) + Matcher(exception_type=ValueError) + Matcher(match="regex") + Matcher(check=check_exc) + Matcher(ValueError, match="regex") + Matcher(match="regex", check=check_exc) + + def check_filenotfound(exc: FileNotFoundError) -> bool: + return not exc.filename.endswith(".tmp") + + # If exception_type is provided, that narrows the `check` method's argument. + Matcher(FileNotFoundError, check=check_filenotfound) + Matcher(ValueError, check=check_filenotfound) # type: ignore + Matcher(check=check_filenotfound) # type: ignore + Matcher(FileNotFoundError, match="regex", check=check_filenotfound) + + +def raisesgroup_check_type_narrowing() -> None: + """Check type narrowing on the `check` argument to `RaisesGroup`. + All `type: ignore`s are correctly pointing out type errors. + """ + + def handle_exc(e: BaseExceptionGroup[BaseException]) -> bool: + return True + + def handle_kbi(e: BaseExceptionGroup[KeyboardInterrupt]) -> bool: + return True + + def handle_value(e: BaseExceptionGroup[ValueError]) -> bool: + return True + + RaisesGroup(BaseException, check=handle_exc) + RaisesGroup(BaseException, check=handle_kbi) # type: ignore + + RaisesGroup(Exception, check=handle_exc) + RaisesGroup(Exception, check=handle_value) # type: ignore + + RaisesGroup(KeyboardInterrupt, check=handle_exc) + RaisesGroup(KeyboardInterrupt, check=handle_kbi) + RaisesGroup(KeyboardInterrupt, check=handle_value) # type: ignore + + RaisesGroup(ValueError, check=handle_exc) + RaisesGroup(ValueError, check=handle_kbi) # type: ignore + RaisesGroup(ValueError, check=handle_value) + + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_exc) + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_kbi) # type: ignore + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_value) # type: ignore + + +def raisesgroup_narrow_baseexceptiongroup() -> None: + """Check type narrowing specifically for the container exceptiongroup.""" + + def handle_group(e: ExceptionGroup[Exception]) -> bool: + return True + + def handle_group_value(e: ExceptionGroup[ValueError]) -> bool: + return True + + RaisesGroup(ValueError, check=handle_group_value) + + RaisesGroup(Exception, check=handle_group) + + +def check_matcher_transparent() -> None: + with RaisesGroup(Matcher(ValueError)) as e: + ... + _: BaseExceptionGroup[ValueError] = e.value + assert_type(e.value, ExceptionGroup[ValueError]) + + +def check_nested_raisesgroups_contextmanager() -> None: + with RaisesGroup(RaisesGroup(ValueError)) as excinfo: + raise ExceptionGroup("foo", (ValueError(),)) + + _: BaseExceptionGroup[BaseExceptionGroup[ValueError]] = excinfo.value + + assert_type( + excinfo.value, + ExceptionGroup[ExceptionGroup[ValueError]], + ) + + assert_type( + excinfo.value.exceptions[0], + # this union is because of how typeshed defines .exceptions + Union[ + ExceptionGroup[ValueError], + ExceptionGroup[ExceptionGroup[ValueError]], + ], + ) + + +def check_nested_raisesgroups_matches() -> None: + """Check nested RaisesGroups with .matches""" + exc: ExceptionGroup[ExceptionGroup[ValueError]] = ExceptionGroup( + "", + (ExceptionGroup("", (ValueError(),)),), + ) + + if RaisesGroup(RaisesGroup(ValueError)).matches(exc): + assert_type(exc, ExceptionGroup[ExceptionGroup[ValueError]]) + + +def check_multiple_exceptions_1() -> None: + a = RaisesGroup(ValueError, ValueError) + b = RaisesGroup(Matcher(ValueError), Matcher(ValueError)) + c = RaisesGroup(ValueError, Matcher(ValueError)) + + d: RaisesGroup[ValueError] + d = a + d = b + d = c + assert d + + +def check_multiple_exceptions_2() -> None: + # This previously failed due to lack of covariance in the TypeVar + a = RaisesGroup(Matcher(ValueError), Matcher(TypeError)) + b = RaisesGroup(Matcher(ValueError), TypeError) + c = RaisesGroup(ValueError, TypeError) + + d: RaisesGroup[Exception] + d = a + d = b + d = c + assert d + + +def check_raisesgroup_overloads() -> None: + # allow_unwrapped=True does not allow: + # multiple exceptions + RaisesGroup(ValueError, TypeError, allow_unwrapped=True) # type: ignore + # nested RaisesGroup + RaisesGroup(RaisesGroup(ValueError), allow_unwrapped=True) # type: ignore + # specifying match + RaisesGroup(ValueError, match="foo", allow_unwrapped=True) # type: ignore + # specifying check + RaisesGroup(ValueError, check=bool, allow_unwrapped=True) # type: ignore + # allowed variants + RaisesGroup(ValueError, allow_unwrapped=True) + RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True) + RaisesGroup(Matcher(ValueError), allow_unwrapped=True) + + # flatten_subgroups=True does not allow nested RaisesGroup + RaisesGroup(RaisesGroup(ValueError), flatten_subgroups=True) # type: ignore + # but rest is plenty fine + RaisesGroup(ValueError, TypeError, flatten_subgroups=True) + RaisesGroup(ValueError, match="foo", flatten_subgroups=True) + RaisesGroup(ValueError, check=bool, flatten_subgroups=True) + RaisesGroup(ValueError, flatten_subgroups=True) + RaisesGroup(Matcher(ValueError), flatten_subgroups=True) + + # if they're both false we can of course specify nested raisesgroup + RaisesGroup(RaisesGroup(ValueError)) + + +def check_triple_nested_raisesgroup() -> None: + with RaisesGroup(RaisesGroup(RaisesGroup(ValueError))) as e: + assert_type(e.value, ExceptionGroup[ExceptionGroup[ExceptionGroup[ValueError]]]) + + +def check_check_typing() -> None: + # `BaseExceptiongroup` should perhaps be `ExceptionGroup`, but close enough + assert_type( + RaisesGroup(ValueError).check, + Union[ + Callable[[BaseExceptionGroup[ValueError]], bool], + None, + ], + ) From e60d2ee5b8eb5b69f2b87f9cdae64d1b9d0acf8f Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 29 Jan 2025 15:39:36 +0100 Subject: [PATCH 6/9] fix (and ignore) E501 --- src/_pytest/_raises_group.py | 71 ++++++++++++++++++++++++++-------- src/pytest/__init__.py | 1 + testing/python/raises_group.py | 39 ++++++++++--------- 3 files changed, 77 insertions(+), 34 deletions(-) diff --git a/src/_pytest/_raises_group.py b/src/_pytest/_raises_group.py index 04106c6ec3c..802a906dcff 100644 --- a/src/_pytest/_raises_group.py +++ b/src/_pytest/_raises_group.py @@ -159,7 +159,10 @@ def _check_match(self, e: BaseException) -> bool: if isinstance(e, BaseExceptionGroup) else "" ) - self._fail_reason = f"Regex pattern {_match_pattern(self.match)!r} did not match {stringified_exception!r}{maybe_specify_type}" + self._fail_reason = ( + f"Regex pattern {_match_pattern(self.match)!r}" + f" did not match {stringified_exception!r}{maybe_specify_type}" + ) if _match_pattern(self.match) == stringified_exception: self._fail_reason += "\n Did you mean to `re.escape()` the regex?" return False @@ -175,8 +178,11 @@ def matches( @final class Matcher(AbstractMatcher[MatchE]): - """Helper class to be used together with RaisesGroups when you want to specify requirements on sub-exceptions. Only specifying the type is redundant, and it's also unnecessary when the type is a nested `RaisesGroup` since it supports the same arguments. - The type is checked with `isinstance`, and does not need to be an exact match. If that is wanted you can use the ``check`` parameter. + """Helper class to be used together with RaisesGroups when you want to specify requirements on sub-exceptions. + Only specifying the type is redundant, and it's also unnecessary when the type is a + nested `RaisesGroup` since it supports the same arguments. + The type is checked with `isinstance`, and does not need to be an exact match. + If that is wanted you can use the ``check`` parameter. :meth:`Matcher.matches` can also be used standalone to check individual exceptions. Examples:: @@ -280,24 +286,36 @@ def _check_type(self, exception: BaseException) -> TypeGuard[MatchE]: class RaisesGroup(AbstractMatcher[BaseExceptionGroup[BaseExcT_co]]): """Contextmanager for checking for an expected `ExceptionGroup`. This works similar to ``pytest.raises``, but allows for specifying the structure of an `ExceptionGroup`. - `ExceptionInfo.group_contains` also tries to handle exception groups, but it is very bad at checking that you *didn't* get exceptions you didn't expect. + `ExceptionInfo.group_contains` also tries to handle exception groups, + but it is very bad at checking that you *didn't* get exceptions you didn't expect. - The catching behaviour differs from :ref:`except* ` in multiple different ways, being much stricter by default. By using ``allow_unwrapped=True`` and ``flatten_subgroups=True`` you can match ``except*`` fully when expecting a single exception. + The catching behaviour differs from :ref:`except* ` in multiple + different ways, being much stricter by default. + By using ``allow_unwrapped=True`` and ``flatten_subgroups=True`` you can match + ``except*`` fully when expecting a single exception. #. All specified exceptions must be present, *and no others*. - * If you expect a variable number of exceptions you need to use ``pytest.raises(ExceptionGroup)`` and manually check the contained exceptions. Consider making use of :func:`Matcher.matches`. + * If you expect a variable number of exceptions you need to use ``pytest.raises(ExceptionGroup)`` and manually + check the contained exceptions. Consider making use of :func:`Matcher.matches`. #. It will only catch exceptions wrapped in an exceptiongroup by default. - * With ``allow_unwrapped=True`` you can specify a single expected exception or `Matcher` and it will match the exception even if it is not inside an `ExceptionGroup`. If you expect one of several different exception types you need to use a `Matcher` object. + * With ``allow_unwrapped=True`` you can specify a single expected exception or `Matcher` and it will match + the exception even if it is not inside an `ExceptionGroup`. + If you expect one of several different exception types you need to use a `Matcher` object. - #. By default it cares about the full structure with nested `ExceptionGroup`'s. You can specify nested `ExceptionGroup`'s by passing `RaisesGroup` objects as expected exceptions. + #. By default it cares about the full structure with nested `ExceptionGroup`'s. You can specify nested + `ExceptionGroup`'s by passing `RaisesGroup` objects as expected exceptions. - * With ``flatten_subgroups=True`` it will "flatten" the raised `ExceptionGroup`, extracting all exceptions inside any nested :class:`ExceptionGroup`, before matching. + * With ``flatten_subgroups=True`` it will "flatten" the raised `ExceptionGroup`, + extracting all exceptions inside any nested :class:`ExceptionGroup`, before matching. - It does not care about the order of the exceptions, so ``RaisesGroups(ValueError, TypeError)`` is equivalent to ``RaisesGroups(TypeError, ValueError)``. + It does not care about the order of the exceptions, so + ``RaisesGroups(ValueError, TypeError)`` + is equivalent to + ``RaisesGroups(TypeError, ValueError)``. Examples:: @@ -636,7 +654,10 @@ def matches( ): assert self.match is not None, "can't be None if _check_match failed" assert self._fail_reason is old_reason is not None - self._fail_reason += f", but matched the expected {self._repr_expected(expected)}. You might want RaisesGroup(Matcher({expected.__name__}, match={_match_pattern(self.match)!r}))" + self._fail_reason += ( + f", but matched the expected {self._repr_expected(expected)}." + f" You might want RaisesGroup(Matcher({expected.__name__}, match={_match_pattern(self.match)!r}))" + ) else: self._fail_reason = old_reason return False @@ -773,11 +794,18 @@ def _check_exceptions( # all expected were found if not failed_expected and results.no_match_for_actual(remaining_actual): - self._fail_reason = f"{successful_str}Unexpected exception(s): {[actual_exceptions[i] for i in remaining_actual]!r}" + self._fail_reason = ( + f"{successful_str}Unexpected exception(s):" + f" {[actual_exceptions[i] for i in remaining_actual]!r}" + ) return False # all raised exceptions were expected if not remaining_actual and results.no_match_for_expected(failed_expected): - self._fail_reason = f"{successful_str}Too few exceptions raised, found no match for: [{', '.join(self._repr_expected(self.expected_exceptions[i]) for i in failed_expected)}]" + no_match_for_str = ", ".join( + self._repr_expected(self.expected_exceptions[i]) + for i in failed_expected + ) + self._fail_reason = f"{successful_str}Too few exceptions raised, found no match for: [{no_match_for_str}]" return False # if there's only one remaining and one failed, and the unmatched didn't match anything else, @@ -812,7 +840,10 @@ def _check_exceptions( for i_actual, actual in enumerate(actual_exceptions): if results.get_result(i_exp, i_actual) is None: # we print full repr of match target - s += f"\n{indent_2}It matches {actual!r} which was paired with {self._repr_expected(self.expected_exceptions[rev_matches[i_actual]])}" + s += ( + f"\n{indent_2}It matches {actual!r} which was " + f"paired with {self._repr_expected(self.expected_exceptions[rev_matches[i_actual]])}" + ) if remaining_actual: s += "\nThe following raised exceptions did not find a match" @@ -827,12 +858,20 @@ def _check_exceptions( s += indent(res, indent_2) if res is None: # we print full repr of match target - s += f"\n{indent_2}It matches {self._repr_expected(expected)} which was paired with {actual_exceptions[matches[i_exp]]!r}" + s += ( + f"\n{indent_2}It matches {self._repr_expected(expected)} " + f"which was paired with {actual_exceptions[matches[i_exp]]!r}" + ) if len(self.expected_exceptions) == len(actual_exceptions) and possible_match( results ): - s += "\nThere exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `Matcher` etc so the greedy algorithm can function." + s += ( + "\nThere exist a possible match when attempting an exhaustive check," + " but RaisesGroup uses a greedy algorithm. " + "Please make your expected exceptions more stringent with `Matcher` etc" + " so the greedy algorithm can function." + ) self._fail_reason = s return False diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index e5ef2e440c2..fede4a77501 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -115,6 +115,7 @@ "Mark", "MarkDecorator", "MarkGenerator", + "Matcher", "Metafunc", "Module", "MonkeyPatch", diff --git a/testing/python/raises_group.py b/testing/python/raises_group.py index 7d4aa6e471d..0d6d76396eb 100644 --- a/testing/python/raises_group.py +++ b/testing/python/raises_group.py @@ -1,5 +1,8 @@ from __future__ import annotations +# several expected multi-line strings contain long lines. We don't wanna break them up +# as that makes it confusing to see where the line breaks are. +# ruff: noqa: E501 import re import sys from typing import TYPE_CHECKING @@ -761,48 +764,48 @@ def test_assert_message_nested() -> None: " RaisesGroup(Matcher(TypeError, match='foo'))\n" " RaisesGroup(TypeError, ValueError)\n" "The following raised exceptions did not find a match\n" - " TypeError('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'):\n" + " TypeError('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'):\n" " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" " RaisesGroup(RaisesGroup(ValueError)): 'TypeError' is not an exception group\n" " RaisesGroup(Matcher(TypeError, match='foo')): 'TypeError' is not an exception group\n" " RaisesGroup(TypeError, ValueError): 'TypeError' is not an exception group\n" - " ExceptionGroup('Exceptions from Trio nursery', [TypeError('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')]):\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')]):\n" " RaisesGroup(ValueError): 'TypeError' is not of type 'ValueError'\n" " RaisesGroup(RaisesGroup(ValueError)): RaisesGroup(ValueError): 'TypeError' is not an exception group\n" - " RaisesGroup(Matcher(TypeError, match='foo')): Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'\n" + " RaisesGroup(Matcher(TypeError, match='foo')): Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'\n" " RaisesGroup(TypeError, ValueError): 1 matched exception. Too few exceptions raised, found no match for: ['ValueError']\n" - " ExceptionGroup('Exceptions from Trio nursery', [TypeError('cccccccccccccccccccccccccccccccccccccccc'), TypeError('dddddddddddddddddddddddddddddddddddddddd')]):\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('cccccccccccccccccccccccccccccc'), TypeError('dddddddddddddddddddddddddddddd')]):\n" " RaisesGroup(ValueError): \n" " The following expected exceptions did not find a match:\n" " 'ValueError'\n" " The following raised exceptions did not find a match\n" - " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" " 'TypeError' is not of type 'ValueError'\n" - " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" " 'TypeError' is not of type 'ValueError'\n" " RaisesGroup(RaisesGroup(ValueError)): \n" " The following expected exceptions did not find a match:\n" " RaisesGroup(ValueError)\n" " The following raised exceptions did not find a match\n" - " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" - " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" " RaisesGroup(Matcher(TypeError, match='foo')): \n" " The following expected exceptions did not find a match:\n" " Matcher(TypeError, match='foo')\n" " The following raised exceptions did not find a match\n" - " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" - " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'cccccccccccccccccccccccccccccccccccccccc'\n" - " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" - " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'dddddddddddddddddddddddddddddddddddddddd'\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'cccccccccccccccccccccccccccccc'\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'dddddddddddddddddddddddddddddd'\n" " RaisesGroup(TypeError, ValueError): \n" " 1 matched exception. \n" " The following expected exceptions did not find a match:\n" " 'ValueError'\n" " The following raised exceptions did not find a match\n" - " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" - " It matches 'TypeError' which was paired with TypeError('cccccccccccccccccccccccccccccccccccccccc')\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " It matches 'TypeError' which was paired with TypeError('cccccccccccccccccccccccccccccc')\n" " 'TypeError' is not of type 'ValueError'", add_prefix=False, # to see the full structure ), @@ -816,16 +819,16 @@ def test_assert_message_nested() -> None: raise ExceptionGroup( "", [ - TypeError("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + TypeError("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), ExceptionGroup( "Exceptions from Trio nursery", - [TypeError("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")], + [TypeError("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")], ), ExceptionGroup( "Exceptions from Trio nursery", [ - TypeError("cccccccccccccccccccccccccccccccccccccccc"), - TypeError("dddddddddddddddddddddddddddddddddddddddd"), + TypeError("cccccccccccccccccccccccccccccc"), + TypeError("dddddddddddddddddddddddddddddd"), ], ), ], From 08eedec34ba8289bf6cea884f6e43c7b4ab06a65 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 29 Jan 2025 16:24:00 +0100 Subject: [PATCH 7/9] assertions -> Failed, update various comments, only take exc.message if it's a BaseExceptionGroup, ValueError->TypeError on wrong type --- src/_pytest/_raises_group.py | 54 ++++++++++++++++++---------------- src/_pytest/python_api.py | 3 -- testing/python/raises_group.py | 47 ++++++++++++++--------------- 3 files changed, 50 insertions(+), 54 deletions(-) diff --git a/src/_pytest/_raises_group.py b/src/_pytest/_raises_group.py index 802a906dcff..685bc3ff1ac 100644 --- a/src/_pytest/_raises_group.py +++ b/src/_pytest/_raises_group.py @@ -14,13 +14,14 @@ from typing import TYPE_CHECKING from _pytest._code import ExceptionInfo +from _pytest.outcomes import fail if TYPE_CHECKING: - # sphinx will *only* work if we use types.TracebackType, and import - # *inside* TYPE_CHECKING. No other combination works..... from collections.abc import Callable from collections.abc import Sequence + + # for some reason Sphinx does not play well with 'from types import TracebackType' import types from typing_extensions import TypeGuard @@ -50,16 +51,12 @@ from exceptiongroup import ExceptionGroup -# this differs slightly from pytest.ExceptionInfo -# we do `getattr(exc, "message", str(exc)`, they do str(exc) -# this is because we don't want '(x sub-exceptions)' when checking -# exception groups. -# Does it differ in behaviour on any other exceptions? Should we do an -# isinstance check instead? +# this differs slightly from pytest.ExceptionInfo._stringify_exception +# as we don't want '(1 sub-exception)' when matching group strings def _stringify_exception(exc: BaseException) -> str: return "\n".join( [ - getattr(exc, "message", str(exc)), + exc.message if isinstance(exc, BaseExceptionGroup) else str(exc), *getattr(exc, "__notes__", []), ], ) @@ -77,7 +74,7 @@ def _match_pattern(match: Pattern[str]) -> str | Pattern[str]: def repr_callable(fun: Callable[[BaseExcT_1], bool]) -> str: """Get the repr of a ``check`` parameter. - Split out so it can be monkeypatched (e.g. by our hypothesis plugin) + Split out so it can be monkeypatched (e.g. by hypothesis) """ return repr(fun) @@ -127,10 +124,9 @@ def __init__( @property def fail_reason(self) -> str | None: - """Set after a call to `matches` to give a human-readable - reason for why the match failed. + """Set after a call to `matches` to give a human-readable reason for why the match failed. When used as a context manager the string will be given as the text of an - `AssertionError`""" + `Failed`""" return self._fail_reason def _check_check( @@ -198,6 +194,11 @@ class Matcher(AbstractMatcher[MatchE]): readable ``repr``s of ``check`` callables in the output. """ + # Trio bundled hypothesis monkeypatching, we will probably instead assume that + # hypothesis will handle that in their pytest plugin by the time this is released. + # Alternatively we could add a version of get_pretty_function_description ourselves + # https://github.com/HypothesisWorks/hypothesis/blob/8ced2f59f5c7bea3344e35d2d53e1f8f8eb9fcd8/hypothesis-python/src/hypothesis/internal/reflection.py#L439 + # At least one of the three parameters must be passed. @overload def __init__( @@ -229,7 +230,7 @@ def __init__( if exception_type is None and match is None and check is None: raise ValueError("You must specify at least one parameter to match on.") if exception_type is not None and not issubclass(exception_type, BaseException): - raise ValueError( + raise TypeError( f"exception_type {exception_type} must be a subclass of BaseException", ) self.exception_type = exception_type @@ -533,7 +534,7 @@ def __init__( elif isinstance(exc, type) and issubclass(exc, BaseException): self.is_baseexceptiongroup |= not issubclass(exc, Exception) else: - raise ValueError( + raise TypeError( f'Invalid argument "{exc!r}" must be exception type, Matcher, or' " RaisesGroup.", ) @@ -841,8 +842,10 @@ def _check_exceptions( if results.get_result(i_exp, i_actual) is None: # we print full repr of match target s += ( - f"\n{indent_2}It matches {actual!r} which was " - f"paired with {self._repr_expected(self.expected_exceptions[rev_matches[i_actual]])}" + f"\n{indent_2}It matches {actual!r} which was paired with " + + self._repr_expected( + self.expected_exceptions[rev_matches[i_actual]] + ) ) if remaining_actual: @@ -882,10 +885,9 @@ def __exit__( exc_tb: types.TracebackType | None, ) -> bool: __tracebackhide__ = True - # assert vs fail ... probably a thing everywhere? - assert exc_type is not None, ( - f"DID NOT RAISE any exception, expected {self.expected_type()}" - ) + if exc_type is None: + fail(f"DID NOT RAISE any exception, expected {self.expected_type()}") + assert self.excinfo is not None, ( "Internal error - should have been constructed in __enter__" ) @@ -896,11 +898,11 @@ def __exit__( else "group" ) - assert self.matches( - exc_val, - ), f"Raised exception {group_str} did not match: {self._fail_reason}" + if not self.matches(exc_val): + fail(f"Raised exception {group_str} did not match: {self._fail_reason}") - # Cast to narrow the exception type now that it's verified. + # Cast to narrow the exception type now that it's verified.... + # even though the TypeGuard in self.matches should be narrowing exc_info = cast( "tuple[type[BaseExceptionGroup[BaseExcT_co]], BaseExceptionGroup[BaseExcT_co], types.TracebackType]", (exc_type, exc_val, exc_tb), @@ -925,7 +927,7 @@ def expected_type(self) -> str: @final class NotChecked: - """Singleton for null values in ResultHolder""" + """Singleton for unchecked values in ResultHolder""" class ResultHolder: diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index 58ce0056d2c..ddbf9b87251 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -34,9 +34,6 @@ if TYPE_CHECKING: from numpy import ndarray -if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup - def _compare_approx( full_object: object, diff --git a/testing/python/raises_group.py b/testing/python/raises_group.py index 0d6d76396eb..e04074b7ff4 100644 --- a/testing/python/raises_group.py +++ b/testing/python/raises_group.py @@ -10,6 +10,7 @@ from _pytest._raises_group import Matcher from _pytest._raises_group import RaisesGroup from _pytest._raises_group import repr_callable +from _pytest.outcomes import Failed import pytest @@ -25,24 +26,22 @@ def wrap_escape(s: str) -> str: return "^" + re.escape(s) + "$" -def fails_raises_group( - msg: str, add_prefix: bool = True -) -> RaisesContext[AssertionError]: +def fails_raises_group(msg: str, add_prefix: bool = True) -> RaisesContext[Failed]: assert msg[-1] != "\n", ( "developer error, expected string should not end with newline" ) prefix = "Raised exception group did not match: " if add_prefix else "" - return pytest.raises(AssertionError, match=wrap_escape(prefix + msg)) + return pytest.raises(Failed, match=wrap_escape(prefix + msg)) def test_raises_group() -> None: with pytest.raises( - ValueError, + TypeError, match=wrap_escape( - f'Invalid argument "{TypeError()!r}" must be exception type, Matcher, or RaisesGroup.', + f'Invalid argument "{ValueError()!r}" must be exception type, Matcher, or RaisesGroup.', ), ): - RaisesGroup(TypeError()) # type: ignore[call-overload] + RaisesGroup(ValueError()) # type: ignore[call-overload] with RaisesGroup(ValueError): raise ExceptionGroup("foo", (ValueError(),)) @@ -449,7 +448,7 @@ def check_message( ) -> None: with ( pytest.raises( - AssertionError, + Failed, match=f"^DID NOT RAISE any exception, expected {re.escape(message)}$", ), body, @@ -844,23 +843,21 @@ def test_check_no_patched_repr() -> None: # repr. The other tests that use `check` make use of `_check_repr` so they'll # continue passing in case it is patched - but we have this one test that # demonstrates just how nasty it gets otherwise. + match_str = ( + r"^Raised exception group did not match: \n" + r"The following expected exceptions did not find a match:\n" + r" Matcher\(check=. at .*>\)\n" + r" 'TypeError'\n" + r"The following raised exceptions did not find a match\n" + r" ValueError\('foo'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'\n" + r" ValueError\('bar'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'$" + ) with ( - pytest.raises( - AssertionError, - match=( - r"^Raised exception group did not match: \n" - r"The following expected exceptions did not find a match:\n" - r" Matcher\(check=. at .*>\)\n" - r" 'TypeError'\n" - r"The following raised exceptions did not find a match\n" - r" ValueError\('foo'\):\n" - r" Matcher\(check=. at .*>\): check did not return True\n" - r" 'ValueError' is not of type 'TypeError'\n" - r" ValueError\('bar'\):\n" - r" Matcher\(check=. at .*>\): check did not return True\n" - r" 'ValueError' is not of type 'TypeError'$" - ), - ), + pytest.raises(Failed, match=match_str), RaisesGroup(Matcher(check=lambda x: False), TypeError), ): raise ExceptionGroup("", [ValueError("foo"), ValueError("bar")]) @@ -994,7 +991,7 @@ def test_matcher() -> None: ): Matcher() # type: ignore[call-overload] with pytest.raises( - ValueError, + TypeError, match=f"^exception_type {re.escape(repr(object))} must be a subclass of BaseException$", ): Matcher(object) # type: ignore[type-var] From 37aa8e54d4ef90f1e66f3e74c25df381161d0ea1 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 29 Jan 2025 16:29:35 +0100 Subject: [PATCH 8/9] move typing_raises_group to testing/ so it doesn't get executed --- testing/{python => }/typing_raises_group.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename testing/{python => }/typing_raises_group.py (100%) diff --git a/testing/python/typing_raises_group.py b/testing/typing_raises_group.py similarity index 100% rename from testing/python/typing_raises_group.py rename to testing/typing_raises_group.py From aeb5b2888ac54e9496832921fdf0e2691e565803 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Wed, 29 Jan 2025 17:23:30 +0100 Subject: [PATCH 9/9] add changelog. helpful suggestion if check would pass on sub-exception. export raises_group as a convenience alias. --- changelog/11671.feature.rst | 1 + src/_pytest/_raises_group.py | 19 +++++++++++++++++-- src/pytest/__init__.py | 2 ++ testing/python/raises_group.py | 16 +++++++++++++++- 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 changelog/11671.feature.rst diff --git a/changelog/11671.feature.rst b/changelog/11671.feature.rst new file mode 100644 index 00000000000..9e401112ad0 --- /dev/null +++ b/changelog/11671.feature.rst @@ -0,0 +1 @@ +Added `RaisesGroup` (also available as `raises_group`) and `Matcher`, as an equivalent to `raises` for expecting `ExceptionGroup`. It includes the ability to specity multiple different expected exceptions, the structure of nested exception groups, and/or closely emulating `except_star`. diff --git a/src/_pytest/_raises_group.py b/src/_pytest/_raises_group.py index 685bc3ff1ac..68303c4a3fe 100644 --- a/src/_pytest/_raises_group.py +++ b/src/_pytest/_raises_group.py @@ -696,8 +696,23 @@ def matches( return False # Only run `self.check` once we know `exc_val` is of the correct type. - # TODO: if this fails, we should say the *group* did not match - return self._check_check(exc_val) + if not self._check_check(exc_val): + reason = cast(str, self._fail_reason) + f" on the {type(exc_val).__name__}" + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + # we explicitly break typing here :) + and self._check_check(actual_exceptions[0]) # type: ignore[arg-type] + ): + self._fail_reason = reason + ( + f", but did return True for the expected {self._repr_expected(expected)}." + f" You might want RaisesGroup(Matcher({expected.__name__}, check=<...>))" + ) + else: + self._fail_reason = reason + return False + + return True @staticmethod def _check_expected( diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index fede4a77501..ca2c66fee03 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -8,6 +8,7 @@ from _pytest._code import ExceptionInfo from _pytest._raises_group import Matcher from _pytest._raises_group import RaisesGroup +from _pytest._raises_group import RaisesGroup as raises_group from _pytest.assertion import register_assert_rewrite from _pytest.cacheprovider import Cache from _pytest.capture import CaptureFixture @@ -166,6 +167,7 @@ "mark", "param", "raises", + "raises_group", "register_assert_rewrite", "set_trace", "skip", diff --git a/testing/python/raises_group.py b/testing/python/raises_group.py index e04074b7ff4..2ba613ba9b5 100644 --- a/testing/python/raises_group.py +++ b/testing/python/raises_group.py @@ -397,11 +397,25 @@ def is_exc(e: ExceptionGroup[ValueError]) -> bool: raise exc with ( - fails_raises_group(f"check {is_exc_repr} did not return True"), + fails_raises_group( + f"check {is_exc_repr} did not return True on the ExceptionGroup" + ), RaisesGroup(ValueError, check=is_exc), ): raise ExceptionGroup("", (ValueError(),)) + def is_value_error(e: BaseException) -> bool: + return isinstance(e, ValueError) + + # helpful suggestion if the user thinks the check is for the sub-exception + with ( + fails_raises_group( + f"check {is_value_error} did not return True on the ExceptionGroup, but did return True for the expected 'ValueError'. You might want RaisesGroup(Matcher(ValueError, check=<...>))" + ), + RaisesGroup(ValueError, check=is_value_error), + ): + raise ExceptionGroup("", (ValueError(),)) + def test_unwrapped_match_check() -> None: def my_check(e: object) -> bool: # pragma: no cover