From 28f275b10bc812416f9be07b9fc81e81882c5b81 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 29 Jul 2023 09:32:16 +1000 Subject: [PATCH 01/23] Add types to the _util module --- trio/_util.py | 54 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index a87f1fc02c..d49b482307 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -1,7 +1,7 @@ # Little utilities we use internally from __future__ import annotations -import collections +import collections.abc import inspect import os import signal @@ -14,10 +14,23 @@ import trio CallT = t.TypeVar("CallT", bound=t.Callable[..., t.Any]) +T = t.TypeVar("T") +RetT = t.TypeVar("RetT") + +if t.TYPE_CHECKING: + from typing_extensions import ParamSpec, Self + + ArgsT = ParamSpec("ArgsT") + + +if t.TYPE_CHECKING: + + def signal_raise(signum: int) -> None: + ... # Don't check the implementation below, pthread_kill does not exist on Windows. # Equivalent to the C function raise(), which Python doesn't wrap -if os.name == "nt": +elif os.name == "nt": # On Windows, os.kill exists but is really weird. # # If you give it CTRL_C_EVENT or CTRL_BREAK_EVENT, it tries to deliver @@ -61,7 +74,7 @@ signal_raise = getattr(_lib, "raise") else: - def signal_raise(signum): + def signal_raise(signum: int) -> None: signal.pthread_kill(threading.get_ident(), signum) @@ -73,7 +86,7 @@ def signal_raise(signum): # Trying to use signal out of the main thread will fail, so we can then # reliably check if this is the main thread without relying on a # potentially modified threading. -def is_main_thread(): +def is_main_thread() -> bool: """Attempt to reliably check if we are in the main thread.""" try: signal.signal(signal.SIGINT, signal.getsignal(signal.SIGINT)) @@ -86,8 +99,11 @@ def is_main_thread(): # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. Returns coroutine object. ###### -def coroutine_or_error(async_fn, *args): - def _return_value_looks_like_wrong_library(value): +# TODO: Use TypeVarTuple here. +def coroutine_or_error( + async_fn: t.Callable[..., t.Awaitable[RetT]], *args: t.Any +) -> t.Awaitable[RetT]: + def _return_value_looks_like_wrong_library(value: object) -> bool: # Returned by legacy @asyncio.coroutine functions, which includes # a surprising proportion of asyncio builtins. if isinstance(value, collections.abc.Generator): @@ -183,11 +199,11 @@ class ConflictDetector: """ - def __init__(self, msg): + def __init__(self, msg: str): self._msg = msg self._held = False - def __enter__(self): + def __enter__(self) -> None: if self._held: raise trio.BusyResourceError(self._msg) else: @@ -224,10 +240,10 @@ def decorator(func: CallT) -> CallT: return decorator -def fixup_module_metadata(module_name, namespace): - seen_ids = set() +def fixup_module_metadata(module_name: str, namespace: dict[str, t.Any]) -> None: + seen_ids: set[int] = set() - def fix_one(qualname, name, obj): + def fix_one(qualname: str, name: str, obj: t.Any) -> None: # avoid infinite recursion (relevant when using # typing.Generic, for example) if id(obj) in seen_ids: @@ -252,7 +268,10 @@ def fix_one(qualname, name, obj): fix_one(objname, objname, obj) -class generic_function: +# We need ParamSpec to type this "properly", but that requires a runtime typing_extensions import +# to use as a class base. This is only used at runtime and isn't correct for type checkers anyway, +# so don't bother. +class generic_function(t.Generic[RetT]): """Decorator that makes a function indexable, to communicate non-inferrable generic type parameters to a static type checker. @@ -269,14 +288,14 @@ def open_memory_channel(max_buffer_size: int) -> Tuple[ but at least it becomes possible to write those. """ - def __init__(self, fn): + def __init__(self, fn: t.Callable[..., RetT]) -> None: update_wrapper(self, fn) self._fn = fn - def __call__(self, *args, **kwargs): + def __call__(self, *args: t.Any, **kwargs: t.Any) -> RetT: return self._fn(*args, **kwargs) - def __getitem__(self, _): + def __getitem__(self, subscript: object) -> Self: return self @@ -307,9 +326,6 @@ def __new__( return super().__new__(cls, name, bases, cls_namespace) -T = t.TypeVar("T") - - class NoPublicConstructor(Final): """Metaclass that enforces a class to be final (i.e., subclass not allowed) and ensures a private constructor. @@ -338,7 +354,7 @@ def _create(cls: t.Type[T], *args: object, **kwargs: object) -> T: return super().__call__(*args, **kwargs) # type: ignore -def name_asyncgen(agen): +def name_asyncgen(agen: t.Any) -> str: """Return the fully-qualified name of the async generator function that produced the async generator iterator *agen*. """ From 9f526a2fd6b1ecf537ed64b8a105368bb9791ace Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 29 Jul 2023 09:32:31 +1000 Subject: [PATCH 02/23] Add types to _deprecate --- trio/_deprecate.py | 59 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/trio/_deprecate.py b/trio/_deprecate.py index fe00192583..7890da5323 100644 --- a/trio/_deprecate.py +++ b/trio/_deprecate.py @@ -1,10 +1,21 @@ +from __future__ import annotations + import sys import warnings +from collections.abc import Callable from functools import wraps from types import ModuleType +from typing import TYPE_CHECKING, ClassVar, Optional, TypeVar import attr +if TYPE_CHECKING: + from typing_extensions import ParamSpec + + ArgsT = ParamSpec("ArgsT") + +RetT = TypeVar("RetT") + # We want our warnings to be visible by default (at least for now), but we # also want it to be possible to override that using the -W switch. AFAICT @@ -29,17 +40,24 @@ class TrioDeprecationWarning(FutureWarning): """ -def _url_for_issue(issue): +def _url_for_issue(issue: int) -> str: return f"https://github.com/python-trio/trio/issues/{issue}" -def _stringify(thing): +def _stringify(thing: object) -> str: if hasattr(thing, "__module__") and hasattr(thing, "__qualname__"): return f"{thing.__module__}.{thing.__qualname__}" return str(thing) -def warn_deprecated(thing, version, *, issue, instead, stacklevel=2): +def warn_deprecated( + thing: object, + version: str, + *, + issue: int | None, + instead: object, + stacklevel: int = 2, +) -> None: stacklevel += 1 msg = f"{_stringify(thing)} is deprecated since Trio {version}" if instead is None: @@ -53,12 +71,14 @@ def warn_deprecated(thing, version, *, issue, instead, stacklevel=2): # @deprecated("0.2.0", issue=..., instead=...) # def ... -def deprecated(version, *, thing=None, issue, instead): - def do_wrap(fn): +def deprecated( + version: str, *, thing: object = None, issue: int | None, instead: object +) -> Callable[[Callable[ArgsT, RetT]], Callable[ArgsT, RetT]]: + def do_wrap(fn: Callable[ArgsT, RetT]) -> Callable[ArgsT, RetT]: nonlocal thing @wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: warn_deprecated(thing, version, instead=instead, issue=issue) return fn(*args, **kwargs) @@ -87,10 +107,16 @@ def wrapper(*args, **kwargs): return do_wrap -def deprecated_alias(old_qualname, new_fn, version, *, issue): +def deprecated_alias( + old_qualname: str, + new_fn: Callable[ArgsT, RetT], + version: str, + *, + issue: int | None, +) -> Callable[ArgsT, RetT]: @deprecated(version, issue=issue, instead=new_fn) @wraps(new_fn, assigned=("__module__", "__annotations__")) - def wrapper(*args, **kwargs): + def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: "Deprecated alias." return new_fn(*args, **kwargs) @@ -101,16 +127,18 @@ def wrapper(*args, **kwargs): @attr.s(frozen=True) class DeprecatedAttribute: - _not_set = object() + _not_set: ClassVar[object] = object() - value = attr.ib() - version = attr.ib() - issue = attr.ib() - instead = attr.ib(default=_not_set) + value: object = attr.ib() + version: str = attr.ib() + issue: int | None = attr.ib() + instead: object = attr.ib(default=_not_set) class _ModuleWithDeprecations(ModuleType): - def __getattr__(self, name): + __deprecated_attributes__: dict[str, DeprecatedAttribute] + + def __getattr__(self, name: str) -> object: if name in self.__deprecated_attributes__: info = self.__deprecated_attributes__[name] instead = info.instead @@ -124,9 +152,10 @@ def __getattr__(self, name): raise AttributeError(msg.format(self.__name__, name)) -def enable_attribute_deprecations(module_name): +def enable_attribute_deprecations(module_name: str) -> None: module = sys.modules[module_name] module.__class__ = _ModuleWithDeprecations + assert isinstance(module, _ModuleWithDeprecations) # Make sure that this is always defined so that # _ModuleWithDeprecations.__getattr__ can access it without jumping # through hoops or risking infinite recursion. From f23aeebbb50179bceeadf6d7a90bf1e47798d1ea Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 29 Jul 2023 09:25:57 +1000 Subject: [PATCH 03/23] Fix incorrect use of issue parameter --- trio/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/tests.py b/trio/tests.py index 573a076da8..4ffb583a3a 100644 --- a/trio/tests.py +++ b/trio/tests.py @@ -9,7 +9,7 @@ "trio.tests", "0.22.1", instead="trio._tests", - issue="https://github.com/python-trio/trio/issues/274", + issue=274, ) @@ -23,7 +23,7 @@ def __getattr__(self, attr: str) -> Any: f"trio.tests.{attr}", "0.22.1", instead=f"trio._tests.{attr}", - issue="https://github.com/python-trio/trio/issues/274", + issue=274, ) # needed to access e.g. trio._tests.tools, although pytest doesn't need it From fa16c7095a3f5d20301ef58db347c2fb026a1959 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 29 Jul 2023 09:44:17 +1000 Subject: [PATCH 04/23] Add types to _core._ki --- trio/_core/_ki.py | 56 +++++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index cc05ef9177..1e6818585a 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -3,17 +3,21 @@ import inspect import signal import sys +import types from functools import wraps -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable, TypeVar import attr from .._util import is_main_thread +F = TypeVar("F", bound=Callable[..., Any]) +RetT = TypeVar("RetT") + if TYPE_CHECKING: - from typing import Any, Callable, TypeVar + from typing_extensions import ParamSpec, TypeGuard - F = TypeVar("F", bound=Callable[..., Any]) + ArgsT = ParamSpec("ArgsT") # In ordinary single-threaded Python code, when you hit control-C, it raises # an exception and automatically does all the regular unwinding stuff. @@ -85,7 +89,7 @@ # NB: according to the signal.signal docs, 'frame' can be None on entry to # this function: -def ki_protection_enabled(frame): +def ki_protection_enabled(frame: types.FrameType | None) -> bool: while frame is not None: if LOCALS_KEY_KI_PROTECTION_ENABLED in frame.f_locals: return frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] @@ -95,7 +99,7 @@ def ki_protection_enabled(frame): return True -def currently_ki_protected(): +def currently_ki_protected() -> bool: r"""Check whether the calling code has :exc:`KeyboardInterrupt` protection enabled. @@ -115,29 +119,35 @@ def currently_ki_protected(): # functions decorated @async_generator are given this magic property that's a # reference to the object itself # see python-trio/async_generator/async_generator/_impl.py -def legacy_isasyncgenfunction(obj): +def legacy_isasyncgenfunction( + obj: object, +) -> TypeGuard[Callable[..., types.AsyncGeneratorType]]: return getattr(obj, "_async_gen_function", None) == id(obj) -def _ki_protection_decorator(enabled): - def decorator(fn): +def _ki_protection_decorator( + enabled: bool, +) -> Callable[[Callable[ArgsT, RetT]], Callable[ArgsT, RetT]]: + # The "ignore[return-value]" below is because the inspect functions cast away the + # original return type of fn, making it just CoroutineType[Any, Any, Any] etc. + def decorator(fn: Callable[ArgsT, RetT]) -> Callable[ArgsT, RetT]: # In some version of Python, isgeneratorfunction returns true for # coroutine functions, so we have to check for coroutine functions # first. if inspect.iscoroutinefunction(fn): @wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: # See the comment for regular generators below coro = fn(*args, **kwargs) coro.cr_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled - return coro + return coro # type: ignore[return-value] return wrapper elif inspect.isgeneratorfunction(fn): @wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: # It's important that we inject this directly into the # generator's locals, as opposed to setting it here and then # doing 'yield from'. The reason is, if a generator is @@ -148,23 +158,23 @@ def wrapper(*args, **kwargs): # https://bugs.python.org/issue29590 gen = fn(*args, **kwargs) gen.gi_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled - return gen + return gen # type: ignore[return-value] return wrapper elif inspect.isasyncgenfunction(fn) or legacy_isasyncgenfunction(fn): @wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: # See the comment for regular generators above agen = fn(*args, **kwargs) agen.ag_frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled - return agen + return agen # type: ignore[return-value] return wrapper else: @wraps(fn) - def wrapper(*args, **kwargs): + def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = enabled return fn(*args, **kwargs) @@ -173,10 +183,14 @@ def wrapper(*args, **kwargs): return decorator -enable_ki_protection: Callable[[F], F] = _ki_protection_decorator(True) +enable_ki_protection: Callable[ + [Callable[ArgsT, RetT]], Callable[ArgsT, RetT] +] = _ki_protection_decorator(True) enable_ki_protection.__name__ = "enable_ki_protection" -disable_ki_protection: Callable[[F], F] = _ki_protection_decorator(False) +disable_ki_protection: Callable[ + [Callable[ArgsT, RetT]], Callable[ArgsT, RetT] +] = _ki_protection_decorator(False) disable_ki_protection.__name__ = "disable_ki_protection" @@ -184,7 +198,11 @@ def wrapper(*args, **kwargs): class KIManager: handler = attr.ib(default=None) - def install(self, deliver_cb, restrict_keyboard_interrupt_to_checkpoints): + def install( + self, + deliver_cb: Callable[[], object], + restrict_keyboard_interrupt_to_checkpoints: bool, + ) -> None: assert self.handler is None if ( not is_main_thread() @@ -192,7 +210,7 @@ def install(self, deliver_cb, restrict_keyboard_interrupt_to_checkpoints): ): return - def handler(signum, frame): + def handler(signum: int, frame: types.FrameType | None) -> None: assert signum == signal.SIGINT protection_enabled = ki_protection_enabled(frame) if protection_enabled or restrict_keyboard_interrupt_to_checkpoints: From 824dad64ee8ea2ce575a20d2bc3c1b5fd37826bc Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 29 Jul 2023 09:44:42 +1000 Subject: [PATCH 05/23] Coerce LOCALS_KEY_KI_PROTECTION_ENABLED to bool --- trio/_core/_ki.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index 1e6818585a..d199ccb4ee 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -5,7 +5,7 @@ import sys import types from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, TypeVar +from typing import TYPE_CHECKING, Any, Callable, TypeVar, Final import attr @@ -84,7 +84,7 @@ # We use this special string as a unique key into the frame locals dictionary. # The @ ensures it is not a valid identifier and can't clash with any possible # real local name. See: https://github.com/python-trio/trio/issues/469 -LOCALS_KEY_KI_PROTECTION_ENABLED = "@TRIO_KI_PROTECTION_ENABLED" +LOCALS_KEY_KI_PROTECTION_ENABLED: Final = "@TRIO_KI_PROTECTION_ENABLED" # NB: according to the signal.signal docs, 'frame' can be None on entry to @@ -92,7 +92,7 @@ def ki_protection_enabled(frame: types.FrameType | None) -> bool: while frame is not None: if LOCALS_KEY_KI_PROTECTION_ENABLED in frame.f_locals: - return frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED] + return bool(frame.f_locals[LOCALS_KEY_KI_PROTECTION_ENABLED]) if frame.f_code.co_name == "__del__": return True frame = frame.f_back From bda66ec9620afb6e8f8fb15c9303a0f371dad9b5 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 29 Jul 2023 10:08:44 +1000 Subject: [PATCH 06/23] Define type for trio.__deprecated_attributes__ --- trio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/__init__.py b/trio/__init__.py index ac0687f529..706c3e080d 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -117,7 +117,7 @@ _deprecate.enable_attribute_deprecations(__name__) -__deprecated_attributes__ = { +__deprecated_attributes__: "dict[str, _deprecate.DeprecatedAttribute]" = { "open_process": _deprecate.DeprecatedAttribute( value=lowlevel.open_process, version="0.20.0", From ca1f3feb997709e96dc32deaeadae9b057bb8b59 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 09:26:45 +1000 Subject: [PATCH 07/23] Add additional variations of TYPE_CHECKING to coverage --- .coveragerc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.coveragerc b/.coveragerc index d577aa8adf..431a02971b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -21,11 +21,14 @@ exclude_lines = abc.abstractmethod if TYPE_CHECKING: if _t.TYPE_CHECKING: + if t.TYPE_CHECKING: @overload partial_branches = pragma: no branch if not TYPE_CHECKING: if not _t.TYPE_CHECKING: + if not t.TYPE_CHECKING: if .* or not TYPE_CHECKING: if .* or not _t.TYPE_CHECKING: + if .* or not t.TYPE_CHECKING: From 097d1921bbf9eaf5c2da7f556c5674413257fa3d Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 09:27:06 +1000 Subject: [PATCH 08/23] Minor tweaks as suggested --- trio/_util.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index d49b482307..e7a76e095f 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -24,9 +24,9 @@ if t.TYPE_CHECKING: - + # Don't type check the implementation below, pthread_kill does not exist on Windows. def signal_raise(signum: int) -> None: - ... # Don't check the implementation below, pthread_kill does not exist on Windows. + ... # Equivalent to the C function raise(), which Python doesn't wrap @@ -199,7 +199,7 @@ class ConflictDetector: """ - def __init__(self, msg: str): + def __init__(self, msg: str) -> None: self._msg = msg self._held = False @@ -243,7 +243,7 @@ def decorator(func: CallT) -> CallT: def fixup_module_metadata(module_name: str, namespace: dict[str, t.Any]) -> None: seen_ids: set[int] = set() - def fix_one(qualname: str, name: str, obj: t.Any) -> None: + def fix_one(qualname: str, name: str, obj: object) -> None: # avoid infinite recursion (relevant when using # typing.Generic, for example) if id(obj) in seen_ids: @@ -258,7 +258,8 @@ def fix_one(qualname: str, name: str, obj: t.Any) -> None: # rewriting these. if hasattr(obj, "__name__") and "." not in obj.__name__: obj.__name__ = name - obj.__qualname__ = qualname + if hasattr(obj, "__qualname__"): + obj.__qualname__ = qualname if isinstance(obj, type): for attr_name, attr_value in obj.__dict__.items(): fix_one(objname + "." + attr_name, attr_name, attr_value) @@ -315,7 +316,10 @@ class SomeClass(metaclass=Final): """ def __new__( - cls, name: str, bases: tuple[type, ...], cls_namespace: dict[str, object] + cls, + name: str, + bases: tuple[type, ...], + cls_namespace: dict[str, object], ) -> Final: for base in bases: if isinstance(base, Final): From e955f79b18ae404010276c583d046ee16a7569e0 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 09:28:10 +1000 Subject: [PATCH 09/23] Enable stricter options for these modules --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d479442c7a..2718ed9149 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,8 +50,15 @@ disallow_untyped_defs = false module = [ "trio._path", "trio._file_io", + "trio._util", + "trio._ki", + "trio._deprecate", ] +disallow_incomplete_defs = true disallow_untyped_defs = true +disallow_untyped_decorators = true +disallow_any_unimported = true +disallow_subclassing_any = true [[tool.mypy.overrides]] module = [ From ad12306e7ec5fb8a3f8fabf211c91cc2e726f704 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 09:32:48 +1000 Subject: [PATCH 10/23] This is always an AsyncGenerator --- trio/_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index e7a76e095f..c858e830b0 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -358,7 +358,7 @@ def _create(cls: t.Type[T], *args: object, **kwargs: object) -> T: return super().__call__(*args, **kwargs) # type: ignore -def name_asyncgen(agen: t.Any) -> str: +def name_asyncgen(agen: collections.abc.AsyncGenerator[object, t.NoReturn]) -> str: """Return the fully-qualified name of the async generator function that produced the async generator iterator *agen*. """ @@ -369,7 +369,7 @@ def name_asyncgen(agen: t.Any) -> str: except (AttributeError, KeyError): module = f"<{agen.ag_code.co_filename}>" try: - qualname = agen.__qualname__ + qualname = agen.__qualname__ # type: ignore[attr-defined] except AttributeError: qualname = agen.ag_code.co_name return f"{module}.{qualname}" From c00ffc132358f9531b59b1a6c32e5eeb4d921782 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 09:55:08 +1000 Subject: [PATCH 11/23] Update type completeness --- trio/_tests/verify_types.json | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 60132e07fd..91b2b8d09d 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.9072, + "completenessScore": 0.8912, "exportedSymbolCounts": { - "withAmbiguousType": 1, - "withKnownType": 567, - "withUnknownType": 57 + "withAmbiguousType": 0, + "withKnownType": 557, + "withUnknownType": 68 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, @@ -46,12 +46,11 @@ ], "otherSymbolCounts": { "withAmbiguousType": 3, - "withKnownType": 574, - "withUnknownType": 76 + "withKnownType": 531, + "withUnknownType": 100 }, "packageName": "trio", "symbols": [ - "trio.__deprecated_attributes__", "trio._core._entry_queue.TrioToken.run_sync_soon", "trio._core._mock_clock.MockClock.jump", "trio._core._run.Nursery.start", @@ -85,10 +84,8 @@ "trio._ssl.SSLStream.transport_stream", "trio._ssl.SSLStream.unwrap", "trio._ssl.SSLStream.wait_send_all_might_not_block", - "trio._subprocess.Process.__aenter__", "trio._subprocess.Process.__init__", "trio._subprocess.Process.__repr__", - "trio._subprocess.Process.aclose", "trio._subprocess.Process.args", "trio._subprocess.Process.encoding", "trio._subprocess.Process.errors", @@ -107,7 +104,6 @@ "trio.lowlevel.current_root_task", "trio.lowlevel.current_statistics", "trio.lowlevel.current_trio_token", - "trio.lowlevel.currently_ki_protected", "trio.lowlevel.notify_closing", "trio.lowlevel.open_process", "trio.lowlevel.permanently_detach_coroutine_object", From acab690b38c4dc8e9ce4afd7c4bc4aa372294555 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 10:04:02 +1000 Subject: [PATCH 12/23] Use annotations future import, dict[] is flagged by mypy in 3.8 --- trio/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/__init__.py b/trio/__init__.py index 706c3e080d..d4cc1597f5 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -1,5 +1,6 @@ """Trio - A friendly Python library for async concurrency and I/O """ +from __future__ import annotations # isort: skip # General layout: # @@ -117,7 +118,7 @@ _deprecate.enable_attribute_deprecations(__name__) -__deprecated_attributes__: "dict[str, _deprecate.DeprecatedAttribute]" = { +__deprecated_attributes__: dict[str, _deprecate.DeprecatedAttribute] = { "open_process": _deprecate.DeprecatedAttribute( value=lowlevel.open_process, version="0.20.0", From 0e3939c863e98c2a98b4951bc32523ad9ad0ee38 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 10:04:11 +1000 Subject: [PATCH 13/23] Sort imports and type completeness --- trio/_core/_ki.py | 2 +- trio/_tests/verify_types.json | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index d199ccb4ee..b4165dc7c9 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -5,7 +5,7 @@ import sys import types from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, TypeVar, Final +from typing import TYPE_CHECKING, Any, Callable, Final, TypeVar import attr diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 91b2b8d09d..4d632d2589 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.8912, + "completenessScore": 0.9104, "exportedSymbolCounts": { "withAmbiguousType": 0, - "withKnownType": 557, - "withUnknownType": 68 + "withKnownType": 569, + "withUnknownType": 56 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, @@ -46,8 +46,8 @@ ], "otherSymbolCounts": { "withAmbiguousType": 3, - "withKnownType": 531, - "withUnknownType": 100 + "withKnownType": 576, + "withUnknownType": 74 }, "packageName": "trio", "symbols": [ From 492db848aac4d48273e1a789bccab405ca9152b0 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 10:24:23 +1000 Subject: [PATCH 14/23] Unused import --- trio/_deprecate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_deprecate.py b/trio/_deprecate.py index 7890da5323..473e3d53b1 100644 --- a/trio/_deprecate.py +++ b/trio/_deprecate.py @@ -5,7 +5,7 @@ from collections.abc import Callable from functools import wraps from types import ModuleType -from typing import TYPE_CHECKING, ClassVar, Optional, TypeVar +from typing import TYPE_CHECKING, ClassVar, TypeVar import attr From 22a7473dcfa6876f1eafe17ef1d895c4cd8de512 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 10:24:17 +1000 Subject: [PATCH 15/23] Ignore __future__ imports when checking public symbols --- trio/_tests/test_exports.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/trio/_tests/test_exports.py b/trio/_tests/test_exports.py index b5d0a44088..1b1e8df8da 100644 --- a/trio/_tests/test_exports.py +++ b/trio/_tests/test_exports.py @@ -1,3 +1,5 @@ +import __future__ # Regular import, not special! + import enum import functools import importlib @@ -107,6 +109,11 @@ def no_underscores(symbols): if modname == "trio": runtime_names.discard("tests") + # Ignore any __future__ feature objects, if imported under that name. + for name in __future__.all_feature_names: + if getattr(module, name, None) is getattr(__future__, name): + runtime_names.remove(name) + if tool in ("mypy", "pyright_verifytypes"): # create py.typed file py_typed_path = Path(trio.__file__).parent / "py.typed" From 342d25e5679b9001a89e0a4a979d9dcb45fef73d Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 11:07:45 +1000 Subject: [PATCH 16/23] Use isort: split instead of skip, has much more predictable behaviour --- trio/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/__init__.py b/trio/__init__.py index d4cc1597f5..be7de42cde 100644 --- a/trio/__init__.py +++ b/trio/__init__.py @@ -1,6 +1,6 @@ """Trio - A friendly Python library for async concurrency and I/O """ -from __future__ import annotations # isort: skip +from __future__ import annotations # General layout: # @@ -17,7 +17,7 @@ # Uses `from x import y as y` for compatibility with `pyright --verifytypes` (#2625) # must be imported early to avoid circular import -from ._core import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED # isort: skip +from ._core import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED # isort: split # Submodules imported by default from . import abc, from_thread, lowlevel, socket, to_thread From f4eaa82d20fee20a9d41c2b516c7016536466697 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 11:09:48 +1000 Subject: [PATCH 17/23] This typevar is no longer used, so we can import from collections.abc --- trio/_core/_ki.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_core/_ki.py b/trio/_core/_ki.py index b4165dc7c9..8ae83c287a 100644 --- a/trio/_core/_ki.py +++ b/trio/_core/_ki.py @@ -4,14 +4,14 @@ import signal import sys import types +from collections.abc import Callable from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Final, TypeVar +from typing import TYPE_CHECKING, Final, TypeVar import attr from .._util import is_main_thread -F = TypeVar("F", bound=Callable[..., Any]) RetT = TypeVar("RetT") if TYPE_CHECKING: From 0a6f748e98bb4a142117543ec4f430403a544576 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 11:09:58 +1000 Subject: [PATCH 18/23] Use AsyncGeneratorType instead --- trio/_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index c858e830b0..5a8029621d 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -9,7 +9,7 @@ import typing as t from abc import ABCMeta from functools import update_wrapper -from types import TracebackType +from types import AsyncGeneratorType, TracebackType import trio @@ -358,7 +358,7 @@ def _create(cls: t.Type[T], *args: object, **kwargs: object) -> T: return super().__call__(*args, **kwargs) # type: ignore -def name_asyncgen(agen: collections.abc.AsyncGenerator[object, t.NoReturn]) -> str: +def name_asyncgen(agen: AsyncGeneratorType[object, t.NoReturn]) -> str: """Return the fully-qualified name of the async generator function that produced the async generator iterator *agen*. """ @@ -369,7 +369,7 @@ def name_asyncgen(agen: collections.abc.AsyncGenerator[object, t.NoReturn]) -> s except (AttributeError, KeyError): module = f"<{agen.ag_code.co_filename}>" try: - qualname = agen.__qualname__ # type: ignore[attr-defined] + qualname = agen.__qualname__ except AttributeError: qualname = agen.ag_code.co_name return f"{module}.{qualname}" From eb7286969e05a7c76035a5b05981c2bcfec3901c Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 12:22:32 +1000 Subject: [PATCH 19/23] Fix quotations --- trio/_deprecate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_deprecate.py b/trio/_deprecate.py index 473e3d53b1..0a9553b854 100644 --- a/trio/_deprecate.py +++ b/trio/_deprecate.py @@ -117,7 +117,7 @@ def deprecated_alias( @deprecated(version, issue=issue, instead=new_fn) @wraps(new_fn, assigned=("__module__", "__annotations__")) def wrapper(*args: ArgsT.args, **kwargs: ArgsT.kwargs) -> RetT: - "Deprecated alias." + """Deprecated alias.""" return new_fn(*args, **kwargs) wrapper.__qualname__ = old_qualname From 8d198aee44ac59bd78a5efe13f22504d8a718e6d Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 12:26:29 +1000 Subject: [PATCH 20/23] Add a test for fixup_module_metadata --- trio/_tests/test_util.py | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/trio/_tests/test_util.py b/trio/_tests/test_util.py index a4df6d35b4..7d73a9289e 100644 --- a/trio/_tests/test_util.py +++ b/trio/_tests/test_util.py @@ -1,5 +1,6 @@ import signal import sys +import types import pytest @@ -15,6 +16,7 @@ Final, NoPublicConstructor, coroutine_or_error, + fixup_module_metadata, generic_function, is_main_thread, signal_raise, @@ -192,3 +194,61 @@ class SubClass(SpecialClass): # Private constructor should not raise assert isinstance(SpecialClass._create(), SpecialClass) + + +def test_fixup_module_metadata(): + # Ignores modules not in the trio.X tree. + non_trio_module = types.ModuleType('not_trio') + non_trio_module.some_func = lambda: None + non_trio_module.some_func.__name__ = 'some_func' + non_trio_module.some_func.__qualname__ = 'some_func' + + fixup_module_metadata(non_trio_module.__name__, vars(non_trio_module)) + + assert non_trio_module.some_func.__name__ == 'some_func' + assert non_trio_module.some_func.__qualname__ == 'some_func' + + # Bulild up a fake module to test. Just use lambdas since all we care about is the names. + mod = types.ModuleType('trio._somemodule_impl') + mod.some_func = lambda: None + mod.some_func.__name__ = '_something_else' + mod.some_func.__qualname__ = '_something_else' + + # No __module__ means it's unchanged. + mod.not_funclike = types.SimpleNamespace() + mod.not_funclike.__name__ = 'not_funclike' + + # Check __qualname__ being absent works. + mod.only_has_name = types.SimpleNamespace() + mod.only_has_name.__module__ = 'trio._somemodule_impl' + mod.only_has_name.__name__ = 'only_name' + + # Underscored names are unchanged. + mod._private = lambda: None + mod._private.__module__ = 'trio._somemodule_impl' + mod._private.__name__ = mod._private.__qualname__ = '_private' + + # We recurse into classes. + mod.SomeClass = type('SomeClass', (), { + '__init__': lambda self: None, + 'method': lambda self, a, b: a + b, + }) + mod.SomeClass.recursion = mod.SomeClass # Reference loop is fine. + + fixup_module_metadata('trio.somemodule', vars(mod)) + assert mod.some_func.__name__ == 'some_func' + assert mod.some_func.__module__ == 'trio.somemodule' + assert mod.some_func.__qualname__ == 'some_func' + + assert mod.not_funclike.__name__ == 'not_funclike' + assert mod._private.__name__ == '_private' + assert mod._private.__module__ == 'trio._somemodule_impl' + assert mod._private.__qualname__ == '_private' + + assert mod.only_has_name.__name__ == 'only_has_name' + assert mod.only_has_name.__module__ == 'trio.somemodule' + assert not hasattr(mod.only_has_name, '__qualname__') + + assert mod.SomeClass.method.__name__ == 'method' + assert mod.SomeClass.method.__module__ == 'trio.somemodule' + assert mod.SomeClass.method.__qualname__ == 'SomeClass.method' From 15843398818fff8f9dc005c3daa2686e83f5e8e2 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 12:27:45 +1000 Subject: [PATCH 21/23] Use Mapping instead of dict[.., Any] --- trio/_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_util.py b/trio/_util.py index 5a8029621d..eea58a8edd 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -240,7 +240,7 @@ def decorator(func: CallT) -> CallT: return decorator -def fixup_module_metadata(module_name: str, namespace: dict[str, t.Any]) -> None: +def fixup_module_metadata(module_name: str, namespace: collections.abc.Mapping[str, object]) -> None: seen_ids: set[int] = set() def fix_one(qualname: str, name: str, obj: object) -> None: From 84d586d6f5909064191216890a42d3cf67383e38 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 12:31:01 +1000 Subject: [PATCH 22/23] Blacken new test --- trio/_tests/test_util.py | 66 +++++++++++++++++++++------------------- trio/_util.py | 4 ++- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/trio/_tests/test_util.py b/trio/_tests/test_util.py index 7d73a9289e..d612e57b17 100644 --- a/trio/_tests/test_util.py +++ b/trio/_tests/test_util.py @@ -198,57 +198,61 @@ class SubClass(SpecialClass): def test_fixup_module_metadata(): # Ignores modules not in the trio.X tree. - non_trio_module = types.ModuleType('not_trio') + non_trio_module = types.ModuleType("not_trio") non_trio_module.some_func = lambda: None - non_trio_module.some_func.__name__ = 'some_func' - non_trio_module.some_func.__qualname__ = 'some_func' + non_trio_module.some_func.__name__ = "some_func" + non_trio_module.some_func.__qualname__ = "some_func" fixup_module_metadata(non_trio_module.__name__, vars(non_trio_module)) - assert non_trio_module.some_func.__name__ == 'some_func' - assert non_trio_module.some_func.__qualname__ == 'some_func' + assert non_trio_module.some_func.__name__ == "some_func" + assert non_trio_module.some_func.__qualname__ == "some_func" # Bulild up a fake module to test. Just use lambdas since all we care about is the names. - mod = types.ModuleType('trio._somemodule_impl') + mod = types.ModuleType("trio._somemodule_impl") mod.some_func = lambda: None - mod.some_func.__name__ = '_something_else' - mod.some_func.__qualname__ = '_something_else' + mod.some_func.__name__ = "_something_else" + mod.some_func.__qualname__ = "_something_else" # No __module__ means it's unchanged. mod.not_funclike = types.SimpleNamespace() - mod.not_funclike.__name__ = 'not_funclike' + mod.not_funclike.__name__ = "not_funclike" # Check __qualname__ being absent works. mod.only_has_name = types.SimpleNamespace() - mod.only_has_name.__module__ = 'trio._somemodule_impl' - mod.only_has_name.__name__ = 'only_name' + mod.only_has_name.__module__ = "trio._somemodule_impl" + mod.only_has_name.__name__ = "only_name" # Underscored names are unchanged. mod._private = lambda: None - mod._private.__module__ = 'trio._somemodule_impl' - mod._private.__name__ = mod._private.__qualname__ = '_private' + mod._private.__module__ = "trio._somemodule_impl" + mod._private.__name__ = mod._private.__qualname__ = "_private" # We recurse into classes. - mod.SomeClass = type('SomeClass', (), { - '__init__': lambda self: None, - 'method': lambda self, a, b: a + b, - }) + mod.SomeClass = type( + "SomeClass", + (), + { + "__init__": lambda self: None, + "method": lambda self, a, b: a + b, + }, + ) mod.SomeClass.recursion = mod.SomeClass # Reference loop is fine. - fixup_module_metadata('trio.somemodule', vars(mod)) - assert mod.some_func.__name__ == 'some_func' - assert mod.some_func.__module__ == 'trio.somemodule' - assert mod.some_func.__qualname__ == 'some_func' + fixup_module_metadata("trio.somemodule", vars(mod)) + assert mod.some_func.__name__ == "some_func" + assert mod.some_func.__module__ == "trio.somemodule" + assert mod.some_func.__qualname__ == "some_func" - assert mod.not_funclike.__name__ == 'not_funclike' - assert mod._private.__name__ == '_private' - assert mod._private.__module__ == 'trio._somemodule_impl' - assert mod._private.__qualname__ == '_private' + assert mod.not_funclike.__name__ == "not_funclike" + assert mod._private.__name__ == "_private" + assert mod._private.__module__ == "trio._somemodule_impl" + assert mod._private.__qualname__ == "_private" - assert mod.only_has_name.__name__ == 'only_has_name' - assert mod.only_has_name.__module__ == 'trio.somemodule' - assert not hasattr(mod.only_has_name, '__qualname__') + assert mod.only_has_name.__name__ == "only_has_name" + assert mod.only_has_name.__module__ == "trio.somemodule" + assert not hasattr(mod.only_has_name, "__qualname__") - assert mod.SomeClass.method.__name__ == 'method' - assert mod.SomeClass.method.__module__ == 'trio.somemodule' - assert mod.SomeClass.method.__qualname__ == 'SomeClass.method' + assert mod.SomeClass.method.__name__ == "method" + assert mod.SomeClass.method.__module__ == "trio.somemodule" + assert mod.SomeClass.method.__qualname__ == "SomeClass.method" diff --git a/trio/_util.py b/trio/_util.py index eea58a8edd..ba56c18385 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -240,7 +240,9 @@ def decorator(func: CallT) -> CallT: return decorator -def fixup_module_metadata(module_name: str, namespace: collections.abc.Mapping[str, object]) -> None: +def fixup_module_metadata( + module_name: str, namespace: collections.abc.Mapping[str, object] +) -> None: seen_ids: set[int] = set() def fix_one(qualname: str, name: str, obj: object) -> None: From 168b5b09638ea659608612f0da1065ec5b314cf3 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 30 Jul 2023 12:55:10 +1000 Subject: [PATCH 23/23] Fix coverage problems --- trio/_tests/test_util.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/trio/_tests/test_util.py b/trio/_tests/test_util.py index d612e57b17..1ab6f825de 100644 --- a/trio/_tests/test_util.py +++ b/trio/_tests/test_util.py @@ -234,7 +234,7 @@ def test_fixup_module_metadata(): (), { "__init__": lambda self: None, - "method": lambda self, a, b: a + b, + "method": lambda self: None, }, ) mod.SomeClass.recursion = mod.SomeClass # Reference loop is fine. @@ -256,3 +256,8 @@ def test_fixup_module_metadata(): assert mod.SomeClass.method.__name__ == "method" assert mod.SomeClass.method.__module__ == "trio.somemodule" assert mod.SomeClass.method.__qualname__ == "SomeClass.method" + # Make coverage happy. + non_trio_module.some_func() + mod.some_func() + mod._private() + mod.SomeClass().method()