From bd5a451098b54c19fca8c898c5c4a99b3d8e8c0b Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Tue, 22 Aug 2023 07:52:18 +1000 Subject: [PATCH 01/26] Use TypeVarTuple in various functions --- src/trio/_core/_entry_queue.py | 21 +++++++++++++++------ src/trio/_dtls.py | 12 ++++++------ src/trio/_util.py | 7 ++++--- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/trio/_core/_entry_queue.py b/src/trio/_core/_entry_queue.py index cb91025fbb..f92166d3bc 100644 --- a/src/trio/_core/_entry_queue.py +++ b/src/trio/_core/_entry_queue.py @@ -2,7 +2,7 @@ import threading from collections import deque -from typing import Callable, Iterable, NoReturn, Tuple +from typing import TYPE_CHECKING, Callable, Iterable, NoReturn, Tuple import attr @@ -10,10 +10,13 @@ from .._util import NoPublicConstructor, final from ._wakeup_socketpair import WakeupSocketpair -# TODO: Type with TypeVarTuple, at least to an extent where it makes -# the public interface safe. +if TYPE_CHECKING: + from typing_extensions import TypeVarTuple, Unpack + + PosArgsT = TypeVarTuple("PosArgsT") + Function = Callable[..., object] -Job = Tuple[Function, Iterable[object]] +Job = Tuple[Function, Tuple[object, ...]] @attr.s(slots=True) @@ -122,7 +125,10 @@ def size(self) -> int: return len(self.queue) + len(self.idempotent_queue) def run_sync_soon( - self, sync_fn: Function, *args: object, idempotent: bool = False + self, + sync_fn: Callable[[Unpack[PosArgsT]], object], + *args: Unpack[PosArgsT], + idempotent: bool = False, ) -> None: with self.lock: if self.done: @@ -163,7 +169,10 @@ class TrioToken(metaclass=NoPublicConstructor): _reentry_queue: EntryQueue = attr.ib() def run_sync_soon( - self, sync_fn: Function, *args: object, idempotent: bool = False + self, + sync_fn: Callable[[Unpack[PosArgsT]], object], + *args: Unpack[PosArgsT], + idempotent: bool = False, ) -> None: """Schedule a call to ``sync_fn(*args)`` to occur in the context of a Trio task. diff --git a/src/trio/_dtls.py b/src/trio/_dtls.py index 7d1969bab4..fb99167423 100644 --- a/src/trio/_dtls.py +++ b/src/trio/_dtls.py @@ -42,10 +42,12 @@ # See DTLSEndpoint.__init__ for why this is imported here from OpenSSL import SSL # noqa: TCH004 from OpenSSL.SSL import Context - from typing_extensions import Self, TypeAlias + from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack from trio.socket import SocketType + PosArgsT = TypeVarTuple("PosArgsT") + MAX_UDP_PACKET_SIZE = 65527 @@ -1258,14 +1260,11 @@ def _check_closed(self) -> None: if self._closed: raise trio.ClosedResourceError - # async_fn cannot be typed with ParamSpec, since we don't accept - # kwargs. Can be typed with TypeVarTuple once it's fully supported - # in mypy. async def serve( self, ssl_context: Context, - async_fn: Callable[..., Awaitable[object]], - *args: Any, + async_fn: Callable[[DTLSChannel, Unpack[PosArgsT]], Awaitable[object]], + *args: Unpack[PosArgsT], task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED, ) -> None: """Listen for incoming connections, and spawn a handler for each using an @@ -1294,6 +1293,7 @@ async def handler(dtls_channel): incoming connections. async_fn: The handler function that will be invoked for each incoming connection. + *args: Additional arguments to pass to the handler function. """ self._check_closed() diff --git a/src/trio/_util.py b/src/trio/_util.py index 5f514cb532..f4d4195128 100644 --- a/src/trio/_util.py +++ b/src/trio/_util.py @@ -21,9 +21,10 @@ if t.TYPE_CHECKING: from types import AsyncGeneratorType, TracebackType - from typing_extensions import ParamSpec, Self + from typing_extensions import ParamSpec, Self, TypeVarTuple, Unpack ArgsT = ParamSpec("ArgsT") + PosArgsT = TypeVarTuple("PosArgsT") if t.TYPE_CHECKING: @@ -102,9 +103,9 @@ def is_main_thread() -> bool: # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. Returns coroutine object. ###### -# TODO: Use TypeVarTuple here. def coroutine_or_error( - async_fn: t.Callable[..., t.Awaitable[RetT]], *args: t.Any + async_fn: t.Callable[[Unpack[PosArgsT]], t.Awaitable[RetT]], + *args: Unpack[PosArgsT], ) -> collections.abc.Coroutine[object, t.NoReturn, RetT]: def _return_value_looks_like_wrong_library(value: object) -> bool: # Returned by legacy @asyncio.coroutine functions, which includes From 97d6e0eecfe2beeca3c075a18e78c8d830ab6da1 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Tue, 22 Aug 2023 08:13:38 +1000 Subject: [PATCH 02/26] Use TypeVarTuple in _run --- src/trio/_core/_run.py | 57 ++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index d5006eed91..4cbdb6df99 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -53,6 +53,12 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup +FnT = TypeVar("FnT", bound="Callable[..., Any]") +StatusT = TypeVar("StatusT") +StatusT_co = TypeVar("StatusT_co", covariant=True) +StatusT_contra = TypeVar("StatusT_contra", contravariant=True) +RetT = TypeVar("RetT") + if TYPE_CHECKING: import contextvars @@ -70,19 +76,25 @@ # for some strange reason Sphinx works with outcome.Outcome, but not Outcome, in # start_guest_run. Same with types.FrameType in iter_await_frames import outcome - from typing_extensions import Self + from typing_extensions import Self, TypeVarTuple, Unpack + + PosArgT = TypeVarTuple("PosArgT") + + # Needs to be guarded, since Unpack[] would be evaluated at runtime. + class _NurseryStartFunc(Protocol[Unpack[PosArgT], StatusT_co]): + """Type of functions passed to `nursery.start() `.""" + + def __call__( + self, *args: Unpack[PosArgT], task_status: TaskStatus[StatusT_co] + ) -> Awaitable[object]: + ... + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: Final = 1000 # Passed as a sentinel _NO_SEND: Final[Outcome[Any]] = cast("Outcome[Any]", object()) -FnT = TypeVar("FnT", bound="Callable[..., Any]") -StatusT = TypeVar("StatusT") -StatusT_co = TypeVar("StatusT_co", covariant=True) -StatusT_contra = TypeVar("StatusT_contra", contravariant=True) -RetT = TypeVar("RetT") - @final class _NoStatus(metaclass=NoPublicConstructor): @@ -1119,9 +1131,8 @@ def aborted(raise_cancel: _core.RaiseCancelT) -> Abort: def start_soon( self, - # TODO: TypeVarTuple - async_fn: Callable[..., Awaitable[object]], - *args: object, + async_fn: Callable[[PosArgT], Awaitable[object]], + *args: Unpack[PosArgT], name: object = None, ) -> None: """Creates a child task, scheduling ``await async_fn(*args)``. @@ -1167,8 +1178,8 @@ def start_soon( async def start( self, - async_fn: Callable[..., Awaitable[object]], - *args: object, + async_fn: _NurseryStartFunc[Unpack[PosArgT], StatusT], + *args: Unpack[PosArgT], name: object = None, ) -> StatusT: r"""Creates and initializes a child task. @@ -1690,9 +1701,8 @@ def reschedule( # type: ignore[misc] def spawn_impl( self, - # TODO: TypeVarTuple - async_fn: Callable[..., Awaitable[object]], - args: tuple[object, ...], + async_fn: Callable[[Unpack[PosArgT]], Awaitable[object]], + args: tuple[Unpack[PosArgT]], nursery: Nursery | None, name: object, *, @@ -1721,8 +1731,7 @@ def spawn_impl( # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. ###### - # TODO: resolve the type: ignore when implementing TypeVarTuple - coro = context.run(coroutine_or_error, async_fn, *args) # type: ignore[arg-type] + coro = context.run(coroutine_or_error, async_fn, *args) if name is None: name = async_fn @@ -1809,12 +1818,11 @@ def task_exited(self, task: Task, outcome: Outcome[Any]) -> None: # System tasks and init ################ - @_public # Type-ignore due to use of Any here. - def spawn_system_task( # type: ignore[misc] + @_public + def spawn_system_task( self, - # TODO: TypeVarTuple - async_fn: Callable[..., Awaitable[object]], - *args: object, + async_fn: Callable[[Unpack[PosArgT]], Awaitable[object]], + *args: Unpack[PosArgT], name: object = None, context: contextvars.Context | None = None, ) -> Task: @@ -1879,10 +1887,9 @@ def spawn_system_task( # type: ignore[misc] ) async def init( - # TODO: TypeVarTuple self, - async_fn: Callable[..., Awaitable[object]], - args: tuple[object, ...], + async_fn: Callable[[Unpack[PosArgT]], Awaitable[object]], + args: tuple[Unpack[PosArgT]], ) -> None: # run_sync_soon task runs here: async with open_nursery() as run_sync_soon_nursery: From 9bcb1855e4451fc3e849aefd91f1545ff8584b26 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 20 Sep 2023 12:33:22 +1000 Subject: [PATCH 03/26] Cast this partial() definition, we know it's safe --- src/trio/_core/_run.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index 4cbdb6df99..df759bbd23 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1131,7 +1131,7 @@ def aborted(raise_cancel: _core.RaiseCancelT) -> Abort: def start_soon( self, - async_fn: Callable[[PosArgT], Awaitable[object]], + async_fn: Callable[[Unpack[PosArgT]], Awaitable[object]], *args: Unpack[PosArgT], name: object = None, ) -> None: @@ -1231,7 +1231,13 @@ async def async_fn(arg1, arg2, *, task_status=trio.TASK_STATUS_IGNORED): # exception in an extra ExceptionGroup. See #2611. async with open_nursery(strict_exception_groups=False) as old_nursery: task_status: _TaskStatus[StatusT] = _TaskStatus(old_nursery, self) - thunk = functools.partial(async_fn, task_status=task_status) + # Without the task_status keyword argument, this is just a positional-only function. + thunk: Callable[ + [Unpack[PosArgT]], Awaitable[object] + ] = functools.partial( # type: ignore[assignment] + async_fn, + task_status=task_status, + ) task = GLOBAL_RUN_CONTEXT.runner.spawn_impl( thunk, args, old_nursery, name ) From 6f8153f1ce5d90b170e5158c933f5b2b9ba35af8 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 20 Sep 2023 12:34:09 +1000 Subject: [PATCH 04/26] Fix some uncovered type errors --- src/trio/_core/_tests/test_local.py | 4 ++-- src/trio/_core/_tests/test_run.py | 9 +++++++-- src/trio/_subprocess.py | 3 +++ src/trio/_tests/test_channel.py | 16 ++++++++-------- src/trio/_tests/test_scheduler_determinism.py | 2 +- src/trio/_tests/test_subprocess.py | 5 +++++ src/trio/_tests/test_sync.py | 2 +- src/trio/_tests/test_threads.py | 2 +- 8 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/trio/_core/_tests/test_local.py b/src/trio/_core/_tests/test_local.py index 5fdf54b13c..17b10bca35 100644 --- a/src/trio/_core/_tests/test_local.py +++ b/src/trio/_core/_tests/test_local.py @@ -77,8 +77,8 @@ async def task1() -> None: t1.set("plaice") assert t1.get() == "plaice" - async def task2(tok: str) -> None: - t1.reset(token) + async def task2(tok: RunVarToken[str]) -> None: + t1.reset(tok) with pytest.raises(LookupError): t1.get() diff --git a/src/trio/_core/_tests/test_run.py b/src/trio/_core/_tests/test_run.py index f647f7f6c1..5e0df641bc 100644 --- a/src/trio/_core/_tests/test_run.py +++ b/src/trio/_core/_tests/test_run.py @@ -1482,6 +1482,7 @@ async def child2() -> None: assert tasks["child2"].child_nurseries == [] async def child1( + *, task_status: _core.TaskStatus[None] = _core.TASK_STATUS_IGNORED, ) -> None: me = tasks["child1"] = _core.current_task() @@ -1674,7 +1675,7 @@ async def no_args() -> None: # pragma: no cover # Errors in calling convention get raised immediately from start async with _core.open_nursery() as nursery: with pytest.raises(TypeError): - await nursery.start(no_args) + await nursery.start(no_args) # type: ignore[arg-type] async def sleep_then_start( seconds: int, *, task_status: _core.TaskStatus[int] = _core.TASK_STATUS_IGNORED @@ -1704,6 +1705,7 @@ async def sleep_then_start( # calling started twice async def double_started( + *, task_status: _core.TaskStatus[None] = _core.TASK_STATUS_IGNORED, ) -> None: task_status.started() @@ -1715,6 +1717,7 @@ async def double_started( # child crashes before calling started -> error comes out of .start() async def raise_keyerror( + *, task_status: _core.TaskStatus[None] = _core.TASK_STATUS_IGNORED, ) -> None: raise KeyError("oops") @@ -1725,6 +1728,7 @@ async def raise_keyerror( # child exiting cleanly before calling started -> triggers a RuntimeError async def nothing( + *, task_status: _core.TaskStatus[None] = _core.TASK_STATUS_IGNORED, ) -> None: return @@ -1738,6 +1742,7 @@ async def nothing( # nothing -- the child keeps executing under start(). The value it passed # is ignored; start() raises Cancelled. async def just_started( + *, task_status: _core.TaskStatus[str] = _core.TASK_STATUS_IGNORED, ) -> None: task_status.started("hi") @@ -1920,7 +1925,7 @@ def __init__(self, *largs: it) -> None: self.nexts = [obj.__anext__ for obj in largs] async def _accumulate( - self, f: Callable[[], Awaitable[int]], items: list[int | None], i: int + self, f: Callable[[], Awaitable[int]], items: list[int], i: int ) -> None: items[i] = await f() diff --git a/src/trio/_subprocess.py b/src/trio/_subprocess.py index b9bdff1a75..4d325b2fb5 100644 --- a/src/trio/_subprocess.py +++ b/src/trio/_subprocess.py @@ -764,14 +764,17 @@ async def read_output( proc = await open_process(command, **options) # type: ignore[arg-type, call-overload, unused-ignore] try: if input is not None: + assert proc.stdin is not None nursery.start_soon(feed_input, proc.stdin) proc.stdin = None proc.stdio = None if capture_stdout: + assert proc.stdout is not None nursery.start_soon(read_output, proc.stdout, stdout_chunks) proc.stdout = None proc.stdio = None if capture_stderr: + assert proc.stderr is not None nursery.start_soon(read_output, proc.stderr, stderr_chunks) proc.stderr = None task_status.started(proc) diff --git a/src/trio/_tests/test_channel.py b/src/trio/_tests/test_channel.py index c81933b6b7..ae555715cb 100644 --- a/src/trio/_tests/test_channel.py +++ b/src/trio/_tests/test_channel.py @@ -151,17 +151,17 @@ async def receive_block(r: trio.MemoryReceiveChannel[int]) -> None: with pytest.raises(trio.ClosedResourceError): await r.receive() - s, r = open_memory_channel[None](0) + s2, r2 = open_memory_channel[int](0) async with trio.open_nursery() as nursery: - nursery.start_soon(receive_block, r) + nursery.start_soon(receive_block, r2) await wait_all_tasks_blocked() - await r.aclose() + await r2.aclose() # and it's persistent with pytest.raises(trio.ClosedResourceError): - r.receive_nowait() + r2.receive_nowait() with pytest.raises(trio.ClosedResourceError): - await r.receive() + await r2.receive() async def test_close_sync() -> None: @@ -204,7 +204,7 @@ async def send_block( await s.send(None) # closing receive -> other receive gets ClosedResourceError - async def receive_block(r: trio.MemoryReceiveChannel[int]) -> None: + async def receive_block(r: trio.MemoryReceiveChannel[None]) -> None: with pytest.raises(trio.ClosedResourceError): await r.receive() @@ -366,9 +366,9 @@ async def test_channel_fairness() -> None: # But if someone else is waiting to receive, then they "own" the item we # send, so we can't receive it (even though we run first): - result = None + result: int | None = None - async def do_receive(r: trio.MemoryReceiveChannel[int]) -> None: + async def do_receive(r: trio.MemoryReceiveChannel[int | None]) -> None: nonlocal result result = await r.receive() diff --git a/src/trio/_tests/test_scheduler_determinism.py b/src/trio/_tests/test_scheduler_determinism.py index bf0eec3d39..02733226cf 100644 --- a/src/trio/_tests/test_scheduler_determinism.py +++ b/src/trio/_tests/test_scheduler_determinism.py @@ -19,7 +19,7 @@ async def tracer(name: str) -> None: async with trio.open_nursery() as nursery: for i in range(5): - nursery.start_soon(tracer, i) + nursery.start_soon(tracer, str(i)) return tuple(trace) diff --git a/src/trio/_tests/test_subprocess.py b/src/trio/_tests/test_subprocess.py index ec0cf109a5..de60c3e90d 100644 --- a/src/trio/_tests/test_subprocess.py +++ b/src/trio/_tests/test_subprocess.py @@ -225,6 +225,9 @@ async def check_output(stream: Stream, expected: bytes) -> None: seen += chunk assert seen == expected + assert proc.stdout is not None + assert proc.stderr is not None + async with _core.open_nursery() as nursery: # fail eventually if something is broken nursery.cancel_scope.deadline = _core.current_time() + 30.0 @@ -277,6 +280,8 @@ async def drain_one(stream: Stream, count: int, digit: int) -> None: assert count == 0 assert await stream.receive_some(len(newline)) == newline + assert proc.stdout is not None + assert proc.stderr is not None nursery.start_soon(drain_one, proc.stdout, request, idx * 2) nursery.start_soon(drain_one, proc.stderr, request * 2, idx * 2 + 1) diff --git a/src/trio/_tests/test_sync.py b/src/trio/_tests/test_sync.py index 9179c8a5ae..dd78d2e82f 100644 --- a/src/trio/_tests/test_sync.py +++ b/src/trio/_tests/test_sync.py @@ -546,7 +546,7 @@ async def test_generic_lock_fifo_fairness(lock_factory: LockFactory) -> None: record = [] LOOPS = 5 - async def loopy(name: str, lock_like: LockLike) -> None: + async def loopy(name: int, lock_like: LockLike) -> None: # Record the order each task was initially scheduled in initial_order.append(name) for _ in range(LOOPS): diff --git a/src/trio/_tests/test_threads.py b/src/trio/_tests/test_threads.py index d9af3bf5e7..adbd5e3bda 100644 --- a/src/trio/_tests/test_threads.py +++ b/src/trio/_tests/test_threads.py @@ -917,7 +917,7 @@ def get_tid_then_reenter() -> int: nonlocal tid tid = threading.get_ident() # The nesting of wrapper functions loses the return value of threading.get_ident - return from_thread_run(to_thread_run_sync, threading.get_ident) # type: ignore[return-value] + return from_thread_run(to_thread_run_sync, threading.get_ident) # type: ignore[no-any-return] assert tid != await to_thread_run_sync(get_tid_then_reenter) From 52c69c867d3dc1c3ab038e6488b5772faf334635 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 19 Nov 2023 10:02:16 +1000 Subject: [PATCH 05/26] This now passes mypy checking --- src/trio/_core/_run.py | 4 ++-- src/trio/_core/_tests/test_guest_mode.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index df759bbd23..367509a4f8 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -2421,8 +2421,8 @@ def my_done_callback(run_outcome): # straight through. def unrolled_run( runner: Runner, - async_fn: Callable[..., object], - args: tuple[object, ...], + async_fn: Callable[[Unpack[PosArgT]], object], + args: tuple[Unpack[PosArgT]], host_uses_signal_set_wakeup_fd: bool = False, ) -> Generator[float, EventResult, None]: locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True diff --git a/src/trio/_core/_tests/test_guest_mode.py b/src/trio/_core/_tests/test_guest_mode.py index 8d63d5246b..aa912ab70e 100644 --- a/src/trio/_core/_tests/test_guest_mode.py +++ b/src/trio/_core/_tests/test_guest_mode.py @@ -658,9 +658,6 @@ async def trio_main() -> None: # Ensure we don't pollute the thread-level context if run under # an asyncio without contextvars support (3.6) context = contextvars.copy_context() - if TYPE_CHECKING: - aiotrio_run(trio_main, host_uses_signal_set_wakeup_fd=True) - # this type error is a bug in typeshed or mypy, as it's equivalent to the above line - context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) # type: ignore[arg-type] + context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) assert record == {("asyncio", "asyncio"), ("trio", "trio")} From 13d5d63eba2a44c3110f1300a6a9d82e3e99b11f Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 19 Nov 2023 10:03:04 +1000 Subject: [PATCH 06/26] Fix return types here --- src/trio/_tests/test_subprocess.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/trio/_tests/test_subprocess.py b/src/trio/_tests/test_subprocess.py index de60c3e90d..95fa9e3a44 100644 --- a/src/trio/_tests/test_subprocess.py +++ b/src/trio/_tests/test_subprocess.py @@ -41,7 +41,7 @@ from typing_extensions import TypeAlias - from .._abc import Stream + from .._abc import ReceiveStream if sys.platform == "win32": SignalType: TypeAlias = None @@ -219,7 +219,7 @@ async def feed_input() -> None: await proc.stdin.send_all(msg) await proc.stdin.aclose() - async def check_output(stream: Stream, expected: bytes) -> None: + async def check_output(stream: ReceiveStream, expected: bytes) -> None: seen = bytearray() async for chunk in stream: seen += chunk @@ -272,7 +272,9 @@ async def test_interactive(background_process: BackgroundProcessType) -> None: async def expect(idx: int, request: int) -> None: async with _core.open_nursery() as nursery: - async def drain_one(stream: Stream, count: int, digit: int) -> None: + async def drain_one( + stream: ReceiveStream, count: int, digit: int + ) -> None: while count > 0: result = await stream.receive_some(count) assert result == (f"{digit}".encode() * len(result)) @@ -614,9 +616,7 @@ async def test_warn_on_cancel_SIGKILL_escalation( async def test_run_process_background_fail() -> None: with pytest.raises(subprocess.CalledProcessError): async with _core.open_nursery() as nursery: - proc: subprocess.CompletedProcess[bytes] = await nursery.start( - run_process, EXIT_FALSE - ) + proc: Process = await nursery.start(run_process, EXIT_FALSE) assert proc.returncode == 1 From 9ed1d08cd73b1e89dd5f5bb71d92714bf8c8c120 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 19 Nov 2023 11:24:37 +1000 Subject: [PATCH 07/26] Fix more errors --- src/trio/_core/_run.py | 2 +- src/trio/_tests/test_util.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index 367509a4f8..f1f545722a 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -2421,7 +2421,7 @@ def my_done_callback(run_outcome): # straight through. def unrolled_run( runner: Runner, - async_fn: Callable[[Unpack[PosArgT]], object], + async_fn: Callable[[Unpack[PosArgT]], Awaitable[object]], args: tuple[Unpack[PosArgT]], host_uses_signal_set_wakeup_fd: bool = False, ) -> Generator[float, EventResult, None]: diff --git a/src/trio/_tests/test_util.py b/src/trio/_tests/test_util.py index 40c2fd11bb..a593954639 100644 --- a/src/trio/_tests/test_util.py +++ b/src/trio/_tests/test_util.py @@ -162,8 +162,8 @@ async def async_gen(_: object) -> Any: # pragma: no cover def test_generic_function() -> None: - @generic_function - def test_func(arg: T) -> T: + @generic_function # Decorated function contains "Any". + def test_func(arg: T) -> T: # type: ignore[misc] """Look, a docstring!""" return arg From 093371833a5882bb27a5568f660e404b89cb4a90 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 19 Nov 2023 11:38:16 +1000 Subject: [PATCH 08/26] Bump Mypy version --- src/trio/_core/_entry_queue.py | 2 +- test-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/trio/_core/_entry_queue.py b/src/trio/_core/_entry_queue.py index f92166d3bc..582441e7d8 100644 --- a/src/trio/_core/_entry_queue.py +++ b/src/trio/_core/_entry_queue.py @@ -2,7 +2,7 @@ import threading from collections import deque -from typing import TYPE_CHECKING, Callable, Iterable, NoReturn, Tuple +from typing import TYPE_CHECKING, Callable, NoReturn, Tuple import attr diff --git a/test-requirements.txt b/test-requirements.txt index 0dc9d07a15..f6f821e089 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -54,7 +54,7 @@ jedi==0.19.1 # via -r test-requirements.in mccabe==0.7.0 # via pylint -mypy==1.6.1 ; implementation_name == "cpython" +mypy==1.7.0 ; implementation_name == "cpython" # via -r test-requirements.in mypy-extensions==1.0.0 ; implementation_name == "cpython" # via From a0e99666228fb7e423509cd2736245eb999f1f65 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 19 Nov 2023 13:31:22 +1000 Subject: [PATCH 09/26] Fix docs failure --- docs/source/conf.py | 3 ++- src/trio/_dtls.py | 25 +++++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index af358abf15..1e508997ca 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -73,7 +73,6 @@ # aliasing doesn't actually fix the warning for types.FrameType, but displaying # "types.FrameType" is more helpful than just "frame" "FrameType": "types.FrameType", - "Context": "OpenSSL.SSL.Context", # SSLListener.accept's return type is seen as trio._ssl.SSLStream "SSLStream": "trio.SSLStream", } @@ -91,6 +90,8 @@ def autodoc_process_signature( # Strip the type from the union, make it look like = ... signature = signature.replace(" | type[trio._core._local._NoValue]", "") signature = signature.replace("", "...") + if "DTLS" in name: + signature = signature.replace("SSL.Context", "OpenSSL.SSL.Context") # Don't specify PathLike[str] | PathLike[bytes], this is just for humans. signature = signature.replace("StrOrBytesPath", "str | bytes | os.PathLike") diff --git a/src/trio/_dtls.py b/src/trio/_dtls.py index fb99167423..4ad6d21751 100644 --- a/src/trio/_dtls.py +++ b/src/trio/_dtls.py @@ -41,7 +41,6 @@ # See DTLSEndpoint.__init__ for why this is imported here from OpenSSL import SSL # noqa: TCH004 - from OpenSSL.SSL import Context from typing_extensions import Self, TypeAlias, TypeVarTuple, Unpack from trio.socket import SocketType @@ -832,7 +831,12 @@ class DTLSChannel(trio.abc.Channel[bytes], metaclass=NoPublicConstructor): """ - def __init__(self, endpoint: DTLSEndpoint, peer_address: Any, ctx: Context): + def __init__( + self, + endpoint: DTLSEndpoint, + peer_address: Any, + ctx: SSL.Context, + ) -> None: self.endpoint = endpoint self.peer_address = peer_address self._packets_dropped_in_trio = 0 @@ -1178,7 +1182,12 @@ class DTLSEndpoint: """ - def __init__(self, socket: SocketType, *, incoming_packets_buffer: int = 10): + def __init__( + self, + socket: SocketType, + *, + incoming_packets_buffer: int = 10, + ) -> None: # We do this lazily on first construction, so only people who actually use DTLS # have to install PyOpenSSL. global SSL @@ -1199,7 +1208,7 @@ def __init__(self, socket: SocketType, *, incoming_packets_buffer: int = 10): # old connection. # {remote address: DTLSChannel} self._streams: WeakValueDictionary[Any, DTLSChannel] = WeakValueDictionary() - self._listening_context: Context | None = None + self._listening_context: SSL.Context | None = None self._listening_key: bytes | None = None self._incoming_connections_q = _Queue[DTLSChannel](float("inf")) self._send_lock = trio.Lock() @@ -1262,7 +1271,7 @@ def _check_closed(self) -> None: async def serve( self, - ssl_context: Context, + ssl_context: SSL.Context, async_fn: Callable[[DTLSChannel, Unpack[PosArgsT]], Awaitable[object]], *args: Unpack[PosArgsT], task_status: trio.TaskStatus[None] = trio.TASK_STATUS_IGNORED, @@ -1324,7 +1333,11 @@ async def handler_wrapper(stream: DTLSChannel) -> None: finally: self._listening_context = None - def connect(self, address: tuple[str, int], ssl_context: Context) -> DTLSChannel: + def connect( + self, + address: tuple[str, int], + ssl_context: SSL.Context, + ) -> DTLSChannel: """Initiate an outgoing DTLS connection. Notice that this is a synchronous method. That's because it doesn't actually From 5899ad4d2764ff1b359237777d6522be6ba17fe1 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 19 Nov 2023 14:14:34 +1000 Subject: [PATCH 10/26] Add type tests for Nursery.start[_soon] methods --- pyproject.toml | 1 + src/trio/_core/type_tests/nursery_start.py | 99 ++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/trio/_core/type_tests/nursery_start.py diff --git a/pyproject.toml b/pyproject.toml index cb24dd9eb8..c752101dbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,7 @@ extend-exclude = [ 'src/trio/lowlevel.py' = ['F401'] 'src/trio/socket.py' = ['F401'] 'src/trio/testing/__init__.py' = ['F401'] +'**/type_tests/*' = ['F841'] # Annotated local vars are used to specify return values. [tool.ruff.isort] combine-as-imports = true diff --git a/src/trio/_core/type_tests/nursery_start.py b/src/trio/_core/type_tests/nursery_start.py new file mode 100644 index 0000000000..ef3d96a8c4 --- /dev/null +++ b/src/trio/_core/type_tests/nursery_start.py @@ -0,0 +1,99 @@ +"""Test variadic generic typing for Nursery.start[_soon]().""" +from trio import TASK_STATUS_IGNORED, Nursery, TaskStatus +from typing_extensions import assert_type + + +async def task_0() -> None: + ... + + +async def task_1a(value: int) -> None: + ... + + +async def task_1b(value: str) -> None: + ... + + +async def task_2a(a: int, b: str) -> None: + ... + + +async def task_2b(a: str, b: int) -> None: + ... + + +async def task_2c(a: str, b: int, optional: bool = False) -> None: + ... + + +async def task_requires_kw(a: int, *, b: bool) -> None: + ... + + +async def task_startable_1( + a: str, + *, + task_status: TaskStatus[bool] = TASK_STATUS_IGNORED, +) -> None: + ... + + +async def task_startable_2( + a: str, + b: float, + *, + task_status: TaskStatus[bool] = TASK_STATUS_IGNORED, +) -> None: + ... + + +async def task_requires_start(*, task_status: TaskStatus[str]) -> None: + """Check a function requiring start() to be used.""" + + +async def task_pos_or_kw(value: str, task_status: TaskStatus[int]) -> None: + """Check a function which doesn't use the *-syntax works.""" + ... + + +def check_start_soon(nursery: Nursery) -> None: + """start_soon() functionality.""" + nursery.start_soon(task_0) + nursery.start_soon(task_1a) # type: ignore + nursery.start_soon(task_2b) # type: ignore + + nursery.start_soon(task_0, 45) # type: ignore + nursery.start_soon(task_1a, 32) + nursery.start_soon(task_1b, 32) # type: ignore + nursery.start_soon(task_1a, "abc") # type: ignore + nursery.start_soon(task_1b, "abc") + + nursery.start_soon(task_2b, "abc") # type: ignore + nursery.start_soon(task_2a, 38, "46") + nursery.start_soon(task_2c, "abc", 12) + nursery.start_soon(task_2c, "abc", 12, True) + + nursery.start_soon(task_requires_kw, 12, True) # type: ignore + # Tasks following the start() API can be made to work. + nursery.start_soon(task_startable_1, "cdf") + + +async def check_start(nursery: Nursery) -> None: + """start() functionality.""" + # Works with and without an explicit return type hint. + res_annotated: bool = await nursery.start(task_startable_1, "hello") + assert_type(res_annotated, bool) + + res_unann = await nursery.start(task_startable_1, "hello") + assert_type(res_unann, bool) + + # Check discarding the return value works. + await nursery.start(task_startable_2, "abc", 3.14) + + # Doesn't match the return type. + res_wrong_type: str = await nursery.start(task_startable_1, "hello") # type: ignore + + # Check variations on the function definition. + res_str: str = await nursery.start(task_requires_start) + res_int: int = await nursery.start(task_pos_or_kw, "abc") From e12273f963559120a7fe70ba0388449ebe45b73f Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 23 Nov 2023 12:11:10 +1000 Subject: [PATCH 11/26] Revert using TypeVarTuple for Nursery.start() --- src/trio/_core/_run.py | 12 +++--------- src/trio/_core/_tests/test_run.py | 2 +- src/trio/_core/type_tests/nursery_start.py | 21 --------------------- 3 files changed, 4 insertions(+), 31 deletions(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index f1f545722a..21730d9b2e 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1178,8 +1178,8 @@ def start_soon( async def start( self, - async_fn: _NurseryStartFunc[Unpack[PosArgT], StatusT], - *args: Unpack[PosArgT], + async_fn: Callable[..., Awaitable[object]], + *args: object, name: object = None, ) -> StatusT: r"""Creates and initializes a child task. @@ -1231,13 +1231,7 @@ async def async_fn(arg1, arg2, *, task_status=trio.TASK_STATUS_IGNORED): # exception in an extra ExceptionGroup. See #2611. async with open_nursery(strict_exception_groups=False) as old_nursery: task_status: _TaskStatus[StatusT] = _TaskStatus(old_nursery, self) - # Without the task_status keyword argument, this is just a positional-only function. - thunk: Callable[ - [Unpack[PosArgT]], Awaitable[object] - ] = functools.partial( # type: ignore[assignment] - async_fn, - task_status=task_status, - ) + thunk = functools.partial(async_fn, task_status=task_status) task = GLOBAL_RUN_CONTEXT.runner.spawn_impl( thunk, args, old_nursery, name ) diff --git a/src/trio/_core/_tests/test_run.py b/src/trio/_core/_tests/test_run.py index 5e0df641bc..e0ce898f41 100644 --- a/src/trio/_core/_tests/test_run.py +++ b/src/trio/_core/_tests/test_run.py @@ -1675,7 +1675,7 @@ async def no_args() -> None: # pragma: no cover # Errors in calling convention get raised immediately from start async with _core.open_nursery() as nursery: with pytest.raises(TypeError): - await nursery.start(no_args) # type: ignore[arg-type] + await nursery.start(no_args) async def sleep_then_start( seconds: int, *, task_status: _core.TaskStatus[int] = _core.TASK_STATUS_IGNORED diff --git a/src/trio/_core/type_tests/nursery_start.py b/src/trio/_core/type_tests/nursery_start.py index ef3d96a8c4..635a5bb944 100644 --- a/src/trio/_core/type_tests/nursery_start.py +++ b/src/trio/_core/type_tests/nursery_start.py @@ -1,6 +1,5 @@ """Test variadic generic typing for Nursery.start[_soon]().""" from trio import TASK_STATUS_IGNORED, Nursery, TaskStatus -from typing_extensions import assert_type async def task_0() -> None: @@ -77,23 +76,3 @@ def check_start_soon(nursery: Nursery) -> None: nursery.start_soon(task_requires_kw, 12, True) # type: ignore # Tasks following the start() API can be made to work. nursery.start_soon(task_startable_1, "cdf") - - -async def check_start(nursery: Nursery) -> None: - """start() functionality.""" - # Works with and without an explicit return type hint. - res_annotated: bool = await nursery.start(task_startable_1, "hello") - assert_type(res_annotated, bool) - - res_unann = await nursery.start(task_startable_1, "hello") - assert_type(res_unann, bool) - - # Check discarding the return value works. - await nursery.start(task_startable_2, "abc", 3.14) - - # Doesn't match the return type. - res_wrong_type: str = await nursery.start(task_startable_1, "hello") # type: ignore - - # Check variations on the function definition. - res_str: str = await nursery.start(task_requires_start) - res_int: int = await nursery.start(task_pos_or_kw, "abc") From 5a39c8b4829bf202364ab1ecd655f968618daf08 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 23 Nov 2023 12:29:43 +1000 Subject: [PATCH 12/26] This seems to be a mypy bug? --- src/trio/_core/_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index 21730d9b2e..83a07aeacd 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1731,7 +1731,8 @@ def spawn_impl( # Call the function and get the coroutine object, while giving helpful # errors for common mistakes. ###### - coro = context.run(coroutine_or_error, async_fn, *args) + # TypeVarTuple passed into ParamSpec function confuses Mypy. + coro = context.run(coroutine_or_error, async_fn, *args) # type: ignore[arg-type] if name is None: name = async_fn From 5b0215c0fc90232add320b1d1e7ce473a99c3857 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 23 Nov 2023 12:40:09 +1000 Subject: [PATCH 13/26] This type-error is resolved in Mypy master --- src/trio/_core/_tests/test_guest_mode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/trio/_core/_tests/test_guest_mode.py b/src/trio/_core/_tests/test_guest_mode.py index aa912ab70e..ee73bbb03a 100644 --- a/src/trio/_core/_tests/test_guest_mode.py +++ b/src/trio/_core/_tests/test_guest_mode.py @@ -658,6 +658,7 @@ async def trio_main() -> None: # Ensure we don't pollute the thread-level context if run under # an asyncio without contextvars support (3.6) context = contextvars.copy_context() - context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) + # Will be fixed in mypy 1.8 (https://github.com/python/mypy/pull/15896) + context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) # type: ignore[arg-type] assert record == {("asyncio", "asyncio"), ("trio", "trio")} From 342733c55b1608de4405de13a9ea68b6f0ca2e34 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 23 Nov 2023 12:54:08 +1000 Subject: [PATCH 14/26] Add missing import for PosArgT in _generated_run --- src/trio/_core/_generated_run.py | 6 ++++-- src/trio/_tools/gen_exports.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/trio/_core/_generated_run.py b/src/trio/_core/_generated_run.py index 1ec415497c..2b73630612 100644 --- a/src/trio/_core/_generated_run.py +++ b/src/trio/_core/_generated_run.py @@ -13,9 +13,11 @@ from collections.abc import Awaitable, Callable from outcome import Outcome + from typing_extensions import Unpack from .._abc import Clock from ._entry_queue import TrioToken + from ._run import PosArgT def current_statistics() -> RunStatistics: @@ -113,8 +115,8 @@ def reschedule(task: Task, next_send: Outcome[Any] = _NO_SEND) -> None: def spawn_system_task( - async_fn: Callable[..., Awaitable[object]], - *args: object, + async_fn: Callable[[Unpack[PosArgT]], Awaitable[object]], + *args: Unpack[PosArgT], name: object = None, context: (contextvars.Context | None) = None, ) -> Task: diff --git a/src/trio/_tools/gen_exports.py b/src/trio/_tools/gen_exports.py index 3227a06018..6c37564c29 100755 --- a/src/trio/_tools/gen_exports.py +++ b/src/trio/_tools/gen_exports.py @@ -346,7 +346,7 @@ def main() -> None: # pragma: no cover IMPORTS_RUN = """\ from collections.abc import Awaitable, Callable -from typing import Any +from typing import Any, TYPE_CHECKING from outcome import Outcome import contextvars @@ -354,6 +354,10 @@ def main() -> None: # pragma: no cover from ._run import _NO_SEND, RunStatistics, Task from ._entry_queue import TrioToken from .._abc import Clock + +if TYPE_CHECKING: + from typing_extensions import Unpack + from ._run import PosArgT """ IMPORTS_INSTRUMENT = """\ from ._instrumentation import Instrument From 749ef945502bb60ba95f96606e5278c2445b4b96 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 23 Nov 2023 13:09:45 +1000 Subject: [PATCH 15/26] The StatusT typevar can't be used here without TVT --- src/trio/_core/_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index 83a07aeacd..d36a399035 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -1181,7 +1181,7 @@ async def start( async_fn: Callable[..., Awaitable[object]], *args: object, name: object = None, - ) -> StatusT: + ) -> Any: r"""Creates and initializes a child task. Like :meth:`start_soon`, but blocks until the new task has @@ -1230,7 +1230,7 @@ async def async_fn(arg1, arg2, *, task_status=trio.TASK_STATUS_IGNORED): # `run` option, which would cause it to wrap a pre-started() # exception in an extra ExceptionGroup. See #2611. async with open_nursery(strict_exception_groups=False) as old_nursery: - task_status: _TaskStatus[StatusT] = _TaskStatus(old_nursery, self) + task_status: _TaskStatus[Any] = _TaskStatus(old_nursery, self) thunk = functools.partial(async_fn, task_status=task_status) task = GLOBAL_RUN_CONTEXT.runner.spawn_impl( thunk, args, old_nursery, name @@ -1243,7 +1243,7 @@ async def async_fn(arg1, arg2, *, task_status=trio.TASK_STATUS_IGNORED): # (Any exceptions propagate directly out of the above.) if task_status._value is _NoStatus: raise RuntimeError("child exited without calling task_status.started()") - return task_status._value # type: ignore[return-value] # Mypy doesn't narrow yet. + return task_status._value finally: self._pending_starts -= 1 self._check_nursery_closed() From 1a0daa625d899595aaeb2bf84390dd8a26b27b67 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 30 Nov 2023 16:24:26 +1000 Subject: [PATCH 16/26] Make gen_exports create an __all__ list --- src/trio/_core/_generated_instrumentation.py | 2 ++ src/trio/_core/_generated_io_epoll.py | 3 +++ src/trio/_core/_generated_io_kqueue.py | 10 ++++++++++ src/trio/_core/_generated_io_windows.py | 13 +++++++++++++ src/trio/_core/_generated_run.py | 12 ++++++++++++ src/trio/_tools/gen_exports.py | 6 ++++++ 6 files changed, 46 insertions(+) diff --git a/src/trio/_core/_generated_instrumentation.py b/src/trio/_core/_generated_instrumentation.py index debd1e7bb5..e9c7250f6e 100644 --- a/src/trio/_core/_generated_instrumentation.py +++ b/src/trio/_core/_generated_instrumentation.py @@ -11,6 +11,8 @@ if TYPE_CHECKING: from ._instrumentation import Instrument +__all__ = ["add_instrument", "remove_instrument"] + def add_instrument(instrument: Instrument) -> None: """Start instrumenting the current run loop with the given instrument. diff --git a/src/trio/_core/_generated_io_epoll.py b/src/trio/_core/_generated_io_epoll.py index d2547e7619..704a67d557 100644 --- a/src/trio/_core/_generated_io_epoll.py +++ b/src/trio/_core/_generated_io_epoll.py @@ -15,6 +15,9 @@ assert not TYPE_CHECKING or sys.platform == "linux" +__all__ = ["notify_closing", "wait_readable", "wait_writable"] + + async def wait_readable(fd: (int | _HasFileNo)) -> None: """Block until the kernel reports that the given object is readable. diff --git a/src/trio/_core/_generated_io_kqueue.py b/src/trio/_core/_generated_io_kqueue.py index 18467f0447..39662fd902 100644 --- a/src/trio/_core/_generated_io_kqueue.py +++ b/src/trio/_core/_generated_io_kqueue.py @@ -19,6 +19,16 @@ assert not TYPE_CHECKING or sys.platform == "darwin" +__all__ = [ + "current_kqueue", + "monitor_kevent", + "notify_closing", + "wait_kevent", + "wait_readable", + "wait_writable", +] + + def current_kqueue() -> select.kqueue: """TODO: these are implemented, but are currently more of a sketch than anything real. See `#26 diff --git a/src/trio/_core/_generated_io_windows.py b/src/trio/_core/_generated_io_windows.py index b705a77267..bb23e630c2 100644 --- a/src/trio/_core/_generated_io_windows.py +++ b/src/trio/_core/_generated_io_windows.py @@ -19,6 +19,19 @@ assert not TYPE_CHECKING or sys.platform == "win32" +__all__ = [ + "current_iocp", + "monitor_completion_key", + "notify_closing", + "readinto_overlapped", + "register_with_iocp", + "wait_overlapped", + "wait_readable", + "wait_writable", + "write_overlapped", +] + + async def wait_readable(sock: (_HasFileNo | int)) -> None: """Block until the kernel reports that the given object is readable. diff --git a/src/trio/_core/_generated_run.py b/src/trio/_core/_generated_run.py index 2b73630612..30b6e7d1c7 100644 --- a/src/trio/_core/_generated_run.py +++ b/src/trio/_core/_generated_run.py @@ -20,6 +20,18 @@ from ._run import PosArgT +__all__ = [ + "current_clock", + "current_root_task", + "current_statistics", + "current_time", + "current_trio_token", + "reschedule", + "spawn_system_task", + "wait_all_tasks_blocked", +] + + def current_statistics() -> RunStatistics: """Returns ``RunStatistics``, which contains run-loop-level debugging information. diff --git a/src/trio/_tools/gen_exports.py b/src/trio/_tools/gen_exports.py index 6c37564c29..7156d7015d 100755 --- a/src/trio/_tools/gen_exports.py +++ b/src/trio/_tools/gen_exports.py @@ -220,10 +220,12 @@ def gen_public_wrappers_source(file: File) -> str: generated = ["".join(header)] source = astor.code_to_ast.parse_file(file.path) + method_names = [] for method in get_public_methods(source): # Remove self from arguments assert method.args.args[0].arg == "self" del method.args.args[0] + method_names.append(method.name) for dec in method.decorator_list: # pragma: no cover if isinstance(dec, ast.Name) and dec.id == "contextmanager": @@ -263,6 +265,10 @@ def gen_public_wrappers_source(file: File) -> str: # Append the snippet to the corresponding module generated.append(snippet) + + method_names.sort() + # Insert after the header, before function definitions + generated.insert(1, f"__all__ = {method_names!r}") return "\n\n".join(generated) From b91b4b9d584f806cd028e093ca2646b7d8a8ad92 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 30 Nov 2023 18:23:12 +1000 Subject: [PATCH 17/26] Run _core/_tests/type_tests under Pyright also --- check.sh | 1 + src/trio/_core/_tests/type_tests/__init__.py | 0 src/trio/_core/{ => _tests}/type_tests/nursery_start.py | 0 3 files changed, 1 insertion(+) create mode 100644 src/trio/_core/_tests/type_tests/__init__.py rename src/trio/_core/{ => _tests}/type_tests/nursery_start.py (100%) diff --git a/check.sh b/check.sh index badad99127..3e07056dcd 100755 --- a/check.sh +++ b/check.sh @@ -110,6 +110,7 @@ if [ $PYRIGHT -ne 0 ]; then fi pyright src/trio/_tests/type_tests || EXIT_STATUS=$? +pyright src/trio/_core/_tests/type_tests || EXIT_STATUS=$? echo "::endgroup::" # Finally, leave a really clear warning of any issues and exit diff --git a/src/trio/_core/_tests/type_tests/__init__.py b/src/trio/_core/_tests/type_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/trio/_core/type_tests/nursery_start.py b/src/trio/_core/_tests/type_tests/nursery_start.py similarity index 100% rename from src/trio/_core/type_tests/nursery_start.py rename to src/trio/_core/_tests/type_tests/nursery_start.py From 972961e718be3de0af81229ad3b24c25b6e5e767 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 30 Nov 2023 18:34:23 +1000 Subject: [PATCH 18/26] Add newsfragment --- newsfragments/2881.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/2881.feature.rst diff --git a/newsfragments/2881.feature.rst b/newsfragments/2881.feature.rst new file mode 100644 index 0000000000..4e8efe47f3 --- /dev/null +++ b/newsfragments/2881.feature.rst @@ -0,0 +1 @@ +`TypeVarTuple `_ is now used to fully type :meth:`nursery.start_soon() `, :func:`trio.run()`, :func:`trio.to_thread.run_sync()`, and other similar functions accepting ``(func, *args)``. This means type checkers will be able to verify types are used correctly. :meth:`nursery.start() ` is not fully typed yet however. From bbd436de90a0d80f9ec51953cb00ff4f61aa2f94 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 30 Nov 2023 19:20:16 +1000 Subject: [PATCH 19/26] Default args are intentionally not considered by Pyright with TypeVarTuple --- pyproject.toml | 1 + src/trio/_core/_tests/type_tests/nursery_start.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c752101dbf..8c7abf3cd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,7 @@ check_untyped_defs = true [tool.pyright] pythonVersion = "3.8" +reportUnnecessaryTypeIgnoreComment = true [tool.pytest.ini_options] addopts = ["--strict-markers", "--strict-config", "-p trio._tests.pytest_plugin"] diff --git a/src/trio/_core/_tests/type_tests/nursery_start.py b/src/trio/_core/_tests/type_tests/nursery_start.py index 635a5bb944..cfcf603df8 100644 --- a/src/trio/_core/_tests/type_tests/nursery_start.py +++ b/src/trio/_core/_tests/type_tests/nursery_start.py @@ -1,4 +1,6 @@ """Test variadic generic typing for Nursery.start[_soon]().""" +from typing import Awaitable, Callable + from trio import TASK_STATUS_IGNORED, Nursery, TaskStatus @@ -70,9 +72,18 @@ def check_start_soon(nursery: Nursery) -> None: nursery.start_soon(task_2b, "abc") # type: ignore nursery.start_soon(task_2a, 38, "46") - nursery.start_soon(task_2c, "abc", 12) nursery.start_soon(task_2c, "abc", 12, True) + # Calling a 3-arg positional func, but using the default. + # Pyright intentionally ignores the default arg status here when converting + # callable -> typevartuple, but Mypy supports this. + # https://github.com/microsoft/pyright/issues/3775 + nursery.start_soon(task_2c, "abc", 12) # pyright: ignore + task_2c_cast: Callable[ + [str, int], Awaitable[object] + ] = task_2c # The assignment makes it work. + nursery.start_soon(task_2c_cast, "abc", 12) + nursery.start_soon(task_requires_kw, 12, True) # type: ignore # Tasks following the start() API can be made to work. nursery.start_soon(task_startable_1, "cdf") From bac655864317544c56123cacc4f1563934817bf2 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 1 Dec 2023 14:44:41 +1000 Subject: [PATCH 20/26] Type tests shouldn't be checked by coverage --- pyproject.toml | 2 ++ src/trio/_core/_tests/type_tests/__init__.py | 0 2 files changed, 2 insertions(+) delete mode 100644 src/trio/_core/_tests/type_tests/__init__.py diff --git a/pyproject.toml b/pyproject.toml index 8c7abf3cd0..6e3c16a91b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,6 +237,8 @@ omit = [ "*/trio/_core/_tests/test_multierror_scripts/*", # Omit the generated files in trio/_core starting with _generated_ "*/trio/_core/_generated_*", + # Type tests aren't intended to be run, just passed to type checkers. + "*/type_tests/*", ] # The test suite spawns subprocesses to test some stuff, so make sure # this doesn't corrupt the coverage files diff --git a/src/trio/_core/_tests/type_tests/__init__.py b/src/trio/_core/_tests/type_tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 From 9783f8bdceaaf9e2675c24ad835c50061bd0e360 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 1 Dec 2023 15:04:16 +1000 Subject: [PATCH 21/26] Handle Enum.__signature__ not being detected by Mypy --- src/trio/_tests/test_exports.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index c77a08f020..1d467b02cd 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -424,7 +424,11 @@ def lookup_symbol(symbol: str) -> dict[str, str]: and enum.Enum in class_.__mro__ and sys.version_info >= (3, 11) ): - extra.difference_update({"__copy__", "__deepcopy__"}) + extra.remove("__copy__") + extra.remove("__deepcopy__") + if sys.version_info >= (3, 12): + # Another attribute, in 3.12+ only. + extra.remove("__signature__") # TODO: this *should* be visible via `dir`!! if tool == "mypy" and class_ == trio.Nursery: From e1edda0b15e11f50234bd62dc71ac6a65968bf94 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 1 Dec 2023 15:08:23 +1000 Subject: [PATCH 22/26] Mypy does see copy/deepcopy now, this is no longer required --- src/trio/_tests/test_exports.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index 1d467b02cd..5c97e76f65 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -422,13 +422,10 @@ def lookup_symbol(symbol: str) -> dict[str, str]: if ( tool == "mypy" and enum.Enum in class_.__mro__ - and sys.version_info >= (3, 11) + and sys.version_info >= (3, 12) ): - extra.remove("__copy__") - extra.remove("__deepcopy__") - if sys.version_info >= (3, 12): - # Another attribute, in 3.12+ only. - extra.remove("__signature__") + # Another attribute, in 3.12+ only. + extra.remove("__signature__") # TODO: this *should* be visible via `dir`!! if tool == "mypy" and class_ == trio.Nursery: From 2903a59df577fa5801f609647e7160cbaaaf90b8 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Wed, 6 Dec 2023 15:39:23 -0600 Subject: [PATCH 23/26] Fix type issue from merge --- src/trio/_core/_tests/test_guest_mode.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/trio/_core/_tests/test_guest_mode.py b/src/trio/_core/_tests/test_guest_mode.py index 2bf7ce5580..aa912ab70e 100644 --- a/src/trio/_core/_tests/test_guest_mode.py +++ b/src/trio/_core/_tests/test_guest_mode.py @@ -658,9 +658,6 @@ async def trio_main() -> None: # Ensure we don't pollute the thread-level context if run under # an asyncio without contextvars support (3.6) context = contextvars.copy_context() - if TYPE_CHECKING: - aiotrio_run(trio_main, host_uses_signal_set_wakeup_fd=True) - # Will be fixed in mypy 1.8 (https://github.com/python/mypy/pull/15896) - context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) # type: ignore[arg-type] + context.run(aiotrio_run, trio_main, host_uses_signal_set_wakeup_fd=True) assert record == {("asyncio", "asyncio"), ("trio", "trio")} From d9d7a3d5d54a1640c495c3db35226c348f015d5b Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 7 Dec 2023 08:58:29 +1000 Subject: [PATCH 24/26] Remove unnecessary ignore Co-authored-by: EXPLOSION --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6e3c16a91b..1993ddd317 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,7 +127,6 @@ extend-exclude = [ 'src/trio/lowlevel.py' = ['F401'] 'src/trio/socket.py' = ['F401'] 'src/trio/testing/__init__.py' = ['F401'] -'**/type_tests/*' = ['F841'] # Annotated local vars are used to specify return values. [tool.ruff.isort] combine-as-imports = true From e012611d132b7ce66d8bed080406f1cbf361b5ca Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Tue, 12 Dec 2023 21:22:26 -0500 Subject: [PATCH 25/26] Pyright-related changes --- pyproject.toml | 1 + src/trio/_core/_tests/type_tests/nursery_start.py | 6 +----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1993ddd317..b4733d7de9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,7 @@ check_untyped_defs = true [tool.pyright] pythonVersion = "3.8" reportUnnecessaryTypeIgnoreComment = true +typeCheckingMode = "strict" [tool.pytest.ini_options] addopts = ["--strict-markers", "--strict-config", "-p trio._tests.pytest_plugin"] diff --git a/src/trio/_core/_tests/type_tests/nursery_start.py b/src/trio/_core/_tests/type_tests/nursery_start.py index cfcf603df8..c4a2915bf0 100644 --- a/src/trio/_core/_tests/type_tests/nursery_start.py +++ b/src/trio/_core/_tests/type_tests/nursery_start.py @@ -74,11 +74,7 @@ def check_start_soon(nursery: Nursery) -> None: nursery.start_soon(task_2a, 38, "46") nursery.start_soon(task_2c, "abc", 12, True) - # Calling a 3-arg positional func, but using the default. - # Pyright intentionally ignores the default arg status here when converting - # callable -> typevartuple, but Mypy supports this. - # https://github.com/microsoft/pyright/issues/3775 - nursery.start_soon(task_2c, "abc", 12) # pyright: ignore + nursery.start_soon(task_2c, "abc", 12) task_2c_cast: Callable[ [str, int], Awaitable[object] ] = task_2c # The assignment makes it work. From 9f94f9702b7f0219252b511f7b47bee5fd83daed Mon Sep 17 00:00:00 2001 From: EXPLOSION Date: Tue, 12 Dec 2023 21:27:40 -0500 Subject: [PATCH 26/26] Fix strict-mode error --- src/trio/_tests/type_tests/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trio/_tests/type_tests/path.py b/src/trio/_tests/type_tests/path.py index 321fd1043b..7c8c6de4a2 100644 --- a/src/trio/_tests/type_tests/path.py +++ b/src/trio/_tests/type_tests/path.py @@ -6,7 +6,7 @@ from typing import IO, Any, BinaryIO, List, Tuple import trio -from trio._path import _AsyncIOWrapper +from trio._path import _AsyncIOWrapper # pyright: ignore[reportPrivateUsage] from typing_extensions import assert_type