From fc6327086855eb77daf0a27b0e27e18c026a2dd3 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 11:13:36 +0100 Subject: [PATCH 01/11] Add some low-effort type annotations --- trio/_core/_run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 38359c3f7a..7ec9bb8c83 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1910,8 +1910,8 @@ def run( *args, clock=None, instruments=(), - restrict_keyboard_interrupt_to_checkpoints=False, - strict_exception_groups=False, + restrict_keyboard_interrupt_to_checkpoints: bool = False, + strict_exception_groups: bool = False, ): """Run a Trio-flavored async function, and return the result. @@ -2016,11 +2016,11 @@ def start_guest_run( run_sync_soon_threadsafe, done_callback, run_sync_soon_not_threadsafe=None, - host_uses_signal_set_wakeup_fd=False, + host_uses_signal_set_wakeup_fd: bool = False, clock=None, instruments=(), - restrict_keyboard_interrupt_to_checkpoints=False, - strict_exception_groups=False, + restrict_keyboard_interrupt_to_checkpoints: bool = False, + strict_exception_groups: bool = False, ): """Start a "guest" run of Trio on top of some other "host" event loop. @@ -2107,7 +2107,7 @@ def my_done_callback(run_outcome): # mode", where our core event loop gets unrolled into a series of callbacks on # the host loop. If you're doing a regular trio.run then this gets run # straight through. -def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd=False): +def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd: bool = False): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True From e306452de5287d11fb155fa4aa285caba853bfd9 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 16:20:42 +0100 Subject: [PATCH 02/11] Deal with mypy errors --- trio/_core/_run.py | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 7ec9bb8c83..46a898dd5f 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,7 +12,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, TYPE_CHECKING +from typing import Callable, Final, NoReturn, TYPE_CHECKING from sniffio import current_async_library_cvar @@ -46,9 +46,9 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup -DEADLINE_HEAP_MIN_PRUNE_THRESHOLD = 1000 +DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: Final = 1000 -_NO_SEND = object() +_NO_SEND: Final = object() # Decorator to mark methods public. This does nothing by itself, but @@ -62,21 +62,21 @@ def _public(fn): # variable to True, and registers the Random instance _r for Hypothesis # to manage for each test case, which together should make Trio's task # scheduling loop deterministic. We have a test for that, of course. -_ALLOW_DETERMINISTIC_SCHEDULING = False +_ALLOW_DETERMINISTIC_SCHEDULING: Final = False _r = random.Random() # On CPython, Context.run() is implemented in C and doesn't show up in # tracebacks. On PyPy, it is implemented in Python and adds 1 frame to tracebacks. -def _count_context_run_tb_frames(): - def function_with_unique_name_xyzzy(): +def _count_context_run_tb_frames() -> int: + def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] 1 / 0 ctx = copy_context() try: ctx.run(function_with_unique_name_xyzzy) except ZeroDivisionError as exc: - tb = exc.__traceback__ + tb = cast(TracebackType, exc.__traceback__) # Skip the frame where we caught it tb = tb.tb_next count = 0 @@ -84,9 +84,14 @@ def function_with_unique_name_xyzzy(): tb = tb.tb_next count += 1 return count + else: + raise TrioInternalError( + f"The purpose of {function_with_unique_name_xyzzy.__name__} is " + "to raise a ZeroDivisionError, but it didn't." + ) -CONTEXT_RUN_TB_FRAMES = _count_context_run_tb_frames() +CONTEXT_RUN_TB_FRAMES: Final = _count_context_run_tb_frames() @attr.s(frozen=True, slots=True) @@ -1118,7 +1123,7 @@ class Task(metaclass=NoPublicConstructor): name = attr.ib() # PEP 567 contextvars context context = attr.ib() - _counter = attr.ib(init=False, factory=itertools.count().__next__) + _counter: int = attr.ib(init=False, factory=itertools.count().__next__) # Invariant: # - for unscheduled tasks, _next_send_fn and _next_send are both None @@ -1245,7 +1250,7 @@ class RunContext(threading.local): task: Task -GLOBAL_RUN_CONTEXT = RunContext() +GLOBAL_RUN_CONTEXT: Final = RunContext() @attr.s(frozen=True) @@ -2100,14 +2105,19 @@ def my_done_callback(run_outcome): # 24 hours is arbitrary, but it avoids issues like people setting timeouts of # 10**20 and then getting integer overflows in the underlying system calls. -_MAX_TIMEOUT = 24 * 60 * 60 +_MAX_TIMEOUT: Final = 24 * 60 * 60 # Weird quirk: this is written as a generator in order to support "guest # mode", where our core event loop gets unrolled into a series of callbacks on # the host loop. If you're doing a regular trio.run then this gets run # straight through. -def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd: bool = False): +def unrolled_run( + runner: Runner, + async_fn, + args, + host_uses_signal_set_wakeup_fd: bool = False, +): locals()[LOCALS_KEY_KI_PROTECTION_ENABLED] = True __tracebackhide__ = True @@ -2254,8 +2264,10 @@ def unrolled_run(runner, async_fn, args, host_uses_signal_set_wakeup_fd: bool = # frame we always remove, because it's this function # catching it, and then in addition we remove however many # more Context.run adds. - tb = task_exc.__traceback__.tb_next - for _ in range(CONTEXT_RUN_TB_FRAMES): + tb = task_exc.__traceback__ + for _ in range(1 + CONTEXT_RUN_TB_FRAMES): + if tb is None: + break tb = tb.tb_next final_outcome = Error(task_exc.with_traceback(tb)) # Remove local refs so that e.g. cancelled coroutine locals @@ -2350,7 +2362,7 @@ def started(self, value=None): pass -TASK_STATUS_IGNORED = _TaskStatusIgnored() +TASK_STATUS_IGNORED: Final = _TaskStatusIgnored() def current_task(): From 7027c2b317ce4650ab91634dbb4414e789820464 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 16:36:33 +0100 Subject: [PATCH 03/11] Remove cast --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 46a898dd5f..069e2e9a23 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -76,7 +76,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] try: ctx.run(function_with_unique_name_xyzzy) except ZeroDivisionError as exc: - tb = cast(TracebackType, exc.__traceback__) + tb = exc.__traceback__ # Skip the frame where we caught it tb = tb.tb_next count = 0 From 390976f108e723c249d511d112e3cf0e268aecad Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 17:05:01 +0100 Subject: [PATCH 04/11] Import Final from typing_extensions --- trio/_core/_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 069e2e9a23..c46644ef37 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,7 +12,8 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, Final, NoReturn, TYPE_CHECKING +from typing import Callable, NoReturn, TYPE_CHECKING +from typing_extensions import Final from sniffio import current_async_library_cvar From 5ac1ba59c34b9e6a18bb13c21122398440498f85 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 20:42:30 +0100 Subject: [PATCH 05/11] Ensure mypy succeeds --- mypy.ini | 3 +-- trio/_core/_run.py | 52 +++++++++++++++++++++++++++++++--------------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/mypy.ini b/mypy.ini index 31eeef1cd0..a4ff21dd91 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,5 @@ [mypy] -# TODO: run mypy against several OS/version combos in CI -# https://mypy.readthedocs.io/en/latest/command_line.html#platform-configuration +show_error_codes = True # Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True diff --git a/trio/_core/_run.py b/trio/_core/_run.py index c46644ef37..5e17e4a595 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -13,7 +13,9 @@ from math import inf from time import perf_counter from typing import Callable, NoReturn, TYPE_CHECKING -from typing_extensions import Final +from typing import Deque + +from typing_extensions import Final as FinalT from sniffio import current_async_library_cvar @@ -47,9 +49,9 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup -DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: Final = 1000 +DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 -_NO_SEND: Final = object() +_NO_SEND: FinalT = object() # Decorator to mark methods public. This does nothing by itself, but @@ -63,15 +65,31 @@ def _public(fn): # variable to True, and registers the Random instance _r for Hypothesis # to manage for each test case, which together should make Trio's task # scheduling loop deterministic. We have a test for that, of course. -_ALLOW_DETERMINISTIC_SCHEDULING: Final = False +_ALLOW_DETERMINISTIC_SCHEDULING: FinalT = False _r = random.Random() -# On CPython, Context.run() is implemented in C and doesn't show up in -# tracebacks. On PyPy, it is implemented in Python and adds 1 frame to tracebacks. def _count_context_run_tb_frames() -> int: - def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] - 1 / 0 + """Count implementation dependent traceback frames from Context.run() + + On CPython, Context.run() is implemented in C and doesn't show up in + tracebacks. On PyPy, it is implemented in Python and adds 1 frame to + tracebacks. + + Returns: + int: Traceback frame count + + """ + + def function_with_unique_name_xyzzy() -> NoReturn: + try: + 1 / 0 + except ZeroDivisionError: + raise + else: + raise TrioInternalError( + "A ZeroDivisionError should have been raised, but it wasn't." + ) ctx = copy_context() try: @@ -79,10 +97,10 @@ def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] except ZeroDivisionError as exc: tb = exc.__traceback__ # Skip the frame where we caught it - tb = tb.tb_next + tb = tb.tb_next # type: ignore[union-attr] count = 0 - while tb.tb_frame.f_code.co_name != "function_with_unique_name_xyzzy": - tb = tb.tb_next + while tb.tb_frame.f_code.co_name != "function_with_unique_name_xyzzy": # type: ignore[union-attr] + tb = tb.tb_next # type: ignore[union-attr] count += 1 return count else: @@ -92,7 +110,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: # type: ignore[misc] ) -CONTEXT_RUN_TB_FRAMES: Final = _count_context_run_tb_frames() +CONTEXT_RUN_TB_FRAMES: FinalT = _count_context_run_tb_frames() @attr.s(frozen=True, slots=True) @@ -1251,7 +1269,7 @@ class RunContext(threading.local): task: Task -GLOBAL_RUN_CONTEXT: Final = RunContext() +GLOBAL_RUN_CONTEXT: FinalT = RunContext() @attr.s(frozen=True) @@ -1338,7 +1356,7 @@ class Runner: # Run-local values, see _local.py _locals = attr.ib(factory=dict) - runq = attr.ib(factory=deque) + runq: Deque[Task] = attr.ib(factory=deque) tasks = attr.ib(factory=set) deadlines = attr.ib(factory=Deadlines) @@ -2106,7 +2124,7 @@ def my_done_callback(run_outcome): # 24 hours is arbitrary, but it avoids issues like people setting timeouts of # 10**20 and then getting integer overflows in the underlying system calls. -_MAX_TIMEOUT: Final = 24 * 60 * 60 +_MAX_TIMEOUT: FinalT = 24 * 60 * 60 # Weird quirk: this is written as a generator in order to support "guest @@ -2137,7 +2155,7 @@ def unrolled_run( # here is our event loop: while runner.tasks: if runner.runq: - timeout = 0 + timeout: float = 0 else: deadline = runner.deadlines.next_deadline() timeout = runner.clock.deadline_to_sleep_time(deadline) @@ -2363,7 +2381,7 @@ def started(self, value=None): pass -TASK_STATUS_IGNORED: Final = _TaskStatusIgnored() +TASK_STATUS_IGNORED: FinalT = _TaskStatusIgnored() def current_task(): From 532f5e4519897296541dc091e2317fe52a0e3fc5 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Wed, 16 Nov 2022 21:14:29 +0100 Subject: [PATCH 06/11] Use pragmas to ignore coverage for defensive code --- trio/_core/_run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 5e17e4a595..3ce85b5cac 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -12,9 +12,10 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, NoReturn, TYPE_CHECKING +from typing import Callable, NoReturn, TypeVar, TYPE_CHECKING from typing import Deque +# An unfortunate name collision here with trio._util.Final from typing_extensions import Final as FinalT from sniffio import current_async_library_cvar @@ -49,6 +50,8 @@ if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup +T = TypeVar("T") + DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 _NO_SEND: FinalT = object() @@ -56,7 +59,7 @@ # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. -def _public(fn): +def _public(fn: T) -> T: return fn @@ -86,7 +89,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: 1 / 0 except ZeroDivisionError: raise - else: + else: # pragma: no cover raise TrioInternalError( "A ZeroDivisionError should have been raised, but it wasn't." ) @@ -103,7 +106,7 @@ def function_with_unique_name_xyzzy() -> NoReturn: tb = tb.tb_next # type: ignore[union-attr] count += 1 return count - else: + else: # pragma: no cover raise TrioInternalError( f"The purpose of {function_with_unique_name_xyzzy.__name__} is " "to raise a ZeroDivisionError, but it didn't." From 9be6779d62180aac190b6a6fa7326c631d0724db Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 22 Nov 2022 13:31:13 +0100 Subject: [PATCH 07/11] Update mypy config given defaults in new version of mypy --- mypy.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/mypy.ini b/mypy.ini index a4ff21dd91..6ccc92f5a4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,4 @@ [mypy] -show_error_codes = True - # Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True From f1aca01181f70e9fd2261e7e7a47d9f09401acad Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 11:01:16 +0100 Subject: [PATCH 08/11] Address feedback --- trio/_core/_run.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 588886858c..ef24065327 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -13,8 +13,7 @@ from contextvars import copy_context from math import inf from time import perf_counter -from typing import Callable, NoReturn, TypeVar, TYPE_CHECKING -from typing import Deque +from typing import Any, Deque, NoReturn, TypeVar, TYPE_CHECKING # An unfortunate name collision here with trio._util.Final from typing_extensions import Final as FinalT @@ -48,19 +47,23 @@ from .. import _core from .._util import Final, NoPublicConstructor, coroutine_or_error +if sys.version_info < (3, 9): + from typing import Callable +else: + from collections.abc import Callable + if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup -T = TypeVar("T") - DEADLINE_HEAP_MIN_PRUNE_THRESHOLD: FinalT = 1000 _NO_SEND: FinalT = object() +FnT = TypeVar("FnT", bound=Callable[..., Any]) # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. -def _public(fn: T) -> T: +def _public(fn: FnT) -> FnT: return fn @@ -122,18 +125,18 @@ class SystemClock: # Add a large random offset to our clock to ensure that if people # accidentally call time.perf_counter() directly or start comparing clocks # between different runs, then they'll notice the bug quickly: - offset = attr.ib(factory=lambda: _r.uniform(10000, 200000)) + offset: float = attr.ib(factory=lambda: _r.uniform(10000, 200000)) - def start_clock(self): + def start_clock(self) -> None: pass # In cPython 3, on every platform except Windows, perf_counter is # exactly the same as time.monotonic; and on Windows, it uses # QueryPerformanceCounter instead of GetTickCount64. - def current_time(self): + def current_time(self) -> float: return self.offset + perf_counter() - def deadline_to_sleep_time(self, deadline): + def deadline_to_sleep_time(self, deadline: float) -> float: return deadline - self.current_time() From b3a20f8c8d885652b418dba3143a5751075c85e1 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 12:39:44 +0100 Subject: [PATCH 09/11] Delay annotation eval --- trio/_core/_run.py | 62 ++++++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index ef24065327..166232e2db 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -1,56 +1,48 @@ +from __future__ import annotations + +import enum import functools +import gc import itertools import random import select import sys import threading -import gc +import warnings from collections import deque +from collections.abc import Callable from contextlib import contextmanager -import warnings -import enum - from contextvars import copy_context +from heapq import heapify, heappop, heappush from math import inf from time import perf_counter -from typing import Any, Deque, NoReturn, TypeVar, TYPE_CHECKING - -# An unfortunate name collision here with trio._util.Final -from typing_extensions import Final as FinalT - -from sniffio import current_async_library_cvar +from typing import TYPE_CHECKING, Any, NoReturn, TypeVar import attr -from heapq import heapify, heappop, heappush -from sortedcontainers import SortedDict from outcome import Error, Outcome, Value, capture +from sniffio import current_async_library_cvar +from sortedcontainers import SortedDict +# An unfortunate name collision here with trio._util.Final +from typing_extensions import Final as FinalT + +from .. import _core +from .._util import Final, NoPublicConstructor, coroutine_or_error +from ._asyncgens import AsyncGenerators from ._entry_queue import EntryQueue, TrioToken -from ._exceptions import TrioInternalError, RunFinishedError, Cancelled -from ._ki import ( - LOCALS_KEY_KI_PROTECTION_ENABLED, - KIManager, - enable_ki_protection, -) +from ._exceptions import Cancelled, RunFinishedError, TrioInternalError +from ._instrumentation import Instruments +from ._ki import LOCALS_KEY_KI_PROTECTION_ENABLED, KIManager, enable_ki_protection from ._multierror import MultiError, concat_tb +from ._thread_cache import start_thread_soon from ._traps import ( Abort, - wait_task_rescheduled, - cancel_shielded_checkpoint, CancelShieldedCheckpoint, PermanentlyDetachCoroutineObject, WaitTaskRescheduled, + cancel_shielded_checkpoint, + wait_task_rescheduled, ) -from ._asyncgens import AsyncGenerators -from ._thread_cache import start_thread_soon -from ._instrumentation import Instruments -from .. import _core -from .._util import Final, NoPublicConstructor, coroutine_or_error - -if sys.version_info < (3, 9): - from typing import Callable -else: - from collections.abc import Callable if sys.version_info < (3, 11): from exceptiongroup import BaseExceptionGroup @@ -1410,7 +1402,7 @@ class Runner: # Run-local values, see _local.py _locals = attr.ib(factory=dict) - runq: Deque[Task] = attr.ib(factory=deque) + runq: deque[Task] = attr.ib(factory=deque) tasks = attr.ib(factory=set) deadlines = attr.ib(factory=Deadlines) @@ -2530,16 +2522,16 @@ async def checkpoint_if_cancelled(): if sys.platform == "win32": - from ._io_windows import WindowsIOManager as TheIOManager from ._generated_io_windows import * + from ._io_windows import WindowsIOManager as TheIOManager elif sys.platform == "linux" or (not TYPE_CHECKING and hasattr(select, "epoll")): - from ._io_epoll import EpollIOManager as TheIOManager from ._generated_io_epoll import * + from ._io_epoll import EpollIOManager as TheIOManager elif TYPE_CHECKING or hasattr(select, "kqueue"): - from ._io_kqueue import KqueueIOManager as TheIOManager from ._generated_io_kqueue import * + from ._io_kqueue import KqueueIOManager as TheIOManager else: # pragma: no cover raise NotImplementedError("unsupported platform") -from ._generated_run import * from ._generated_instrumentation import * +from ._generated_run import * From c082f633d3b71eeb692c0297798e12742d34f5d5 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 12:43:29 +0100 Subject: [PATCH 10/11] Defer bound eval --- trio/_core/_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trio/_core/_run.py b/trio/_core/_run.py index 166232e2db..31ff874a40 100644 --- a/trio/_core/_run.py +++ b/trio/_core/_run.py @@ -51,7 +51,7 @@ _NO_SEND: FinalT = object() -FnT = TypeVar("FnT", bound=Callable[..., Any]) +FnT = TypeVar("FnT", bound="Callable[..., Any]") # Decorator to mark methods public. This does nothing by itself, but # trio/_tools/gen_exports.py looks for it. From fc4ed29fe20dbcd9619c7d2966465f0ee15f5e01 Mon Sep 17 00:00:00 2001 From: Harald Husum Date: Tue, 17 Jan 2023 12:47:17 +0100 Subject: [PATCH 11/11] Revert removal of incomplete todo --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 6ccc92f5a4..31eeef1cd0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,7 @@ [mypy] +# TODO: run mypy against several OS/version combos in CI +# https://mypy.readthedocs.io/en/latest/command_line.html#platform-configuration + # Be flexible about dependencies that don't have stubs yet (like pytest) ignore_missing_imports = True