From d5a3c29d66e917a96a94eafb9ded15ae1d42c1a8 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 12 Jul 2023 13:08:42 +1000 Subject: [PATCH 01/28] Add types to directly defined objects in _file_io --- trio/_file_io.py | 52 ++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 9f7d81adef..37381a8ce1 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import io from functools import partial +from typing import AnyStr, Generic, IO, Iterable, TYPE_CHECKING import trio @@ -7,7 +10,7 @@ from .abc import AsyncResource # This list is also in the docs, make sure to keep them in sync -_FILE_SYNC_ATTRS = { +_FILE_SYNC_ATTRS: set[str] = { "closed", "encoding", "errors", @@ -29,7 +32,7 @@ } # This list is also in the docs, make sure to keep them in sync -_FILE_ASYNC_METHODS = { +_FILE_ASYNC_METHODS: set[str] = { "flush", "read", "read1", @@ -48,7 +51,7 @@ } -class AsyncIOWrapper(AsyncResource): +class AsyncIOWrapper(AsyncResource, Generic[AnyStr]): """A generic :class:`~io.IOBase` wrapper that implements the :term:`asynchronous file object` interface. Wrapped methods that could block are executed in :meth:`trio.to_thread.run_sync`. @@ -58,39 +61,40 @@ class AsyncIOWrapper(AsyncResource): """ - def __init__(self, file): + def __init__(self, file: IO[AnyStr]) -> None: self._wrapped = file @property - def wrapped(self): + def wrapped(self) -> IO[AnyStr]: """object: A reference to the wrapped file object""" return self._wrapped - def __getattr__(self, name): - if name in _FILE_SYNC_ATTRS: - return getattr(self._wrapped, name) - if name in _FILE_ASYNC_METHODS: - meth = getattr(self._wrapped, name) + if not TYPE_CHECKING: + def __getattr__(self, name: str) -> object: + if name in _FILE_SYNC_ATTRS: + return getattr(self._wrapped, name) + if name in _FILE_ASYNC_METHODS: + meth = getattr(self._wrapped, name) - @async_wraps(self.__class__, self._wrapped.__class__, name) - async def wrapper(*args, **kwargs): - func = partial(meth, *args, **kwargs) - return await trio.to_thread.run_sync(func) + @async_wraps(self.__class__, self._wrapped.__class__, name) + async def wrapper(*args, **kwargs): + func = partial(meth, *args, **kwargs) + return await trio.to_thread.run_sync(func) - # cache the generated method - setattr(self, name, wrapper) - return wrapper + # cache the generated method + setattr(self, name, wrapper) + return wrapper - raise AttributeError(name) + raise AttributeError(name) - def __dir__(self): + def __dir__(self) -> Iterable[str]: attrs = set(super().__dir__()) attrs.update(a for a in _FILE_SYNC_ATTRS if hasattr(self.wrapped, a)) attrs.update(a for a in _FILE_ASYNC_METHODS if hasattr(self.wrapped, a)) return attrs - def __aiter__(self): + def __aiter__(self) -> AsyncIOWrapper[AnyStr]: return self async def __anext__(self): @@ -100,7 +104,7 @@ async def __anext__(self): else: raise StopAsyncIteration - async def detach(self): + async def detach(self) -> AsyncIOWrapper[AnyStr]: """Like :meth:`io.BufferedIOBase.detach`, but async. This also re-wraps the result in a new :term:`asynchronous file object` @@ -111,7 +115,7 @@ async def detach(self): raw = await trio.to_thread.run_sync(self._wrapped.detach) return wrap_file(raw) - async def aclose(self): + async def aclose(self) -> None: """Like :meth:`io.IOBase.close`, but async. This is also shielded from cancellation; if a cancellation scope is @@ -161,7 +165,7 @@ async def open_file( return _file -def wrap_file(file): +def wrap_file(file: IO[AnyStr]) -> AsyncIOWrapper[AnyStr]: """This wraps any file object in a wrapper that provides an asynchronous file object interface. @@ -179,7 +183,7 @@ def wrap_file(file): """ - def has(attr): + def has(attr: str) -> bool: return hasattr(file, attr) and callable(getattr(file, attr)) if not (has("close") and (has("read") or has("write"))): From ef272eb840d2368904ca969a6e6fe1c875e17d08 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 12 Jul 2023 14:16:37 +1000 Subject: [PATCH 02/28] Implement methods on AsyncIOWrapper using a pile of self-type-properties. --- trio/_file_io.py | 188 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 9 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 37381a8ce1..b3cbfa066a 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -2,7 +2,7 @@ import io from functools import partial -from typing import AnyStr, Generic, IO, Iterable, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, AnyStr, BinaryIO, Generic, Iterable, TypeVar import trio @@ -51,7 +51,130 @@ } -class AsyncIOWrapper(AsyncResource, Generic[AnyStr]): +FileT = TypeVar('FileT') +FileT_co = TypeVar('FileT_co', covariant=True) +T = TypeVar('T') +T_co = TypeVar('T_co', covariant=True) +T_contra = TypeVar('T_contra', contravariant=True) +AnyStr_co = TypeVar('AnyStr_co', str, bytes, covariant=True) +AnyStr_contra = TypeVar('AnyStr_contra', str, bytes, contravariant=True) + +# Define a protocol for every method/property we expose. Each is effectively a predicate checking +# whether a class defines this method/property. +if TYPE_CHECKING: + from typing_extensions import Buffer, Protocol + + class _HasClosed(Protocol): + @property + def closed(self) -> bool: ... + + class _HasEncoding(Protocol): + @property + def encoding(self) -> str: ... + + class _HasErrors(Protocol): + @property + def errors(self) -> str | None: ... + + class _HasFileNo(Protocol): + def fileno(self) -> int: ... + + class _HasIsATTY(Protocol): + def isatty(self) -> bool: ... + + class _HasNewlines(Protocol): + @property + def newlines(self) -> Any: ... # None, str or tuple + + class _HasReadable(Protocol): + def readable(self) -> bool: ... + + class _HasSeekable(Protocol): + def seekable(self) -> bool: ... + + class _HasWritable(Protocol): + def writable(self) -> bool: ... + + class _HasBuffer(Protocol): + @property + def buffer(self) -> BinaryIO: ... + + class _HasRaw(Protocol): + @property + def raw(self) -> io.RawIOBase: ... + + class _HasLineBuffering(Protocol): + @property + def line_buffering(self) -> bool: ... + + class _HasCloseFD(Protocol): + @property + def closefd(self) -> bool: ... + + class _HasName(Protocol): + @property + def name(self) -> str: ... + + class _HasMode(Protocol): + @property + def mode(self) -> str: ... + + class _CanGetValue(Protocol[AnyStr_co]): + def getvalue(self) -> AnyStr_co: ... + + class _CanGetBuffer(Protocol): + def getbuffer(self) -> memoryview: ... + + class _CanFlush(Protocol): + def flush(self) -> None: ... + + class _CanRead(Protocol[AnyStr_co]): + def read(self, __size: int | None = ...) -> AnyStr_co: ... + + class _CanRead1(Protocol): + def read1(self, __size: int | None = ...) -> bytes: ... + + class _CanReadAll(Protocol[AnyStr_co]): + def readall(self) -> AnyStr_co: ... + + class _CanReadInto(Protocol): + def readinto(self, __buf: Buffer) -> int | None: ... + + class _CanReadInto1(Protocol): + def readinto1(self, __buffer: Buffer) -> int: ... + + class _CanReadLine(Protocol[AnyStr_co]): + def readline(self, __size: int = ...) -> AnyStr_co: ... + + class _CanReadLines(Protocol[AnyStr]): + def readlines(self, __hint: int = ...) -> list[AnyStr]: ... + + class _CanSeek(Protocol): + def seek(self, __target: int, __whence: int = 0) -> int: ... + + class _CanTell(Protocol): + def tell(self) -> int: ... + + class _CanTruncate(Protocol): + def truncate(self, __size: int | None = ...) -> int: ... + + class _CanWrite(Protocol[AnyStr_contra]): + def write(self, __data: AnyStr_contra) -> int: ... + + class _CanWriteLines(Protocol[T_contra]): + def writelines(self, __lines: Iterable[T_contra]) -> None: ... + + class _CanPeek(Protocol[AnyStr_co]): + def peek(self, __size: int = 0) -> AnyStr_co: ... + + class _CanDetach(Protocol[T_co]): + def detach(self) -> T_co: ... + + class _CanClose(Protocol): + def close(self) -> None: ... + + +class AsyncIOWrapper(AsyncResource, Generic[FileT_co]): """A generic :class:`~io.IOBase` wrapper that implements the :term:`asynchronous file object` interface. Wrapped methods that could block are executed in :meth:`trio.to_thread.run_sync`. @@ -61,11 +184,11 @@ class AsyncIOWrapper(AsyncResource, Generic[AnyStr]): """ - def __init__(self, file: IO[AnyStr]) -> None: + def __init__(self, file: FileT_co) -> None: self._wrapped = file @property - def wrapped(self) -> IO[AnyStr]: + def wrapped(self) -> FileT_co: """object: A reference to the wrapped file object""" return self._wrapped @@ -94,17 +217,17 @@ def __dir__(self) -> Iterable[str]: attrs.update(a for a in _FILE_ASYNC_METHODS if hasattr(self.wrapped, a)) return attrs - def __aiter__(self) -> AsyncIOWrapper[AnyStr]: + def __aiter__(self) -> AsyncIOWrapper[FileT_co]: return self - async def __anext__(self): + async def __anext__(self: AsyncIOWrapper[_CanReadLine[AnyStr]]) -> AnyStr: line = await self.readline() if line: return line else: raise StopAsyncIteration - async def detach(self) -> AsyncIOWrapper[AnyStr]: + async def detach(self: AsyncIOWrapper[_CanDetach[T]]) -> AsyncIOWrapper[T]: """Like :meth:`io.BufferedIOBase.detach`, but async. This also re-wraps the result in a new :term:`asynchronous file object` @@ -115,7 +238,7 @@ async def detach(self) -> AsyncIOWrapper[AnyStr]: raw = await trio.to_thread.run_sync(self._wrapped.detach) return wrap_file(raw) - async def aclose(self) -> None: + async def aclose(self: AsyncIOWrapper[_CanClose]) -> None: """Like :meth:`io.IOBase.close`, but async. This is also shielded from cancellation; if a cancellation scope is @@ -129,6 +252,53 @@ async def aclose(self) -> None: await trio.lowlevel.checkpoint_if_cancelled() + if TYPE_CHECKING: + # Based on typing.IO and io stubs. + # For every method/property we type self, restricting these to only be available if + # the original also is. + @property + def closed(self: AsyncIOWrapper[_HasClosed]) -> bool: ... + @property + def encoding(self: AsyncIOWrapper[_HasEncoding]) -> str: ... + @property + def errors(self: AsyncIOWrapper[_HasErrors]) -> str | None: ... + @property + def newlines(self: AsyncIOWrapper[_HasNewlines]) -> Any: ... # None, str or tuple + @property + def buffer(self: AsyncIOWrapper[_HasBuffer]) -> BinaryIO: ... + @property + def raw(self: AsyncIOWrapper[_HasRaw]) -> io.RawIOBase: ... + @property + def line_buffering(self: AsyncIOWrapper[_HasLineBuffering]) -> int: ... + @property + def closefd(self: AsyncIOWrapper[_HasCloseFD]) -> bool: ... + @property + def name(self: AsyncIOWrapper[_HasName]) -> str: ... + @property + def mode(self: AsyncIOWrapper[_HasMode]) -> str: ... + + def fileno(self: AsyncIOWrapper[_HasFileNo]) -> int: ... + def isatty(self: AsyncIOWrapper[_HasIsATTY]) -> bool: ... + def readable(self: AsyncIOWrapper[_HasReadable]) -> bool: ... + def seekable(self: AsyncIOWrapper[_HasSeekable]) -> bool: ... + def writable(self: AsyncIOWrapper[_HasWritable]) -> bool: ... + def getvalue(self: AsyncIOWrapper[_CanGetValue[AnyStr]]) -> AnyStr: ... + def getbuffer(self: AsyncIOWrapper[_CanGetBuffer]) -> memoryview: ... + async def flush(self: AsyncIOWrapper[_CanFlush]) -> None: ... + async def read(self: AsyncIOWrapper[_CanRead[AnyStr]], __size: int | None = ...) -> AnyStr: ... + async def read1(self: AsyncIOWrapper[_CanRead1], __size: int | None = ...) -> bytes: ... + async def readall(self: AsyncIOWrapper[_CanReadAll[AnyStr]]) -> AnyStr: ... + async def readinto(self: AsyncIOWrapper[_CanReadInto], __buf: Buffer) -> int | None: ... + async def readline(self: AsyncIOWrapper[_CanReadLine[AnyStr]], __size: int = ...) -> AnyStr: ... + async def readlines(self: AsyncIOWrapper[_CanReadLines[AnyStr]]) -> list[AnyStr]: ... + async def seek(self: AsyncIOWrapper[_CanSeek], __target: int, __whence: int = 0) -> int: ... + async def tell(self: AsyncIOWrapper[_CanTell]) -> int: ... + async def truncate(self: AsyncIOWrapper[_CanTruncate], __size: int | None = ...) -> int: ... + async def write(self: AsyncIOWrapper[_CanWrite[AnyStr]], __data: AnyStr) -> int: ... + async def writelines(self: AsyncIOWrapper[_CanWriteLines[T]], __lines: Iterable[T]) -> None: ... + async def readinto1(self: AsyncIOWrapper[_CanReadInto1], __buffer: Buffer) -> int: ... + async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], __size: int = 0) -> AnyStr: ... + async def open_file( file, @@ -165,7 +335,7 @@ async def open_file( return _file -def wrap_file(file: IO[AnyStr]) -> AsyncIOWrapper[AnyStr]: +def wrap_file(file: FileT) -> AsyncIOWrapper[FileT]: """This wraps any file object in a wrapper that provides an asynchronous file object interface. From 822381a3e61f4b7d7a2a1be4d80004dc1cff8339 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 12 Jul 2023 15:28:36 +1000 Subject: [PATCH 03/28] Add types for open_file() --- trio/_file_io.py | 129 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 10 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index b3cbfa066a..acf6718028 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -2,13 +2,36 @@ import io from functools import partial -from typing import TYPE_CHECKING, Any, AnyStr, BinaryIO, Generic, Iterable, TypeVar +from typing import ( + IO, + TYPE_CHECKING, + Any, + AnyStr, + BinaryIO, + Callable, + Generic, + Iterable, + TypeVar, + Union, + overload, +) import trio from ._util import async_wraps from .abc import AsyncResource +if TYPE_CHECKING: + from _typeshed import ( + OpenBinaryMode, + OpenBinaryModeReading, + OpenBinaryModeUpdating, + OpenBinaryModeWriting, + OpenTextMode, + StrOrBytesPath, + ) + from typing_extensions import Literal + # This list is also in the docs, make sure to keep them in sync _FILE_SYNC_ATTRS: set[str] = { "closed", @@ -300,16 +323,102 @@ async def readinto1(self: AsyncIOWrapper[_CanReadInto1], __buffer: Buffer) -> in async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], __size: int = 0) -> AnyStr: ... +# Type hints are copied from builtin open. +_OpenFile = Union[StrOrBytesPath, int] +_Opener = Callable[[str, int], int] + + +@overload +async def open_file( + file: _OpenFile, + mode: OpenTextMode = ..., + buffering: int = ..., + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[io.TextIOWrapper]: ... +@overload +async def open_file( + file: _OpenFile, + mode: OpenBinaryMode, + buffering: Literal[0], + encoding: None = ..., + errors: None = ..., + newline: None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[io.FileIO]: ... +@overload +async def open_file( + file: _OpenFile, + mode: OpenBinaryModeUpdating, + buffering: Literal[-1, 1] = ..., + encoding: None = ..., + errors: None = ..., + newline: None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[io.BufferedRandom]: ... +@overload +async def open_file( + file: _OpenFile, + mode: OpenBinaryModeWriting, + buffering: Literal[-1, 1] = ..., + encoding: None = ..., + errors: None = ..., + newline: None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[io.BufferedWriter]: ... +@overload +async def open_file( + file: _OpenFile, + mode: OpenBinaryModeReading, + buffering: Literal[-1, 1] = ..., + encoding: None = ..., + errors: None = ..., + newline: None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[io.BufferedReader]: ... +@overload +async def open_file( + file: _OpenFile, + mode: OpenBinaryMode, + buffering: int, + encoding: None = ..., + errors: None = ..., + newline: None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[BinaryIO]: ... + +# Fallback if mode is not specified +@overload +async def open_file( + file: _OpenFile, + mode: str, + buffering: int = ..., + encoding: str | None = ..., + errors: str | None = ..., + newline: str | None = ..., + closefd: bool = ..., + opener: _Opener | None = ..., +) -> AsyncIOWrapper[IO[Any]]: ... + + async def open_file( - file, - mode="r", - buffering=-1, - encoding=None, - errors=None, - newline=None, - closefd=True, - opener=None, -): + file: _OpenFile, + mode: str = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: _Opener | None = None, +) -> AsyncIOWrapper: """Asynchronous version of :func:`io.open`. Returns: From c2fa19e4f33751eecd57b657dd0baf35d8d8adc9 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 12 Jul 2023 16:15:19 +1000 Subject: [PATCH 04/28] Set parameter defaults --- trio/_file_io.py | 92 ++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index acf6718028..abfc16aa6d 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -308,15 +308,15 @@ def writable(self: AsyncIOWrapper[_HasWritable]) -> bool: ... def getvalue(self: AsyncIOWrapper[_CanGetValue[AnyStr]]) -> AnyStr: ... def getbuffer(self: AsyncIOWrapper[_CanGetBuffer]) -> memoryview: ... async def flush(self: AsyncIOWrapper[_CanFlush]) -> None: ... - async def read(self: AsyncIOWrapper[_CanRead[AnyStr]], __size: int | None = ...) -> AnyStr: ... - async def read1(self: AsyncIOWrapper[_CanRead1], __size: int | None = ...) -> bytes: ... + async def read(self: AsyncIOWrapper[_CanRead[AnyStr]], __size: int | None = -1) -> AnyStr: ... + async def read1(self: AsyncIOWrapper[_CanRead1], __size: int | None = -1) -> bytes: ... async def readall(self: AsyncIOWrapper[_CanReadAll[AnyStr]]) -> AnyStr: ... async def readinto(self: AsyncIOWrapper[_CanReadInto], __buf: Buffer) -> int | None: ... - async def readline(self: AsyncIOWrapper[_CanReadLine[AnyStr]], __size: int = ...) -> AnyStr: ... + async def readline(self: AsyncIOWrapper[_CanReadLine[AnyStr]], __size: int = -1) -> AnyStr: ... async def readlines(self: AsyncIOWrapper[_CanReadLines[AnyStr]]) -> list[AnyStr]: ... async def seek(self: AsyncIOWrapper[_CanSeek], __target: int, __whence: int = 0) -> int: ... async def tell(self: AsyncIOWrapper[_CanTell]) -> int: ... - async def truncate(self: AsyncIOWrapper[_CanTruncate], __size: int | None = ...) -> int: ... + async def truncate(self: AsyncIOWrapper[_CanTruncate], __size: int | None = None) -> int: ... async def write(self: AsyncIOWrapper[_CanWrite[AnyStr]], __data: AnyStr) -> int: ... async def writelines(self: AsyncIOWrapper[_CanWriteLines[T]], __lines: Iterable[T]) -> None: ... async def readinto1(self: AsyncIOWrapper[_CanReadInto1], __buffer: Buffer) -> int: ... @@ -331,68 +331,68 @@ async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], __size: int = 0) -> AnySt @overload async def open_file( file: _OpenFile, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + mode: OpenTextMode = 'r', + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[io.TextIOWrapper]: ... @overload async def open_file( file: _OpenFile, mode: OpenBinaryMode, buffering: Literal[0], - encoding: None = ..., - errors: None = ..., - newline: None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[io.FileIO]: ... @overload async def open_file( file: _OpenFile, mode: OpenBinaryModeUpdating, - buffering: Literal[-1, 1] = ..., - encoding: None = ..., - errors: None = ..., - newline: None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[io.BufferedRandom]: ... @overload async def open_file( file: _OpenFile, mode: OpenBinaryModeWriting, - buffering: Literal[-1, 1] = ..., - encoding: None = ..., - errors: None = ..., - newline: None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[io.BufferedWriter]: ... @overload async def open_file( file: _OpenFile, mode: OpenBinaryModeReading, - buffering: Literal[-1, 1] = ..., - encoding: None = ..., - errors: None = ..., - newline: None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[io.BufferedReader]: ... @overload async def open_file( file: _OpenFile, mode: OpenBinaryMode, buffering: int, - encoding: None = ..., - errors: None = ..., - newline: None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + encoding: None = None, + errors: None = None, + newline: None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[BinaryIO]: ... # Fallback if mode is not specified @@ -400,12 +400,12 @@ async def open_file( async def open_file( file: _OpenFile, mode: str, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: _Opener | None = ..., + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + closefd: bool = True, + opener: _Opener | None = None, ) -> AsyncIOWrapper[IO[Any]]: ... @@ -418,7 +418,7 @@ async def open_file( newline: str | None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper: +) -> AsyncIOWrapper[Any]: """Asynchronous version of :func:`io.open`. Returns: From bec946d940d8c913042141c2bd38ec7b89488490 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 12 Jul 2023 16:35:28 +1000 Subject: [PATCH 05/28] Reformat --- trio/_file_io.py | 55 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index abfc16aa6d..8bd80864c4 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -74,19 +74,21 @@ } -FileT = TypeVar('FileT') -FileT_co = TypeVar('FileT_co', covariant=True) -T = TypeVar('T') -T_co = TypeVar('T_co', covariant=True) -T_contra = TypeVar('T_contra', contravariant=True) -AnyStr_co = TypeVar('AnyStr_co', str, bytes, covariant=True) -AnyStr_contra = TypeVar('AnyStr_contra', str, bytes, contravariant=True) +FileT = TypeVar("FileT") +FileT_co = TypeVar("FileT_co", covariant=True) +T = TypeVar("T") +T_co = TypeVar("T_co", covariant=True) +T_contra = TypeVar("T_contra", contravariant=True) +AnyStr_co = TypeVar("AnyStr_co", str, bytes, covariant=True) +AnyStr_contra = TypeVar("AnyStr_contra", str, bytes, contravariant=True) # Define a protocol for every method/property we expose. Each is effectively a predicate checking # whether a class defines this method/property. if TYPE_CHECKING: from typing_extensions import Buffer, Protocol + # fmt: off + class _HasClosed(Protocol): @property def closed(self) -> bool: ... @@ -217,6 +219,7 @@ def wrapped(self) -> FileT_co: return self._wrapped if not TYPE_CHECKING: + def __getattr__(self, name: str) -> object: if name in _FILE_SYNC_ATTRS: return getattr(self._wrapped, name) @@ -276,6 +279,7 @@ async def aclose(self: AsyncIOWrapper[_CanClose]) -> None: await trio.lowlevel.checkpoint_if_cancelled() if TYPE_CHECKING: + # fmt: off # Based on typing.IO and io stubs. # For every method/property we type self, restricting these to only be available if # the original also is. @@ -324,21 +328,24 @@ async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], __size: int = 0) -> AnySt # Type hints are copied from builtin open. -_OpenFile = Union[StrOrBytesPath, int] +_OpenFile = Union['StrOrBytesPath', int] _Opener = Callable[[str, int], int] @overload async def open_file( file: _OpenFile, - mode: OpenTextMode = 'r', + mode: OpenTextMode = "r", buffering: int = -1, encoding: str | None = None, errors: str | None = None, newline: str | None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[io.TextIOWrapper]: ... +) -> AsyncIOWrapper[io.TextIOWrapper]: + ... + + @overload async def open_file( file: _OpenFile, @@ -349,7 +356,10 @@ async def open_file( newline: None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[io.FileIO]: ... +) -> AsyncIOWrapper[io.FileIO]: + ... + + @overload async def open_file( file: _OpenFile, @@ -360,7 +370,10 @@ async def open_file( newline: None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[io.BufferedRandom]: ... +) -> AsyncIOWrapper[io.BufferedRandom]: + ... + + @overload async def open_file( file: _OpenFile, @@ -371,7 +384,10 @@ async def open_file( newline: None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[io.BufferedWriter]: ... +) -> AsyncIOWrapper[io.BufferedWriter]: + ... + + @overload async def open_file( file: _OpenFile, @@ -382,7 +398,10 @@ async def open_file( newline: None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[io.BufferedReader]: ... +) -> AsyncIOWrapper[io.BufferedReader]: + ... + + @overload async def open_file( file: _OpenFile, @@ -393,9 +412,10 @@ async def open_file( newline: None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[BinaryIO]: ... +) -> AsyncIOWrapper[BinaryIO]: + ... + -# Fallback if mode is not specified @overload async def open_file( file: _OpenFile, @@ -406,7 +426,8 @@ async def open_file( newline: str | None = None, closefd: bool = True, opener: _Opener | None = None, -) -> AsyncIOWrapper[IO[Any]]: ... +) -> AsyncIOWrapper[IO[Any]]: + ... async def open_file( From 67858f3275bebca6e925449f8b1daf5bfe606911 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Wed, 12 Jul 2023 17:38:52 +1000 Subject: [PATCH 06/28] Suppress the duplicate type hints for trio.open_file() --- docs/source/reference-io.rst | 6 ++++-- trio/_file_io.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/source/reference-io.rst b/docs/source/reference-io.rst index a3291ef2ae..bc521773d0 100644 --- a/docs/source/reference-io.rst +++ b/docs/source/reference-io.rst @@ -634,9 +634,11 @@ Asynchronous path objects Asynchronous file objects ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. autofunction:: open_file +.. Suppress type annotations here, they refer to lots of internal types. + The normal Python docs go into better detail. +.. autofunction:: open_file(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=None, opener=None) -.. autofunction:: wrap_file +.. autofunction:: wrap_file(file) .. interface:: Asynchronous file interface diff --git a/trio/_file_io.py b/trio/_file_io.py index 8bd80864c4..25eee09d97 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -328,7 +328,7 @@ async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], __size: int = 0) -> AnySt # Type hints are copied from builtin open. -_OpenFile = Union['StrOrBytesPath', int] +_OpenFile = Union["StrOrBytesPath", int] _Opener = Callable[[str, int], int] @@ -440,7 +440,7 @@ async def open_file( closefd: bool = True, opener: _Opener | None = None, ) -> AsyncIOWrapper[Any]: - """Asynchronous version of :func:`io.open`. + """Asynchronous version of :func:`open`. Returns: An :term:`asynchronous file object` From 2bb8bf12c57c8def54385ef2ea6525923374bfaa Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sat, 15 Jul 2023 16:54:32 +1000 Subject: [PATCH 07/28] Add types to Path wrapper class --- trio/_path.py | 284 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 210 insertions(+), 74 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index bb81759ecf..f2f0c16cad 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -1,19 +1,38 @@ +from __future__ import annotations + import os import pathlib import sys import types +from collections.abc import Iterable from functools import partial, wraps -from typing import TYPE_CHECKING, Any +from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper +from typing import IO, TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar, overload import trio +from trio._file_io import AsyncIOWrapper as _AsyncIOWrapper from trio._util import Final, async_wraps +if TYPE_CHECKING: + from _typeshed import ( + OpenBinaryMode, + OpenBinaryModeReading, + OpenBinaryModeUpdating, + OpenBinaryModeWriting, + OpenTextMode, + ) + from typing_extensions import Literal, TypeAlias + +T = TypeVar("T") +StrPath: TypeAlias = "str | os.PathLike[str]" + # re-wrap return value from methods that return new instances of pathlib.Path -def rewrap_path(value): +def rewrap_path(value: T) -> T | Path: if isinstance(value, pathlib.Path): - value = Path(value) - return value + return Path(value) + else: + return value def _forward_factory(cls, attr_name, attr): @@ -78,7 +97,15 @@ async def wrapper(cls, *args, **kwargs): class AsyncAutoWrapperType(Final): - def __init__(cls, name, bases, attrs): + _forwards: type + _wraps: type + _forward_magic: list[str] + _wrap_iter: list[str] + _forward: list[str] + + def __init__( + cls, name: str, bases: tuple[type, ...], attrs: dict[str, Any] + ) -> None: super().__init__(name, bases, attrs) cls._forward = [] @@ -87,7 +114,7 @@ def __init__(cls, name, bases, attrs): type(cls).generate_magic(cls, attrs) type(cls).generate_iter(cls, attrs) - def generate_forwards(cls, attrs): + def generate_forwards(cls, attrs: dict[str, Any]) -> None: # forward functions of _forwards for attr_name, attr in cls._forwards.__dict__.items(): if attr_name.startswith("_") or attr_name in attrs: @@ -101,7 +128,7 @@ def generate_forwards(cls, attrs): else: raise TypeError(attr_name, type(attr)) - def generate_wraps(cls, attrs): + def generate_wraps(cls, attrs: dict[str, Any]) -> None: # generate wrappers for functions of _wraps for attr_name, attr in cls._wraps.__dict__.items(): # .z. exclude cls._wrap_iter @@ -116,14 +143,14 @@ def generate_wraps(cls, attrs): else: raise TypeError(attr_name, type(attr)) - def generate_magic(cls, attrs): + def generate_magic(cls, attrs: dict[str, Any]) -> None: # generate wrappers for magic for attr_name in cls._forward_magic: attr = getattr(cls._forwards, attr_name) wrapper = _forward_magic(cls, attr) setattr(cls, attr_name, wrapper) - def generate_iter(cls, attrs): + def generate_iter(cls, attrs: dict[str, Any]) -> None: # generate wrappers for methods that return iterators for attr_name, attr in cls._wraps.__dict__.items(): if attr_name in cls._wrap_iter: @@ -137,9 +164,10 @@ class Path(metaclass=AsyncAutoWrapperType): """ - _wraps = pathlib.Path - _forwards = pathlib.PurePath - _forward_magic = [ + _forward: ClassVar[list[str]] + _wraps: ClassVar[type] = pathlib.Path + _forwards: ClassVar[type] = pathlib.PurePath + _forward_magic: ClassVar[list[str]] = [ "__str__", "__bytes__", "__truediv__", @@ -151,9 +179,9 @@ class Path(metaclass=AsyncAutoWrapperType): "__ge__", "__hash__", ] - _wrap_iter = ["glob", "rglob", "iterdir"] + _wrap_iter: ClassVar[list[str]] = ["glob", "rglob", "iterdir"] - def __init__(self, *args): + def __init__(self, *args: StrPath) -> None: self._wrapped = pathlib.Path(*args) # type checkers allow accessing any attributes on class instances with `__getattr__` @@ -167,17 +195,94 @@ def __getattr__(self, name): return rewrap_path(value) raise AttributeError(name) - def __dir__(self): - return super().__dir__() + self._forward + def __dir__(self) -> list[str]: + return [*super().__dir__(), *self._forward] - def __repr__(self): + def __repr__(self) -> str: return f"trio.Path({repr(str(self))})" - def __fspath__(self): + def __fspath__(self) -> str: return os.fspath(self._wrapped) - @wraps(pathlib.Path.open) - async def open(self, *args, **kwargs): + @overload + def open( + self, + mode: OpenTextMode = "r", + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> _AsyncIOWrapper[TextIOWrapper]: + ... + + @overload + def open( + self, + mode: OpenBinaryMode, + buffering: Literal[0], + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> _AsyncIOWrapper[FileIO]: + ... + + @overload + def open( + self, + mode: OpenBinaryModeUpdating, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> _AsyncIOWrapper[BufferedRandom]: + ... + + @overload + def open( + self, + mode: OpenBinaryModeWriting, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> _AsyncIOWrapper[BufferedWriter]: + ... + + @overload + def open( + self, + mode: OpenBinaryModeReading, + buffering: Literal[-1, 1] = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> _AsyncIOWrapper[BufferedReader]: + ... + + @overload + def open( + self, + mode: OpenBinaryMode, + buffering: int = -1, + encoding: None = None, + errors: None = None, + newline: None = None, + ) -> _AsyncIOWrapper[BinaryIO]: + ... + + @overload + def open( + self, + mode: str, + buffering: int = -1, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> _AsyncIOWrapper[IO[Any]]: + ... + + @wraps(pathlib.Path.open) # type: ignore[misc] # Overload return mismatch. + async def open(self, *args: Any, **kwargs: Any) -> _AsyncIOWrapper[IO[Any]]: """Open the file pointed to by the path, like the :func:`trio.open_file` function does. @@ -189,9 +294,10 @@ async def open(self, *args, **kwargs): if TYPE_CHECKING: # the dunders listed in _forward_magic that aren't seen otherwise - __bytes__ = pathlib.Path.__bytes__ - __truediv__ = pathlib.Path.__truediv__ - __rtruediv__ = pathlib.Path.__rtruediv__ + # fmt: off + def __bytes__(self) -> bytes: ... + def __truediv__(self, other: StrPath) -> Path: ... + def __rtruediv__(self, other: StrPath) -> Path: ... # These should be fully typed, either manually or with some magic wrapper # function that copies the type of pathlib.Path except sticking an async in @@ -199,65 +305,95 @@ async def open(self, *args, **kwargs): # https://github.com/python-trio/trio/issues/2630 # wrapped methods handled by __getattr__ - absolute: Any - as_posix: Any - as_uri: Any - chmod: Any - cwd: Any - exists: Any - expanduser: Any - glob: Any - home: Any - is_absolute: Any - is_block_device: Any - is_char_device: Any - is_dir: Any - is_fifo: Any - is_file: Any - is_reserved: Any - is_socket: Any - is_symlink: Any - iterdir: Any - joinpath: Any - lchmod: Any - lstat: Any - match: Any - mkdir: Any - read_bytes: Any - read_text: Any - relative_to: Any - rename: Any - replace: Any - resolve: Any - rglob: Any - rmdir: Any - samefile: Any - stat: Any - symlink_to: Any - touch: Any - unlink: Any - with_name: Any - with_suffix: Any - write_bytes: Any - write_text: Any + async def absolute(self) -> Path: ... + async def as_posix(self) -> str: ... + async def as_uri(self) -> str: ... + + if sys.version_info >= (3, 10): + async def stat(self, *, follow_symlinks: bool = True) -> os.stat_result: ... + async def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: ... + else: + async def stat(self) -> os.stat_result: ... + async def chmod(self, mode: int) -> None: ... + + @classmethod + async def cwd(self) -> Path: ... + + async def exists(self) -> bool: ... + async def expanduser(self) -> Path: ... + async def glob(self, pattern: str) -> Iterable[Path]: ... + async def home(self) -> Path: ... + async def is_absolute(self) -> bool: ... + async def is_block_device(self) -> bool: ... + async def is_char_device(self) -> bool: ... + async def is_dir(self) -> bool: ... + async def is_fifo(self) -> bool: ... + async def is_file(self) -> bool: ... + async def is_reserved(self) -> bool: ... + async def is_socket(self) -> bool: ... + async def is_symlink(self) -> bool: ... + async def iterdir(self) -> Iterable[Path]: ... + async def joinpath(self, *other: StrPath) -> Path: ... + async def lchmod(self, mode: int) -> None: ... + async def lstat(self) -> os.stat_result: ... + async def match(self, path_pattern: str) -> bool: ... + async def mkdir(self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False) -> None: ... + async def read_bytes(self) -> bytes: ... + async def read_text(self, encoding: str | None = None, errors: str | None = None) -> str: ... + async def relative_to(self, *other: StrPath) -> Path: ... + + if sys.version_info >= (3, 8): + def rename(self, target: str | pathlib.PurePath) -> Path: ... + def replace(self, target: str | pathlib.PurePath) -> Path: ... + else: + def rename(self, target: str | pathlib.PurePath) -> None: ... + def replace(self, target: str | pathlib.PurePath) -> None: ... + + async def resolve(self, strict: bool = False) -> Path: ... + async def rglob(self, pattern: str) -> Iterable[Path]: ... + async def rmdir(self) -> None: ... + async def samefile(self, other_path: str | bytes | int | Path) -> bool: ... + async def symlink_to(self, target: str | Path, target_is_directory: bool = False) -> None: ... + async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: ... + if sys.version_info >= (3, 8): + def unlink(self, missing_ok: bool = False) -> None: ... + else: + def unlink(self) -> None: ... + async def with_name(self, name: str) -> Path: ... + async def with_suffix(self, suffix: str) -> Path: ... + async def write_bytes(self, data: bytes) -> int: ... + + if sys.version_info >= (3, 10): + async def write_text( + self, data: str, + encoding: str | None = None, + errors: str | None = None, + newline: str | None = None, + ) -> int: ... + else: + async def write_text( + self, data: str, + encoding: str | None = None, + errors: str | None = None, + ) -> int: ... if sys.platform != "win32": - group: Any - is_mount: Any - owner: Any + async def owner(self) -> str: ... + async def group(self) -> str: ... + async def is_mount(self) -> bool: ... if sys.version_info >= (3, 8) and sys.version_info < (3, 12): - link_to: Any + async def link_to(self, target: StrPath | bytes) -> None: ... if sys.version_info >= (3, 9): - is_relative_to: Any - with_stem: Any - readlink: Any + async def is_relative_to(self, *other: StrPath) -> bool: ... + async def with_stem(self, stem: str) -> Path: ... + async def readlink(self) -> Path: ... if sys.version_info >= (3, 10): - hardlink_to: Any + async def hardlink_to(self, target: str | pathlib.Path) -> None: ... if sys.version_info >= (3, 12): - is_junction: Any + async def is_junction(self) -> bool: ... walk: Any - with_segments: Any + async def with_segments(self, *pathsegments: StrPath) -> Path: ... Path.iterdir.__doc__ = """ From d3474d3c2e43b8bc3269c3689c701f343984d6f6 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Sun, 16 Jul 2023 11:51:56 +1000 Subject: [PATCH 08/28] Overloads are never executed, so they should be ignored by coverage --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 98f923bd8e..d577aa8adf 100644 --- a/.coveragerc +++ b/.coveragerc @@ -21,6 +21,7 @@ exclude_lines = abc.abstractmethod if TYPE_CHECKING: if _t.TYPE_CHECKING: + @overload partial_branches = pragma: no branch From ae706f65c9c0ac3925ef74941d4c5eba6eb45328 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 09:46:44 +1000 Subject: [PATCH 09/28] Type trio._util.async_wraps --- trio/_util.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index 0a0795fc15..75f6e9ee73 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -13,6 +13,10 @@ import trio + +CallT = t.TypeVar("CallT", bound=t.Callable[..., t.Any]) + + # Equivalent to the C function raise(), which Python doesn't wrap if os.name == "nt": # On Windows, os.kill exists but is really weird. @@ -198,10 +202,14 @@ def __exit__( self._held = False -def async_wraps(cls, wrapped_cls, attr_name): +def async_wraps( + cls: type[t.Any], + wrapped_cls: type[t.Any], + attr_name: str, +) -> t.Callable[[CallT], CallT]: """Similar to wraps, but for async wrappers of non-async functions.""" - def decorator(func): + def decorator(func: CallT) -> CallT: func.__name__ = attr_name func.__qualname__ = ".".join((cls.__qualname__, attr_name)) From b063e500f6bccb722ef54ee1efae419f0cacd006 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 09:48:23 +1000 Subject: [PATCH 10/28] Add mostly-any types for internal _path functions These are way too dynamic to properly type. --- trio/_path.py | 59 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index f2f0c16cad..55f62e21b7 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -4,10 +4,10 @@ import pathlib import sys import types -from collections.abc import Iterable +from collections.abc import Awaitable, Callable, Iterable from functools import partial, wraps from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper -from typing import IO, TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar, overload +from typing import IO, TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar, cast, overload import trio from trio._file_io import AsyncIOWrapper as _AsyncIOWrapper @@ -21,7 +21,9 @@ OpenBinaryModeWriting, OpenTextMode, ) - from typing_extensions import Literal, TypeAlias + from typing_extensions import Literal, TypeAlias, ParamSpec, Concatenate + + P = ParamSpec("P") T = TypeVar("T") StrPath: TypeAlias = "str | os.PathLike[str]" @@ -35,9 +37,13 @@ def rewrap_path(value: T) -> T | Path: return value -def _forward_factory(cls, attr_name, attr): +def _forward_factory( + cls: AsyncAutoWrapperType, + attr_name: str, + attr: Callable[Concatenate[pathlib.Path, P], T], +) -> Callable[Concatenate[Path, P], T | Path]: @wraps(attr) - def wrapper(self, *args, **kwargs): + def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> Path | Any: attr = getattr(self._wrapped, attr_name) value = attr(*args, **kwargs) return rewrap_path(value) @@ -45,24 +51,28 @@ def wrapper(self, *args, **kwargs): return wrapper -def _forward_magic(cls, attr): - sentinel = object() +def _forward_magic( + cls: AsyncAutoWrapperType, attr: Callable[..., object] +) -> Callable[..., Path | Any]: + sentinel: Any = object() @wraps(attr) - def wrapper(self, other=sentinel): + def wrapper(self: Path, other: object = sentinel) -> Any: if other is sentinel: return attr(self._wrapped) if isinstance(other, cls): - other = other._wrapped + other = cast(Path, other)._wrapped value = attr(self._wrapped, other) return rewrap_path(value) return wrapper -def iter_wrapper_factory(cls, meth_name): +def iter_wrapper_factory( + cls: AsyncAutoWrapperType, meth_name: str +) -> Callable[Concatenate[Path, P], Awaitable[Iterable[Path]]]: @async_wraps(cls, cls._wraps, meth_name) - async def wrapper(self, *args, **kwargs): + async def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> Iterable[Path]: meth = getattr(self._wrapped, meth_name) func = partial(meth, *args, **kwargs) # Make sure that the full iteration is performed in the thread @@ -73,9 +83,11 @@ async def wrapper(self, *args, **kwargs): return wrapper -def thread_wrapper_factory(cls, meth_name): +def thread_wrapper_factory( + cls: AsyncAutoWrapperType, meth_name: str +) -> Callable[Concatenate[Path, P], Awaitable[Path]]: @async_wraps(cls, cls._wraps, meth_name) - async def wrapper(self, *args, **kwargs): + async def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> Path: meth = getattr(self._wrapped, meth_name) func = partial(meth, *args, **kwargs) value = await trio.to_thread.run_sync(func) @@ -84,16 +96,17 @@ async def wrapper(self, *args, **kwargs): return wrapper -def classmethod_wrapper_factory(cls, meth_name): - @classmethod +def classmethod_wrapper_factory( + cls: AsyncAutoWrapperType, meth_name: str +) -> classmethod: @async_wraps(cls, cls._wraps, meth_name) - async def wrapper(cls, *args, **kwargs): + async def wrapper(cls: type[Path], *args: Any, **kwargs: Any) -> Path: meth = getattr(cls._wraps, meth_name) func = partial(meth, *args, **kwargs) value = await trio.to_thread.run_sync(func) return rewrap_path(value) - return wrapper + return classmethod(wrapper) class AsyncAutoWrapperType(Final): @@ -104,7 +117,7 @@ class AsyncAutoWrapperType(Final): _forward: list[str] def __init__( - cls, name: str, bases: tuple[type, ...], attrs: dict[str, Any] + cls, name: str, bases: tuple[type, ...], attrs: dict[str, object] ) -> None: super().__init__(name, bases, attrs) @@ -114,7 +127,7 @@ def __init__( type(cls).generate_magic(cls, attrs) type(cls).generate_iter(cls, attrs) - def generate_forwards(cls, attrs: dict[str, Any]) -> None: + def generate_forwards(cls, attrs: dict[str, object]) -> None: # forward functions of _forwards for attr_name, attr in cls._forwards.__dict__.items(): if attr_name.startswith("_") or attr_name in attrs: @@ -128,8 +141,9 @@ def generate_forwards(cls, attrs: dict[str, Any]) -> None: else: raise TypeError(attr_name, type(attr)) - def generate_wraps(cls, attrs: dict[str, Any]) -> None: + def generate_wraps(cls, attrs: dict[str, object]) -> None: # generate wrappers for functions of _wraps + wrapper: classmethod | Callable for attr_name, attr in cls._wraps.__dict__.items(): # .z. exclude cls._wrap_iter if attr_name.startswith("_") or attr_name in attrs: @@ -143,15 +157,16 @@ def generate_wraps(cls, attrs: dict[str, Any]) -> None: else: raise TypeError(attr_name, type(attr)) - def generate_magic(cls, attrs: dict[str, Any]) -> None: + def generate_magic(cls, attrs: dict[str, object]) -> None: # generate wrappers for magic for attr_name in cls._forward_magic: attr = getattr(cls._forwards, attr_name) wrapper = _forward_magic(cls, attr) setattr(cls, attr_name, wrapper) - def generate_iter(cls, attrs: dict[str, Any]) -> None: + def generate_iter(cls, attrs: dict[str, object]) -> None: # generate wrappers for methods that return iterators + wrapper: Callable for attr_name, attr in cls._wraps.__dict__.items(): if attr_name in cls._wrap_iter: wrapper = iter_wrapper_factory(cls, attr_name) From 1b190e52321a0d11c942cd1e4e3bdecc1726ee9e Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 09:53:22 +1000 Subject: [PATCH 11/28] Disallow untyped definitions in _path and _file_io --- pyproject.toml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cfb4060ee7..b754db0661 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,12 @@ disallow_untyped_defs = false # DO NOT use `ignore_errors`; it doesn't apply # downstream and users have to deal with them. +[[tool.mypy.overrides]] +module = [ + "trio._path", + "trio._file_io", +] +disallow_untyped_defs = true [tool.pytest.ini_options] addopts = ["--strict-markers", "--strict-config"] From 616bc665c89dfaf113fa391cfa38a008e41919fd Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 09:54:56 +1000 Subject: [PATCH 12/28] Update verify_types.json --- trio/_tests/verify_types.json | 38 ++++++++++++++--------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 9d7d7aa912..3d87cf7727 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.8764044943820225, + "completenessScore": 0.869983948635634, "exportedSymbolCounts": { "withAmbiguousType": 1, - "withKnownType": 546, - "withUnknownType": 76 + "withKnownType": 542, + "withUnknownType": 80 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, @@ -45,9 +45,9 @@ } ], "otherSymbolCounts": { - "withAmbiguousType": 8, - "withKnownType": 433, - "withUnknownType": 135 + "withAmbiguousType": 5, + "withKnownType": 473, + "withUnknownType": 137 }, "packageName": "trio", "symbols": [ @@ -73,6 +73,8 @@ "trio._core._mock_clock.MockClock.jump", "trio._core._run.Nursery.start", "trio._core._run.Nursery.start_soon", + "trio._core._run.Task.eventual_parent_nursery", + "trio._core._run.Task.parent_nursery", "trio._core._unbounded_queue.UnboundedQueue.__aiter__", "trio._core._unbounded_queue.UnboundedQueue.__anext__", "trio._core._unbounded_queue.UnboundedQueue.__repr__", @@ -103,20 +105,6 @@ "trio._highlevel_socket.SocketStream.getsockopt", "trio._highlevel_socket.SocketStream.send_all", "trio._highlevel_socket.SocketStream.setsockopt", - "trio._path.AsyncAutoWrapperType.__init__", - "trio._path.AsyncAutoWrapperType.generate_forwards", - "trio._path.AsyncAutoWrapperType.generate_iter", - "trio._path.AsyncAutoWrapperType.generate_magic", - "trio._path.AsyncAutoWrapperType.generate_wraps", - "trio._path.Path", - "trio._path.Path.__bytes__", - "trio._path.Path.__dir__", - "trio._path.Path.__fspath__", - "trio._path.Path.__init__", - "trio._path.Path.__repr__", - "trio._path.Path.__rtruediv__", - "trio._path.Path.__truediv__", - "trio._path.Path.open", "trio._socket._SocketType.__getattr__", "trio._socket._SocketType.accept", "trio._socket._SocketType.connect", @@ -157,6 +145,11 @@ "trio._subprocess.Process.send_signal", "trio._subprocess.Process.terminate", "trio._subprocess.Process.wait", + "trio._sync.Condition.__init__", + "trio._sync.Condition.statistics", + "trio._sync.Lock", + "trio._sync.StrictFIFOLock", + "trio._sync._LockImpl.statistics", "trio.current_time", "trio.from_thread.run", "trio.from_thread.run_sync", @@ -165,6 +158,7 @@ "trio.lowlevel.current_clock", "trio.lowlevel.current_root_task", "trio.lowlevel.current_statistics", + "trio.lowlevel.current_task", "trio.lowlevel.current_trio_token", "trio.lowlevel.currently_ki_protected", "trio.lowlevel.notify_closing", @@ -179,7 +173,6 @@ "trio.lowlevel.temporarily_detach_coroutine_object", "trio.lowlevel.wait_readable", "trio.lowlevel.wait_writable", - "trio.open_file", "trio.open_ssl_over_tcp_listeners", "trio.open_ssl_over_tcp_stream", "trio.open_tcp_listeners", @@ -231,8 +224,7 @@ "trio.testing.trio_test", "trio.testing.wait_all_tasks_blocked", "trio.tests.TestsDeprecationWrapper", - "trio.to_thread.current_default_thread_limiter", - "trio.wrap_file" + "trio.to_thread.current_default_thread_limiter" ] } } From 4d722d254755a8ff8ba819e931ef7558fabc66fe Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 10:21:54 +1000 Subject: [PATCH 13/28] Try and explain the type hints better --- trio/_file_io.py | 24 +++++++++++++++++------- trio/_path.py | 2 +- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 25eee09d97..66d73cf2a7 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -82,8 +82,18 @@ AnyStr_co = TypeVar("AnyStr_co", str, bytes, covariant=True) AnyStr_contra = TypeVar("AnyStr_contra", str, bytes, contravariant=True) -# Define a protocol for every method/property we expose. Each is effectively a predicate checking -# whether a class defines this method/property. +# This is a little complicated. IO objects have a lot of methods, and which are available on +# different types varies wildly. We want to match the interface of whatever file we're wrapping. +# This pile of protocols each has one sync method/property, meaning they're going to be compatible +# with a file class that supports that method/property. The ones parameterized with AnyStr take +# either str or bytes depending. + +# The wrapper is then a generic class, where the typevar is set to the type of the sync file we're +# wrapping. For generics, adding a type to self has a special meaning - properties/methods can be +# conditional - it's only valid to call them if the object you're accessing them on is compatible +# with that type hint. By using the protocols, the type checker will be checking to see if the +# wrapped type has that method, and only allow the methods that do to be called. We can then alter +# the signature however it needs to match runtime behaviour. if TYPE_CHECKING: from typing_extensions import Buffer, Protocol @@ -187,12 +197,14 @@ class _CanWrite(Protocol[AnyStr_contra]): def write(self, __data: AnyStr_contra) -> int: ... class _CanWriteLines(Protocol[T_contra]): + """The lines parameter varies for bytes/str, so use a typevar to make the async match.""" def writelines(self, __lines: Iterable[T_contra]) -> None: ... class _CanPeek(Protocol[AnyStr_co]): def peek(self, __size: int = 0) -> AnyStr_co: ... class _CanDetach(Protocol[T_co]): + """The T typevar will be the unbuffered/binary file this file wraps.""" def detach(self) -> T_co: ... class _CanClose(Protocol): @@ -204,11 +216,11 @@ class AsyncIOWrapper(AsyncResource, Generic[FileT_co]): file object` interface. Wrapped methods that could block are executed in :meth:`trio.to_thread.run_sync`. - All properties and methods defined in in :mod:`~io` are exposed by this + All properties and methods defined in :mod:`~io` are exposed by this wrapper, if they exist in the wrapped file object. - """ - + # FileT needs to be covariant for the protocol trick to work - the real IO types are a subtype + # of the protocols. def __init__(self, file: FileT_co) -> None: self._wrapped = file @@ -281,8 +293,6 @@ async def aclose(self: AsyncIOWrapper[_CanClose]) -> None: if TYPE_CHECKING: # fmt: off # Based on typing.IO and io stubs. - # For every method/property we type self, restricting these to only be available if - # the original also is. @property def closed(self: AsyncIOWrapper[_HasClosed]) -> bool: ... @property diff --git a/trio/_path.py b/trio/_path.py index 55f62e21b7..5121d7d9f2 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -407,7 +407,7 @@ async def readlink(self) -> Path: ... async def hardlink_to(self, target: str | pathlib.Path) -> None: ... if sys.version_info >= (3, 12): async def is_junction(self) -> bool: ... - walk: Any + walk: Any # TODO async def with_segments(self, *pathsegments: StrPath) -> Path: ... From abddbf62585449c0e852092b6fc2dc2fada814bb Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 10:29:03 +1000 Subject: [PATCH 14/28] Reformat --- trio/_file_io.py | 1 + trio/_path.py | 2 +- trio/_tests/verify_types.json | 18 +++++------------- trio/_util.py | 1 - 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 66d73cf2a7..707ce0545a 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -219,6 +219,7 @@ class AsyncIOWrapper(AsyncResource, Generic[FileT_co]): All properties and methods defined in :mod:`~io` are exposed by this wrapper, if they exist in the wrapped file object. """ + # FileT needs to be covariant for the protocol trick to work - the real IO types are a subtype # of the protocols. def __init__(self, file: FileT_co) -> None: diff --git a/trio/_path.py b/trio/_path.py index 5121d7d9f2..9d370f0809 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -21,7 +21,7 @@ OpenBinaryModeWriting, OpenTextMode, ) - from typing_extensions import Literal, TypeAlias, ParamSpec, Concatenate + from typing_extensions import Concatenate, Literal, ParamSpec, TypeAlias P = ParamSpec("P") diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 3d87cf7727..c30b029d60 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.869983948635634, + "completenessScore": 0.8812199036918138, "exportedSymbolCounts": { "withAmbiguousType": 1, - "withKnownType": 542, - "withUnknownType": 80 + "withKnownType": 549, + "withUnknownType": 73 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, @@ -46,8 +46,8 @@ ], "otherSymbolCounts": { "withAmbiguousType": 5, - "withKnownType": 473, - "withUnknownType": 137 + "withKnownType": 487, + "withUnknownType": 123 }, "packageName": "trio", "symbols": [ @@ -73,8 +73,6 @@ "trio._core._mock_clock.MockClock.jump", "trio._core._run.Nursery.start", "trio._core._run.Nursery.start_soon", - "trio._core._run.Task.eventual_parent_nursery", - "trio._core._run.Task.parent_nursery", "trio._core._unbounded_queue.UnboundedQueue.__aiter__", "trio._core._unbounded_queue.UnboundedQueue.__anext__", "trio._core._unbounded_queue.UnboundedQueue.__repr__", @@ -145,11 +143,6 @@ "trio._subprocess.Process.send_signal", "trio._subprocess.Process.terminate", "trio._subprocess.Process.wait", - "trio._sync.Condition.__init__", - "trio._sync.Condition.statistics", - "trio._sync.Lock", - "trio._sync.StrictFIFOLock", - "trio._sync._LockImpl.statistics", "trio.current_time", "trio.from_thread.run", "trio.from_thread.run_sync", @@ -158,7 +151,6 @@ "trio.lowlevel.current_clock", "trio.lowlevel.current_root_task", "trio.lowlevel.current_statistics", - "trio.lowlevel.current_task", "trio.lowlevel.current_trio_token", "trio.lowlevel.currently_ki_protected", "trio.lowlevel.notify_closing", diff --git a/trio/_util.py b/trio/_util.py index 75f6e9ee73..05ffc28b92 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -13,7 +13,6 @@ import trio - CallT = t.TypeVar("CallT", bound=t.Callable[..., t.Any]) From a624ffe571e2d316149acb04e51041e7f3023352 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 10:59:42 +1000 Subject: [PATCH 15/28] Suppress errors about missing paramspec annotations --- docs/source/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 68a5a22a81..bc05dc845f 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -63,6 +63,8 @@ ("py:obj", "trio._abc.T"), ("py:obj", "trio._abc.T_resource"), ("py:class", "types.FrameType"), + ("py:class", "P.args"), + ("py:class", "P.kwargs"), ] autodoc_inherit_docstrings = False default_role = "obj" From 67653e6157d7954336c583f0f55a939984321b62 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 11:09:07 +1000 Subject: [PATCH 16/28] Copy signature for path methods, improving docs display --- trio/_path.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/trio/_path.py b/trio/_path.py index 9d370f0809..04b6b06981 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -1,5 +1,6 @@ from __future__ import annotations +import inspect import os import pathlib import sys @@ -47,6 +48,8 @@ def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> Path | Any: attr = getattr(self._wrapped, attr_name) value = attr(*args, **kwargs) return rewrap_path(value) + # Assigning this makes inspect and therefore Sphinx show the original parameters. + wrapper.__signature__ = inspect.signature(attr) return wrapper @@ -65,6 +68,7 @@ def wrapper(self: Path, other: object = sentinel) -> Any: value = attr(self._wrapped, other) return rewrap_path(value) + wrapper.__signature__ = inspect.signature(attr) return wrapper @@ -106,6 +110,7 @@ async def wrapper(cls: type[Path], *args: Any, **kwargs: Any) -> Path: value = await trio.to_thread.run_sync(func) return rewrap_path(value) + wrapper.__signature__ = inspect.signature(getattr(cls._wraps, meth_name)) return classmethod(wrapper) @@ -153,6 +158,7 @@ def generate_wraps(cls, attrs: dict[str, object]) -> None: setattr(cls, attr_name, wrapper) elif isinstance(attr, types.FunctionType): wrapper = thread_wrapper_factory(cls, attr_name) + wrapper.__signature__ = inspect.signature(attr) setattr(cls, attr_name, wrapper) else: raise TypeError(attr_name, type(attr)) @@ -170,6 +176,7 @@ def generate_iter(cls, attrs: dict[str, object]) -> None: for attr_name, attr in cls._wraps.__dict__.items(): if attr_name in cls._wrap_iter: wrapper = iter_wrapper_factory(cls, attr_name) + wrapper.__signature__ = inspect.signature(attr) setattr(cls, attr_name, wrapper) From f787ed668e7dbe4a595f77e6828b8586974b4418 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Mon, 17 Jul 2023 11:24:02 +1000 Subject: [PATCH 17/28] Make signature assignment typesafe. --- trio/_path.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/trio/_path.py b/trio/_path.py index 04b6b06981..99ff06d31a 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -48,7 +48,10 @@ def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> Path | Any: attr = getattr(self._wrapped, attr_name) value = attr(*args, **kwargs) return rewrap_path(value) + # Assigning this makes inspect and therefore Sphinx show the original parameters. + # It's not defined on functions normally though, this is a custom attribute. + assert isinstance(wrapper, types.FunctionType) wrapper.__signature__ = inspect.signature(attr) return wrapper @@ -68,6 +71,7 @@ def wrapper(self: Path, other: object = sentinel) -> Any: value = attr(self._wrapped, other) return rewrap_path(value) + assert isinstance(wrapper, types.FunctionType) wrapper.__signature__ = inspect.signature(attr) return wrapper @@ -110,6 +114,7 @@ async def wrapper(cls: type[Path], *args: Any, **kwargs: Any) -> Path: value = await trio.to_thread.run_sync(func) return rewrap_path(value) + assert isinstance(wrapper, types.FunctionType) wrapper.__signature__ = inspect.signature(getattr(cls._wraps, meth_name)) return classmethod(wrapper) @@ -158,6 +163,7 @@ def generate_wraps(cls, attrs: dict[str, object]) -> None: setattr(cls, attr_name, wrapper) elif isinstance(attr, types.FunctionType): wrapper = thread_wrapper_factory(cls, attr_name) + assert isinstance(wrapper, types.FunctionType) wrapper.__signature__ = inspect.signature(attr) setattr(cls, attr_name, wrapper) else: @@ -176,6 +182,7 @@ def generate_iter(cls, attrs: dict[str, object]) -> None: for attr_name, attr in cls._wraps.__dict__.items(): if attr_name in cls._wrap_iter: wrapper = iter_wrapper_factory(cls, attr_name) + assert isinstance(wrapper, types.FunctionType) wrapper.__signature__ = inspect.signature(attr) setattr(cls, attr_name, wrapper) From ce7a9850cbf52cdf5e8bc8b7eb15bcc89d3880c1 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 09:47:28 +1000 Subject: [PATCH 18/28] These parameters do not need to be Any Co-authored-by: EXPLOSION --- trio/_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_util.py b/trio/_util.py index 369ca669bc..a87f1fc02c 100644 --- a/trio/_util.py +++ b/trio/_util.py @@ -203,8 +203,8 @@ def __exit__( def async_wraps( - cls: type[t.Any], - wrapped_cls: type[t.Any], + cls: type[object], + wrapped_cls: type[object], attr_name: str, ) -> t.Callable[[CallT], CallT]: """Similar to wraps, but for async wrappers of non-async functions.""" From 41b513b6d608d49c55c54522b98b624eb9e18813 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 09:40:03 +1000 Subject: [PATCH 19/28] Use modern positional argument syntax --- trio/_file_io.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 707ce0545a..385cbc4d60 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -164,44 +164,44 @@ class _CanFlush(Protocol): def flush(self) -> None: ... class _CanRead(Protocol[AnyStr_co]): - def read(self, __size: int | None = ...) -> AnyStr_co: ... + def read(self, size: int | None = ..., /) -> AnyStr_co: ... class _CanRead1(Protocol): - def read1(self, __size: int | None = ...) -> bytes: ... + def read1(self, size: int | None = ..., /) -> bytes: ... class _CanReadAll(Protocol[AnyStr_co]): def readall(self) -> AnyStr_co: ... class _CanReadInto(Protocol): - def readinto(self, __buf: Buffer) -> int | None: ... + def readinto(self, buf: Buffer, /) -> int | None: ... class _CanReadInto1(Protocol): - def readinto1(self, __buffer: Buffer) -> int: ... + def readinto1(self, buffer: Buffer, /) -> int: ... class _CanReadLine(Protocol[AnyStr_co]): - def readline(self, __size: int = ...) -> AnyStr_co: ... + def readline(self, size: int = ..., /) -> AnyStr_co: ... class _CanReadLines(Protocol[AnyStr]): - def readlines(self, __hint: int = ...) -> list[AnyStr]: ... + def readlines(self, hint: int = ...) -> list[AnyStr]: ... class _CanSeek(Protocol): - def seek(self, __target: int, __whence: int = 0) -> int: ... + def seek(self, target: int, whence: int = 0, /) -> int: ... class _CanTell(Protocol): def tell(self) -> int: ... class _CanTruncate(Protocol): - def truncate(self, __size: int | None = ...) -> int: ... + def truncate(self, size: int | None = ..., /) -> int: ... class _CanWrite(Protocol[AnyStr_contra]): - def write(self, __data: AnyStr_contra) -> int: ... + def write(self, data: AnyStr_contra, /) -> int: ... class _CanWriteLines(Protocol[T_contra]): """The lines parameter varies for bytes/str, so use a typevar to make the async match.""" - def writelines(self, __lines: Iterable[T_contra]) -> None: ... + def writelines(self, lines: Iterable[T_contra], /) -> None: ... class _CanPeek(Protocol[AnyStr_co]): - def peek(self, __size: int = 0) -> AnyStr_co: ... + def peek(self, size: int = 0, /) -> AnyStr_co: ... class _CanDetach(Protocol[T_co]): """The T typevar will be the unbuffered/binary file this file wraps.""" @@ -323,19 +323,19 @@ def writable(self: AsyncIOWrapper[_HasWritable]) -> bool: ... def getvalue(self: AsyncIOWrapper[_CanGetValue[AnyStr]]) -> AnyStr: ... def getbuffer(self: AsyncIOWrapper[_CanGetBuffer]) -> memoryview: ... async def flush(self: AsyncIOWrapper[_CanFlush]) -> None: ... - async def read(self: AsyncIOWrapper[_CanRead[AnyStr]], __size: int | None = -1) -> AnyStr: ... - async def read1(self: AsyncIOWrapper[_CanRead1], __size: int | None = -1) -> bytes: ... + async def read(self: AsyncIOWrapper[_CanRead[AnyStr]], size: int | None = -1, /) -> AnyStr: ... + async def read1(self: AsyncIOWrapper[_CanRead1], size: int | None = -1, /) -> bytes: ... async def readall(self: AsyncIOWrapper[_CanReadAll[AnyStr]]) -> AnyStr: ... - async def readinto(self: AsyncIOWrapper[_CanReadInto], __buf: Buffer) -> int | None: ... - async def readline(self: AsyncIOWrapper[_CanReadLine[AnyStr]], __size: int = -1) -> AnyStr: ... + async def readinto(self: AsyncIOWrapper[_CanReadInto], buf: Buffer, /) -> int | None: ... + async def readline(self: AsyncIOWrapper[_CanReadLine[AnyStr]], size: int = -1, /) -> AnyStr: ... async def readlines(self: AsyncIOWrapper[_CanReadLines[AnyStr]]) -> list[AnyStr]: ... - async def seek(self: AsyncIOWrapper[_CanSeek], __target: int, __whence: int = 0) -> int: ... + async def seek(self: AsyncIOWrapper[_CanSeek], target: int, whence: int = 0, /) -> int: ... async def tell(self: AsyncIOWrapper[_CanTell]) -> int: ... - async def truncate(self: AsyncIOWrapper[_CanTruncate], __size: int | None = None) -> int: ... - async def write(self: AsyncIOWrapper[_CanWrite[AnyStr]], __data: AnyStr) -> int: ... - async def writelines(self: AsyncIOWrapper[_CanWriteLines[T]], __lines: Iterable[T]) -> None: ... - async def readinto1(self: AsyncIOWrapper[_CanReadInto1], __buffer: Buffer) -> int: ... - async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], __size: int = 0) -> AnyStr: ... + async def truncate(self: AsyncIOWrapper[_CanTruncate], size: int | None = None, /) -> int: ... + async def write(self: AsyncIOWrapper[_CanWrite[AnyStr]], data: AnyStr, /) -> int: ... + async def writelines(self: AsyncIOWrapper[_CanWriteLines[T]], lines: Iterable[T], /) -> None: ... + async def readinto1(self: AsyncIOWrapper[_CanReadInto1], buffer: Buffer, /) -> int: ... + async def peek(self: AsyncIOWrapper[_CanPeek[AnyStr]], size: int = 0, /) -> AnyStr: ... # Type hints are copied from builtin open. From 1efebb0a65871dd50160659b1b162e3d76fd7aba Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 09:59:24 +1000 Subject: [PATCH 20/28] Apply suggested changes from A5Rocks --- trio/_file_io.py | 16 +++++++++------- trio/_path.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 385cbc4d60..8a3865eec3 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -94,6 +94,7 @@ # with that type hint. By using the protocols, the type checker will be checking to see if the # wrapped type has that method, and only allow the methods that do to be called. We can then alter # the signature however it needs to match runtime behaviour. +# More info: https://mypy.readthedocs.io/en/stable/more_types.html#advanced-uses-of-self-types if TYPE_CHECKING: from typing_extensions import Buffer, Protocol @@ -117,9 +118,10 @@ def fileno(self) -> int: ... class _HasIsATTY(Protocol): def isatty(self) -> bool: ... - class _HasNewlines(Protocol): + class _HasNewlines(Protocol[T_co]): + # Type varies here - documented to be None, tuple of strings, strings. Typeshed uses Any. @property - def newlines(self) -> Any: ... # None, str or tuple + def newlines(self) -> T: ... class _HasReadable(Protocol): def readable(self) -> bool: ... @@ -197,20 +199,22 @@ class _CanWrite(Protocol[AnyStr_contra]): def write(self, data: AnyStr_contra, /) -> int: ... class _CanWriteLines(Protocol[T_contra]): - """The lines parameter varies for bytes/str, so use a typevar to make the async match.""" + # The lines parameter varies for bytes/str, so use a typevar to make the async match. def writelines(self, lines: Iterable[T_contra], /) -> None: ... class _CanPeek(Protocol[AnyStr_co]): def peek(self, size: int = 0, /) -> AnyStr_co: ... class _CanDetach(Protocol[T_co]): - """The T typevar will be the unbuffered/binary file this file wraps.""" + # The T typevar will be the unbuffered/binary file this file wraps. def detach(self) -> T_co: ... class _CanClose(Protocol): def close(self) -> None: ... +# FileT needs to be covariant for the protocol trick to work - the real IO types are effectively a +# subtype of the protocols. class AsyncIOWrapper(AsyncResource, Generic[FileT_co]): """A generic :class:`~io.IOBase` wrapper that implements the :term:`asynchronous file object` interface. Wrapped methods that could block are executed in @@ -220,8 +224,6 @@ class AsyncIOWrapper(AsyncResource, Generic[FileT_co]): wrapper, if they exist in the wrapped file object. """ - # FileT needs to be covariant for the protocol trick to work - the real IO types are a subtype - # of the protocols. def __init__(self, file: FileT_co) -> None: self._wrapped = file @@ -301,7 +303,7 @@ def encoding(self: AsyncIOWrapper[_HasEncoding]) -> str: ... @property def errors(self: AsyncIOWrapper[_HasErrors]) -> str | None: ... @property - def newlines(self: AsyncIOWrapper[_HasNewlines]) -> Any: ... # None, str or tuple + def newlines(self: AsyncIOWrapper[_HasNewlines[T]]) -> T: ... @property def buffer(self: AsyncIOWrapper[_HasBuffer]) -> BinaryIO: ... @property diff --git a/trio/_path.py b/trio/_path.py index ed0f435d90..466ae96e24 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -44,7 +44,7 @@ def _forward_factory( attr: Callable[Concatenate[pathlib.Path, P], T], ) -> Callable[Concatenate[Path, P], T | Path]: @wraps(attr) - def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> Path | Any: + def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> T | Path: attr = getattr(self._wrapped, attr_name) value = attr(*args, **kwargs) return rewrap_path(value) From 7dc96478a91755dd0b5f5d05180ec90e86ce485c Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 10:54:40 +1000 Subject: [PATCH 21/28] Add test to confirm file_io type stubs match the to-wrap lists. --- trio/_tests/test_file_io.py | 39 ++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/trio/_tests/test_file_io.py b/trio/_tests/test_file_io.py index e99788efc5..61345ca688 100644 --- a/trio/_tests/test_file_io.py +++ b/trio/_tests/test_file_io.py @@ -1,12 +1,14 @@ +import importlib import io import os +import re from unittest import mock from unittest.mock import sentinel import pytest import trio -from trio import _core +from trio import _core, _file_io from trio._file_io import _FILE_ASYNC_METHODS, _FILE_SYNC_ATTRS, AsyncIOWrapper @@ -78,6 +80,41 @@ def unsupported_attr(self): # pragma: no cover getattr(async_file, "unsupported_attr") +def test_type_stubs_match_lists() -> None: + """Check the manual stubs match the list of wrapped methods.""" + source = io.StringIO(_file_io.__spec__.loader.get_source("trio._file_io")) + # Find the class, then find the TYPE_CHECKING block. + for line in source: + if "class AsyncIOWrapper" in line: + break + else: + pytest.fail("No class definition line?") + + for line in source: + if "if TYPE_CHECKING" in line: + break + else: + pytest.fail("No TYPE CHECKING line?") + + # Now we should be at the type checking block. + found: list[tuple[str, str]] = [] + for line in source: + if line.strip() and not line.startswith(" " * 8): + break # Dedented out of the if TYPE_CHECKING block. + match = re.match(r"\s*(async )?def ([a-zA-Z0-9_]+)\(", line) + if match is not None: + kind = "async" if match.group(1) is not None else "sync" + found.append((match.group(2), kind)) + + # Compare two lists so that we can easily see duplicates, and see what is different overall. + expected = [(fname, "async") for fname in _FILE_ASYNC_METHODS] + expected += [(fname, "sync") for fname in _FILE_SYNC_ATTRS] + # Ignore order, error if duplicates are present. + found.sort() + expected.sort() + assert found == expected + + def test_sync_attrs_forwarded(async_file, wrapped): for attr_name in _FILE_SYNC_ATTRS: if attr_name not in dir(async_file): From 1e56d4264b7faf622ab28706ca74976ec3becd1c Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 10:56:51 +1000 Subject: [PATCH 22/28] Remove outdated comment. --- trio/_path.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index 466ae96e24..c954cce7bb 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -328,11 +328,6 @@ def __bytes__(self) -> bytes: ... def __truediv__(self, other: StrPath) -> Path: ... def __rtruediv__(self, other: StrPath) -> Path: ... - # These should be fully typed, either manually or with some magic wrapper - # function that copies the type of pathlib.Path except sticking an async in - # front of all of them. The latter is unfortunately not trivial, see attempts in - # https://github.com/python-trio/trio/issues/2630 - # wrapped methods handled by __getattr__ async def absolute(self) -> Path: ... async def as_posix(self) -> str: ... From dd9c15704b9c95befca142d4e95cf34605afa6b9 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 11:09:19 +1000 Subject: [PATCH 23/28] Fix type check errors --- trio/_file_io.py | 2 +- trio/_tests/test_file_io.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/trio/_file_io.py b/trio/_file_io.py index 8a3865eec3..6b79ae25b5 100644 --- a/trio/_file_io.py +++ b/trio/_file_io.py @@ -121,7 +121,7 @@ def isatty(self) -> bool: ... class _HasNewlines(Protocol[T_co]): # Type varies here - documented to be None, tuple of strings, strings. Typeshed uses Any. @property - def newlines(self) -> T: ... + def newlines(self) -> T_co: ... class _HasReadable(Protocol): def readable(self) -> bool: ... diff --git a/trio/_tests/test_file_io.py b/trio/_tests/test_file_io.py index 61345ca688..d95c7c3408 100644 --- a/trio/_tests/test_file_io.py +++ b/trio/_tests/test_file_io.py @@ -2,6 +2,7 @@ import io import os import re +from typing import List, Tuple from unittest import mock from unittest.mock import sentinel @@ -82,7 +83,12 @@ def unsupported_attr(self): # pragma: no cover def test_type_stubs_match_lists() -> None: """Check the manual stubs match the list of wrapped methods.""" - source = io.StringIO(_file_io.__spec__.loader.get_source("trio._file_io")) + # Fetch the module's source code. + assert _file_io.__spec__ is not None + loader = _file_io.__spec__.loader + assert isinstance(loader, importlib.abc.SourceLoader) + source = io.StringIO(loader.get_source("trio._file_io")) + # Find the class, then find the TYPE_CHECKING block. for line in source: if "class AsyncIOWrapper" in line: @@ -97,7 +103,7 @@ def test_type_stubs_match_lists() -> None: pytest.fail("No TYPE CHECKING line?") # Now we should be at the type checking block. - found: list[tuple[str, str]] = [] + found: List[Tuple[str, str]] = [] for line in source: if line.strip() and not line.startswith(" " * 8): break # Dedented out of the if TYPE_CHECKING block. From 9a333b11cd4d6bf976d830c95e0733287cb5c337 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 11:55:21 +1000 Subject: [PATCH 24/28] Exclude from coverage, these should never trigger. --- trio/_tests/test_file_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_tests/test_file_io.py b/trio/_tests/test_file_io.py index d95c7c3408..6428c72f90 100644 --- a/trio/_tests/test_file_io.py +++ b/trio/_tests/test_file_io.py @@ -93,13 +93,13 @@ def test_type_stubs_match_lists() -> None: for line in source: if "class AsyncIOWrapper" in line: break - else: + else: # pragma: no cover pytest.fail("No class definition line?") for line in source: if "if TYPE_CHECKING" in line: break - else: + else: # pragma: no cover pytest.fail("No TYPE CHECKING line?") # Now we should be at the type checking block. From 561fb6a3cb48585ed4aa2849fe47f721e3c28b76 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 21 Jul 2023 12:23:47 +1000 Subject: [PATCH 25/28] This shouldn't exhaust itself either --- trio/_tests/test_file_io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_tests/test_file_io.py b/trio/_tests/test_file_io.py index 6428c72f90..bae426cf48 100644 --- a/trio/_tests/test_file_io.py +++ b/trio/_tests/test_file_io.py @@ -93,18 +93,18 @@ def test_type_stubs_match_lists() -> None: for line in source: if "class AsyncIOWrapper" in line: break - else: # pragma: no cover + else: # pragma: no cover - should always find this pytest.fail("No class definition line?") for line in source: if "if TYPE_CHECKING" in line: break - else: # pragma: no cover + else: # pragma: no cover - should always find this pytest.fail("No TYPE CHECKING line?") # Now we should be at the type checking block. found: List[Tuple[str, str]] = [] - for line in source: + for line in source: # pragma: no branch - expected to break early if line.strip() and not line.startswith(" " * 8): break # Dedented out of the if TYPE_CHECKING block. match = re.match(r"\s*(async )?def ([a-zA-Z0-9_]+)\(", line) From b702c4f94303c51c07777790661830d407f8686e Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 27 Jul 2023 17:18:43 +1000 Subject: [PATCH 26/28] Apply suggested changes from A5Rocks --- trio/_path.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index c954cce7bb..c07f06c6e5 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -8,7 +8,7 @@ from collections.abc import Awaitable, Callable, Iterable from functools import partial, wraps from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper -from typing import IO, TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar, cast, overload +from typing import IO, TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar, Union, cast, overload import trio from trio._file_io import AsyncIOWrapper as _AsyncIOWrapper @@ -27,7 +27,7 @@ P = ParamSpec("P") T = TypeVar("T") -StrPath: TypeAlias = "str | os.PathLike[str]" +StrPath: TypeAlias = Union[str, os.PathLike[str]] # re-wrap return value from methods that return new instances of pathlib.Path @@ -60,7 +60,7 @@ def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> T | Path: def _forward_magic( cls: AsyncAutoWrapperType, attr: Callable[..., object] ) -> Callable[..., Path | Any]: - sentinel: Any = object() + sentinel = object() @wraps(attr) def wrapper(self: Path, other: object = sentinel) -> Any: From 65cd7f1c53dfe82aba8ab280854c294ff69d5ab6 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Thu, 27 Jul 2023 17:26:22 +1000 Subject: [PATCH 27/28] Pass tests --- trio/_path.py | 2 +- trio/_tests/verify_types.json | 25 +++++-------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index c07f06c6e5..1339a86900 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -27,7 +27,7 @@ P = ParamSpec("P") T = TypeVar("T") -StrPath: TypeAlias = Union[str, os.PathLike[str]] +StrPath: TypeAlias = Union[str, "os.PathLike[str]"] # Only subscriptable in 3.9+ # re-wrap return value from methods that return new instances of pathlib.Path diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 82903fa545..d08c03060c 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.872, + "completenessScore": 0.888, "exportedSymbolCounts": { "withAmbiguousType": 1, - "withKnownType": 545, - "withUnknownType": 79 + "withKnownType": 555, + "withUnknownType": 69 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, @@ -46,17 +46,12 @@ ], "otherSymbolCounts": { "withAmbiguousType": 3, - "withKnownType": 509, - "withUnknownType": 122 + "withKnownType": 529, + "withUnknownType": 102 }, "packageName": "trio", "symbols": [ "trio.__deprecated_attributes__", - "trio._abc.Instrument.after_task_step", - "trio._abc.Instrument.before_task_step", - "trio._abc.Instrument.task_exited", - "trio._abc.Instrument.task_scheduled", - "trio._abc.Instrument.task_spawned", "trio._abc.SocketFactory.socket", "trio._core._entry_queue.TrioToken.run_sync_soon", "trio._core._local.RunVar.__repr__", @@ -66,8 +61,6 @@ "trio._core._mock_clock.MockClock.jump", "trio._core._run.Nursery.start", "trio._core._run.Nursery.start_soon", - "trio._core._run.Task.eventual_parent_nursery", - "trio._core._run.Task.parent_nursery", "trio._core._run.TaskStatus.__repr__", "trio._core._run.TaskStatus.started", "trio._core._unbounded_queue.UnboundedQueue.__aiter__", @@ -125,27 +118,19 @@ "trio._subprocess.Process.send_signal", "trio._subprocess.Process.terminate", "trio._subprocess.Process.wait", - "trio._sync.Condition.__init__", - "trio._sync.Condition.statistics", - "trio._sync.Lock", - "trio._sync.StrictFIFOLock", - "trio._sync._LockImpl.statistics", "trio.current_time", "trio.from_thread.run", "trio.from_thread.run_sync", - "trio.lowlevel.add_instrument", "trio.lowlevel.cancel_shielded_checkpoint", "trio.lowlevel.current_clock", "trio.lowlevel.current_root_task", "trio.lowlevel.current_statistics", - "trio.lowlevel.current_task", "trio.lowlevel.current_trio_token", "trio.lowlevel.currently_ki_protected", "trio.lowlevel.notify_closing", "trio.lowlevel.open_process", "trio.lowlevel.permanently_detach_coroutine_object", "trio.lowlevel.reattach_detached_coroutine_object", - "trio.lowlevel.remove_instrument", "trio.lowlevel.reschedule", "trio.lowlevel.spawn_system_task", "trio.lowlevel.start_guest_run", From d90a4ed6b4082f4075aee012742e171bc1f299d1 Mon Sep 17 00:00:00 2001 From: Spencer Brown Date: Fri, 28 Jul 2023 11:05:55 +1000 Subject: [PATCH 28/28] Sort imports, replace a use of Any --- trio/_path.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/trio/_path.py b/trio/_path.py index 1339a86900..b7e6b16e4a 100644 --- a/trio/_path.py +++ b/trio/_path.py @@ -8,7 +8,17 @@ from collections.abc import Awaitable, Callable, Iterable from functools import partial, wraps from io import BufferedRandom, BufferedReader, BufferedWriter, FileIO, TextIOWrapper -from typing import IO, TYPE_CHECKING, Any, BinaryIO, ClassVar, TypeVar, Union, cast, overload +from typing import ( + IO, + TYPE_CHECKING, + Any, + BinaryIO, + ClassVar, + TypeVar, + Union, + cast, + overload, +) import trio from trio._file_io import AsyncIOWrapper as _AsyncIOWrapper @@ -58,12 +68,12 @@ def wrapper(self: Path, *args: P.args, **kwargs: P.kwargs) -> T | Path: def _forward_magic( - cls: AsyncAutoWrapperType, attr: Callable[..., object] -) -> Callable[..., Path | Any]: + cls: AsyncAutoWrapperType, attr: Callable[..., T] +) -> Callable[..., Path | T]: sentinel = object() @wraps(attr) - def wrapper(self: Path, other: object = sentinel) -> Any: + def wrapper(self: Path, other: object = sentinel) -> Path | T: if other is sentinel: return attr(self._wrapped) if isinstance(other, cls):