From 366ae6bf80cae8c35cfde1703de4a64d4881ccc9 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sat, 29 Jul 2023 20:10:47 -0500 Subject: [PATCH 01/14] Add type annotations to `_highlevel_open_tcp_stream.py` --- trio/_highlevel_open_tcp_stream.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index a2477104d9..2ef7fd75f8 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -1,5 +1,9 @@ +from __future__ import annotations + import sys +from collections.abc import Generator from contextlib import contextmanager +from socket import AddressFamily, SocketKind import trio from trio._core._multierror import MultiError @@ -109,8 +113,8 @@ @contextmanager -def close_all(): - sockets_to_close = set() +def close_all() -> Generator[set[socket], None, None]: + sockets_to_close: set[socket] = set() try: yield sockets_to_close finally: @@ -126,7 +130,9 @@ def close_all(): raise MultiError(errs) -def reorder_for_rfc_6555_section_5_4(targets): +def reorder_for_rfc_6555_section_5_4( + targets: list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int]]] +) -> None: # RFC 6555 section 5.4 says that if getaddrinfo returns multiple address # families (e.g. IPv4 and IPv6), then you should make sure that your first # and second attempts use different families: @@ -144,7 +150,7 @@ def reorder_for_rfc_6555_section_5_4(targets): break -def format_host_port(host, port): +def format_host_port(host: str | bytes, port: int) -> str: host = host.decode("ascii") if isinstance(host, bytes) else host if ":" in host: return f"[{host}]:{port}" @@ -173,8 +179,12 @@ def format_host_port(host, port): # AF_INET6: "..."} # this might be simpler after async def open_tcp_stream( - host, port, *, happy_eyeballs_delay=DEFAULT_DELAY, local_address=None -): + host: str | bytes, + port: int, + *, + happy_eyeballs_delay: float = DEFAULT_DELAY, + local_address: str | None = None, +) -> trio.abc.Stream: """Connect to the given host and port over TCP. If the given ``host`` has multiple IP addresses associated with it, then From 47db8710e6361b04915d7efb4189c281281479c1 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 04:15:27 -0500 Subject: [PATCH 02/14] Update `verify_types.json` --- trio/_tests/verify_types.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 60132e07fd..d49fae11f5 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.9072, + "completenessScore": 0.9088, "exportedSymbolCounts": { "withAmbiguousType": 1, - "withKnownType": 567, - "withUnknownType": 57 + "withKnownType": 568, + "withUnknownType": 56 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, @@ -122,7 +122,6 @@ "trio.open_ssl_over_tcp_listeners", "trio.open_ssl_over_tcp_stream", "trio.open_tcp_listeners", - "trio.open_tcp_stream", "trio.open_unix_socket", "trio.run", "trio.run_process", From b8bb9eed0f27373fca6c8c8b1434dd5840b3a937 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 04:40:29 -0500 Subject: [PATCH 03/14] Add missing annotations and fix others --- trio/_highlevel_open_tcp_stream.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 2ef7fd75f8..60df0d99e1 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -7,7 +7,7 @@ import trio from trio._core._multierror import MultiError -from trio.socket import SOCK_STREAM, getaddrinfo, socket +from trio.socket import SOCK_STREAM, _SocketType, getaddrinfo, socket if sys.version_info < (3, 11): from exceptiongroup import ExceptionGroup @@ -113,8 +113,8 @@ @contextmanager -def close_all() -> Generator[set[socket], None, None]: - sockets_to_close: set[socket] = set() +def close_all() -> Generator[set[_SocketType], None, None]: + sockets_to_close: set[_SocketType] = set() try: yield sockets_to_close finally: @@ -131,7 +131,15 @@ def close_all() -> Generator[set[socket], None, None]: def reorder_for_rfc_6555_section_5_4( - targets: list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int]]] + targets: list[ + tuple[ + AddressFamily, + SocketKind, + int, + str, + tuple[str, int] | tuple[str, int, int, int], + ] + ] ) -> None: # RFC 6555 section 5.4 says that if getaddrinfo returns multiple address # families (e.g. IPv4 and IPv6), then you should make sure that your first @@ -182,7 +190,7 @@ async def open_tcp_stream( host: str | bytes, port: int, *, - happy_eyeballs_delay: float = DEFAULT_DELAY, + happy_eyeballs_delay: float | None = DEFAULT_DELAY, local_address: str | None = None, ) -> trio.abc.Stream: """Connect to the given host and port over TCP. @@ -222,9 +230,9 @@ async def open_tcp_stream( port (int): The port to connect to. - happy_eyeballs_delay (float): How many seconds to wait for each + happy_eyeballs_delay (float or None): How many seconds to wait for each connection attempt to succeed or fail before getting impatient and - starting another one in parallel. Set to `math.inf` if you want + starting another one in parallel. Set to `None` if you want to limit to only one connection attempt at a time (like :func:`socket.create_connection`). Default: 0.25 (250 ms). @@ -284,7 +292,7 @@ async def open_tcp_stream( # Keeps track of the socket that we're going to complete with, # need to make sure this isn't automatically closed - winning_socket = None + winning_socket: _SocketType | None = None # Try connecting to the specified address. Possible outcomes: # - success: record connected socket in winning_socket and cancel @@ -293,7 +301,11 @@ async def open_tcp_stream( # the next connection attempt to start early # code needs to ensure sockets can be closed appropriately in the # face of crash or cancellation - async def attempt_connect(socket_args, sockaddr, attempt_failed): + async def attempt_connect( + socket_args: tuple[AddressFamily, SocketKind], + sockaddr: tuple[str, int] | tuple[str, int, int, int], + attempt_failed: trio.Event, + ) -> None: nonlocal winning_socket try: From 51347e3772aa54cbc5e1b431c285e01a7a813fc7 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 04:59:39 -0500 Subject: [PATCH 04/14] Add `_highlevel_open_tcp_stream.py` to mypy more strict checking block --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 445c40e28c..b8cf8ee52b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ module = [ "trio._core._local", "trio._sync", "trio._file_io", + "trio._highlevel_open_tcp_stream.py", ] disallow_incomplete_defs = true disallow_untyped_defs = true From 62b3c9a045241a3c92d3b3bcba88b7f240f97463 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 05:40:47 -0500 Subject: [PATCH 05/14] Change to `Address` where it makes sense --- trio/_highlevel_open_tcp_stream.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 60df0d99e1..9cba260b5d 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -7,7 +7,7 @@ import trio from trio._core._multierror import MultiError -from trio.socket import SOCK_STREAM, _SocketType, getaddrinfo, socket +from trio.socket import SOCK_STREAM, _SocketType, getaddrinfo, socket, Address if sys.version_info < (3, 11): from exceptiongroup import ExceptionGroup @@ -191,7 +191,7 @@ async def open_tcp_stream( port: int, *, happy_eyeballs_delay: float | None = DEFAULT_DELAY, - local_address: str | None = None, + local_address: Address | None = None, ) -> trio.abc.Stream: """Connect to the given host and port over TCP. From e82ce69116d0723dfcdcc15e20f2a4e9dd97b923 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 18:04:44 -0500 Subject: [PATCH 06/14] After discussion change to `Address` --- trio/_highlevel_open_tcp_stream.py | 6 +++--- trio/socket.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 9cba260b5d..d23c0f15ea 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -7,7 +7,7 @@ import trio from trio._core._multierror import MultiError -from trio.socket import SOCK_STREAM, _SocketType, getaddrinfo, socket, Address +from trio.socket import SOCK_STREAM, Address, _SocketType, getaddrinfo, socket if sys.version_info < (3, 11): from exceptiongroup import ExceptionGroup @@ -137,7 +137,7 @@ def reorder_for_rfc_6555_section_5_4( SocketKind, int, str, - tuple[str, int] | tuple[str, int, int, int], + Address, ] ] ) -> None: @@ -303,7 +303,7 @@ async def open_tcp_stream( # face of crash or cancellation async def attempt_connect( socket_args: tuple[AddressFamily, SocketKind], - sockaddr: tuple[str, int] | tuple[str, int, int, int], + sockaddr: Address, attempt_failed: trio.Event, ) -> None: nonlocal winning_socket diff --git a/trio/socket.py b/trio/socket.py index f6aebb6a6e..f8d0bc3fc2 100644 --- a/trio/socket.py +++ b/trio/socket.py @@ -34,6 +34,7 @@ # import the overwrites from ._socket import ( + Address as Address, SocketType as SocketType, _SocketType as _SocketType, from_stdlib_socket as from_stdlib_socket, From 1615e647ab828adfa6d4e3bd51303b660150382b Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 18:18:59 -0500 Subject: [PATCH 07/14] Update `verify_types.json` --- trio/_tests/verify_types.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index d49fae11f5..2d120c7899 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,10 +7,10 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.9088, + "completenessScore": 0.9089456869009584, "exportedSymbolCounts": { "withAmbiguousType": 1, - "withKnownType": 568, + "withKnownType": 569, "withUnknownType": 56 }, "ignoreUnknownTypesFromImports": true, From afaf15325816fd03f08ed46d09c9c5d41f8c8954 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 18:44:13 -0500 Subject: [PATCH 08/14] Basically revert e82ce69 from fixing mypy type issues --- trio/_highlevel_open_tcp_stream.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index d23c0f15ea..f1488cfa21 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -137,7 +137,7 @@ def reorder_for_rfc_6555_section_5_4( SocketKind, int, str, - Address, + tuple[str, int] | tuple[str, int, int, int], ] ] ) -> None: @@ -191,7 +191,7 @@ async def open_tcp_stream( port: int, *, happy_eyeballs_delay: float | None = DEFAULT_DELAY, - local_address: Address | None = None, + local_address: str | None = None, ) -> trio.abc.Stream: """Connect to the given host and port over TCP. @@ -265,9 +265,8 @@ async def open_tcp_stream( # To keep our public API surface smaller, rule out some cases that # getaddrinfo will accept in some circumstances, but that act weird or # have non-portable behavior or are just plain not useful. - # No type check on host though b/c we want to allow bytes-likes. - if host is None: - raise ValueError("host cannot be None") + if not isinstance(host, (str, bytes)): + raise ValueError(f"host must be str or bytes, not {host!r}") if not isinstance(port, int): raise TypeError(f"port must be int, not {port!r}") @@ -356,7 +355,7 @@ async def attempt_connect( except OSError: raise OSError( f"local_address={local_address!r} is incompatible " - f"with remote address {sockaddr}" + f"with remote address {sockaddr!r}" ) await sock.connect(sockaddr) From 02d7630681c0b0769242419ffe9e534535f21237 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Sun, 30 Jul 2023 18:50:35 -0500 Subject: [PATCH 09/14] Update `verify_types.json` --- trio/_tests/verify_types.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index f06722ee22..0f6976fadf 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.9089456869009584, + "completenessScore": 0.9121405750798722, "exportedSymbolCounts": { "withAmbiguousType": 0, - "withKnownType": 569, - "withUnknownType": 56 + "withKnownType": 571, + "withUnknownType": 55 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1, From 0631cf9e85b33b709c9438d742375026324b5144 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 31 Jul 2023 14:49:51 +0200 Subject: [PATCH 10/14] fix unpacking not to lose types, add hacky workaround to get the types actually checked despite calling through nursery.start_soon --- trio/_highlevel_open_tcp_stream.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index f1488cfa21..98da2b2f4e 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -4,6 +4,7 @@ from collections.abc import Generator from contextlib import contextmanager from socket import AddressFamily, SocketKind +from typing import TYPE_CHECKING import trio from trio._core._multierror import MultiError @@ -376,12 +377,20 @@ async def attempt_connect( # nursery spawns a task for each connection attempt, will be # cancelled by the task that gets a successful connection async with trio.open_nursery() as nursery: - for *sa, _, addr in targets: + for address_family, socket_kind, *_, addr in targets: # create an event to indicate connection failure, # allowing the next target to be tried early attempt_failed = trio.Event() - nursery.start_soon(attempt_connect, sa, addr, attempt_failed) + # workaround to check types until typing of nursery.start_soon improved + if TYPE_CHECKING: + await attempt_connect( + (address_family, socket_kind), addr, attempt_failed + ) + + nursery.start_soon( + attempt_connect, (address_family, socket_kind), addr, attempt_failed + ) # give this attempt at most this time before moving on with trio.move_on_after(happy_eyeballs_delay): From a13c3cd6d37344ed60e73384766c948b1d65a61a Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 31 Jul 2023 15:05:58 +0200 Subject: [PATCH 11/14] undo changes to behaviour, fix signature --- trio/_highlevel_open_tcp_stream.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/trio/_highlevel_open_tcp_stream.py b/trio/_highlevel_open_tcp_stream.py index 98da2b2f4e..0c4e8a4a8d 100644 --- a/trio/_highlevel_open_tcp_stream.py +++ b/trio/_highlevel_open_tcp_stream.py @@ -302,7 +302,7 @@ async def open_tcp_stream( # code needs to ensure sockets can be closed appropriately in the # face of crash or cancellation async def attempt_connect( - socket_args: tuple[AddressFamily, SocketKind], + socket_args: tuple[AddressFamily, SocketKind, int], sockaddr: Address, attempt_failed: trio.Event, ) -> None: @@ -377,7 +377,7 @@ async def attempt_connect( # nursery spawns a task for each connection attempt, will be # cancelled by the task that gets a successful connection async with trio.open_nursery() as nursery: - for address_family, socket_kind, *_, addr in targets: + for address_family, socket_type, proto, _, addr in targets: # create an event to indicate connection failure, # allowing the next target to be tried early attempt_failed = trio.Event() @@ -385,11 +385,14 @@ async def attempt_connect( # workaround to check types until typing of nursery.start_soon improved if TYPE_CHECKING: await attempt_connect( - (address_family, socket_kind), addr, attempt_failed + (address_family, socket_type, proto), addr, attempt_failed ) nursery.start_soon( - attempt_connect, (address_family, socket_kind), addr, attempt_failed + attempt_connect, + (address_family, socket_type, proto), + addr, + attempt_failed, ) # give this attempt at most this time before moving on From cf546ed0cde1dbf0f9621c38c8576ca7b4ba53b9 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 31 Jul 2023 15:11:07 +0200 Subject: [PATCH 12/14] test --- test | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test diff --git a/test b/test new file mode 100644 index 0000000000..e69de29bb2 From aa4d42a957b44a67fc3e3a8c9220a6249f55e849 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 31 Jul 2023 15:11:58 +0200 Subject: [PATCH 13/14] Revert "test" This reverts commit cf546ed0cde1dbf0f9621c38c8576ca7b4ba53b9. --- test | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 test diff --git a/test b/test deleted file mode 100644 index e69de29bb2..0000000000 From e1b4b3f8ca7a63e43f9cf8e67500615d8739e8f3 Mon Sep 17 00:00:00 2001 From: CoolCat467 <52022020+CoolCat467@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:17:39 -0500 Subject: [PATCH 14/14] Update `verify_types.json` --- trio/_tests/verify_types.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/trio/_tests/verify_types.json b/trio/_tests/verify_types.json index 2fff667e6a..ac2cfbd197 100644 --- a/trio/_tests/verify_types.json +++ b/trio/_tests/verify_types.json @@ -7,11 +7,11 @@ "warningCount": 0 }, "typeCompleteness": { - "completenessScore": 0.9138755980861244, + "completenessScore": 0.9154704944178629, "exportedSymbolCounts": { "withAmbiguousType": 0, - "withKnownType": 573, - "withUnknownType": 54 + "withKnownType": 574, + "withUnknownType": 53 }, "ignoreUnknownTypesFromImports": true, "missingClassDocStringCount": 1,