From 970bdd5d1bb32760e1cbac8dc1b231b6097c1589 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Wed, 17 Mar 2021 12:27:31 +0000 Subject: [PATCH 1/9] Added httpx.BaseTransport and httpx.AsyncBaseTransport --- docs/advanced.md | 16 ++++---- httpx/__init__.py | 3 ++ httpx/_client.py | 39 +++++++++----------- httpx/_transports/asgi.py | 23 ++++++++---- httpx/_transports/base.py | 61 +++++++++++++++++++++++++++++++ httpx/_transports/default.py | 19 +++++----- httpx/_transports/mock.py | 14 +++---- httpx/_transports/wsgi.py | 15 ++++---- tests/client/test_async_client.py | 8 ++-- tests/client/test_client.py | 9 ++--- tests/test_exceptions.py | 2 +- 11 files changed, 137 insertions(+), 72 deletions(-) create mode 100644 httpx/_transports/base.py diff --git a/docs/advanced.md b/docs/advanced.md index 61bf4c1938..c023a8a12d 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -1015,20 +1015,20 @@ This [public gist](https://gist.github.com/florimondmanca/d56764d78d748eb9f73165 ### Writing custom transports -A transport instance must implement the Transport API defined by -[`httpcore`](https://www.encode.io/httpcore/api/). You -should either subclass `httpcore.AsyncHTTPTransport` to implement a transport to -use with `AsyncClient`, or subclass `httpcore.SyncHTTPTransport` to implement a -transport to use with `Client`. +A transport instance must implement the low-level Transport API, which deals +with sending a single request, and returning a response. You should either +subclass `httpx.BaseTransport` to implement a transport to use with `Client`, +or subclass `httpx.AsyncBaseTransport` to implement a transport to +use with `AsyncClient`. A complete example of a custom transport implementation would be: ```python import json -import httpcore +import httpx -class HelloWorldTransport(httpcore.SyncHTTPTransport): +class HelloWorldTransport(httpx.BaseTransport): """ A mock transport that always returns a JSON "Hello, world!" response. """ @@ -1036,7 +1036,7 @@ class HelloWorldTransport(httpcore.SyncHTTPTransport): def request(self, method, url, headers=None, stream=None, ext=None): message = {"text": "Hello, world!"} content = json.dumps(message).encode("utf-8") - stream = httpcore.PlainByteStream(content) + stream = [content] headers = [(b"content-type", b"application/json")] ext = {"http_version": b"HTTP/1.1"} return 200, headers, stream, ext diff --git a/httpx/__init__.py b/httpx/__init__.py index 96d9e0c2f8..a441669bf6 100644 --- a/httpx/__init__.py +++ b/httpx/__init__.py @@ -36,6 +36,7 @@ from ._models import URL, Cookies, Headers, QueryParams, Request, Response from ._status_codes import StatusCode, codes from ._transports.asgi import ASGITransport +from ._transports.base import AsyncBaseTransport, BaseTransport from ._transports.default import AsyncHTTPTransport, HTTPTransport from ._transports.mock import MockTransport from ._transports.wsgi import WSGITransport @@ -45,9 +46,11 @@ "__title__", "__version__", "ASGITransport", + "AsyncBaseTransport", "AsyncClient", "AsyncHTTPTransport", "Auth", + "BaseTransport", "BasicAuth", "Client", "CloseError", diff --git a/httpx/_client.py b/httpx/_client.py index 3465a10b75..d8ba38f499 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -4,8 +4,6 @@ import warnings from types import TracebackType -import httpcore - from .__version__ import __version__ from ._auth import Auth, BasicAuth, FunctionAuth from ._config import ( @@ -29,6 +27,7 @@ from ._models import URL, Cookies, Headers, QueryParams, Request, Response from ._status_codes import codes from ._transports.asgi import ASGITransport +from ._transports.base import AsyncBaseTransport, BaseTransport from ._transports.default import AsyncHTTPTransport, HTTPTransport from ._transports.wsgi import WSGITransport from ._types import ( @@ -560,14 +559,14 @@ def __init__( cert: CertTypes = None, http2: bool = False, proxies: ProxiesTypes = None, - mounts: typing.Mapping[str, httpcore.SyncHTTPTransport] = None, + mounts: typing.Mapping[str, BaseTransport] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, limits: Limits = DEFAULT_LIMITS, pool_limits: Limits = None, max_redirects: int = DEFAULT_MAX_REDIRECTS, event_hooks: typing.Mapping[str, typing.List[typing.Callable]] = None, base_url: URLTypes = "", - transport: httpcore.SyncHTTPTransport = None, + transport: BaseTransport = None, app: typing.Callable = None, trust_env: bool = True, ): @@ -611,9 +610,7 @@ def __init__( app=app, trust_env=trust_env, ) - self._mounts: typing.Dict[ - URLPattern, typing.Optional[httpcore.SyncHTTPTransport] - ] = { + self._mounts: typing.Dict[URLPattern, typing.Optional[BaseTransport]] = { URLPattern(key): None if proxy is None else self._init_proxy_transport( @@ -639,10 +636,10 @@ def _init_transport( cert: CertTypes = None, http2: bool = False, limits: Limits = DEFAULT_LIMITS, - transport: httpcore.SyncHTTPTransport = None, + transport: BaseTransport = None, app: typing.Callable = None, trust_env: bool = True, - ) -> httpcore.SyncHTTPTransport: + ) -> BaseTransport: if transport is not None: return transport @@ -661,7 +658,7 @@ def _init_proxy_transport( http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, - ) -> httpcore.SyncHTTPTransport: + ) -> BaseTransport: return HTTPTransport( verify=verify, cert=cert, @@ -671,7 +668,7 @@ def _init_proxy_transport( proxy=proxy, ) - def _transport_for_url(self, url: URL) -> httpcore.SyncHTTPTransport: + def _transport_for_url(self, url: URL) -> BaseTransport: """ Returns the transport instance that should be used for a given URL. This will either be the standard connection pool, or a proxy. @@ -864,7 +861,7 @@ def _send_single_request(self, request: Request, timeout: Timeout) -> Response: def on_close(response: Response) -> None: response.elapsed = datetime.timedelta(seconds=timer.sync_elapsed()) if hasattr(stream, "close"): - stream.close() + stream.close() # type: ignore response = Response( status_code, @@ -1193,14 +1190,14 @@ def __init__( cert: CertTypes = None, http2: bool = False, proxies: ProxiesTypes = None, - mounts: typing.Mapping[str, httpcore.AsyncHTTPTransport] = None, + mounts: typing.Mapping[str, AsyncBaseTransport] = None, timeout: TimeoutTypes = DEFAULT_TIMEOUT_CONFIG, limits: Limits = DEFAULT_LIMITS, pool_limits: Limits = None, max_redirects: int = DEFAULT_MAX_REDIRECTS, event_hooks: typing.Mapping[str, typing.List[typing.Callable]] = None, base_url: URLTypes = "", - transport: httpcore.AsyncHTTPTransport = None, + transport: AsyncBaseTransport = None, app: typing.Callable = None, trust_env: bool = True, ): @@ -1245,9 +1242,7 @@ def __init__( trust_env=trust_env, ) - self._mounts: typing.Dict[ - URLPattern, typing.Optional[httpcore.AsyncHTTPTransport] - ] = { + self._mounts: typing.Dict[URLPattern, typing.Optional[AsyncBaseTransport]] = { URLPattern(key): None if proxy is None else self._init_proxy_transport( @@ -1272,10 +1267,10 @@ def _init_transport( cert: CertTypes = None, http2: bool = False, limits: Limits = DEFAULT_LIMITS, - transport: httpcore.AsyncHTTPTransport = None, + transport: AsyncBaseTransport = None, app: typing.Callable = None, trust_env: bool = True, - ) -> httpcore.AsyncHTTPTransport: + ) -> AsyncBaseTransport: if transport is not None: return transport @@ -1294,7 +1289,7 @@ def _init_proxy_transport( http2: bool = False, limits: Limits = DEFAULT_LIMITS, trust_env: bool = True, - ) -> httpcore.AsyncHTTPTransport: + ) -> AsyncBaseTransport: return AsyncHTTPTransport( verify=verify, cert=cert, @@ -1304,7 +1299,7 @@ def _init_proxy_transport( proxy=proxy, ) - def _transport_for_url(self, url: URL) -> httpcore.AsyncHTTPTransport: + def _transport_for_url(self, url: URL) -> AsyncBaseTransport: """ Returns the transport instance that should be used for a given URL. This will either be the standard connection pool, or a proxy. @@ -1501,7 +1496,7 @@ async def on_close(response: Response) -> None: response.elapsed = datetime.timedelta(seconds=await timer.async_elapsed()) if hasattr(stream, "aclose"): with map_exceptions(HTTPCORE_EXC_MAP, request=request): - await stream.aclose() + await stream.aclose() # type: ignore response = Response( status_code, diff --git a/httpx/_transports/asgi.py b/httpx/_transports/asgi.py index 758d8375b2..a30495eccd 100644 --- a/httpx/_transports/asgi.py +++ b/httpx/_transports/asgi.py @@ -1,9 +1,11 @@ +import typing from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union from urllib.parse import unquote -import httpcore import sniffio +from .base import AsyncBaseTransport + if TYPE_CHECKING: # pragma: no cover import asyncio @@ -23,7 +25,7 @@ def create_event() -> "Event": return asyncio.Event() -class ASGITransport(httpcore.AsyncHTTPTransport): +class ASGITransport(AsyncBaseTransport): """ A custom AsyncTransport that handles sending requests directly to an ASGI app. The simplest way to use this functionality is to use the `app` argument. @@ -73,11 +75,10 @@ async def arequest( method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, - stream: httpcore.AsyncByteStream = None, + stream: typing.AsyncIterator[bytes] = None, ext: dict = None, - ) -> Tuple[int, List[Tuple[bytes, bytes]], httpcore.AsyncByteStream, dict]: + ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict]: headers = [] if headers is None else headers - stream = httpcore.PlainByteStream(content=b"") if stream is None else stream # ASGI scope. scheme, host, port, full_path = url @@ -98,7 +99,7 @@ async def arequest( } # Request. - request_body_chunks = stream.__aiter__() + request_body_chunks = None if stream is None else stream.__aiter__() request_complete = False # Response. @@ -117,6 +118,10 @@ async def receive() -> dict: await response_complete.wait() return {"type": "http.disconnect"} + if request_body_chunks is None: + request_complete = True + return {"type": "http.request", "body": b"", "more_body": False} + try: body = await request_body_chunks.__anext__() except StopAsyncIteration: @@ -155,7 +160,9 @@ async def send(message: dict) -> None: assert status_code is not None assert response_headers is not None - stream = httpcore.PlainByteStream(content=b"".join(body_parts)) + async def response_stream() -> typing.AsyncIterator[bytes]: + yield b"".join(body_parts) + ext = {} - return (status_code, response_headers, stream, ext) + return (status_code, response_headers, response_stream(), ext) diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py new file mode 100644 index 0000000000..860c7df49d --- /dev/null +++ b/httpx/_transports/base.py @@ -0,0 +1,61 @@ +import typing +from types import TracebackType + +T = typing.TypeVar("T", bound="BaseTransport") +A = typing.TypeVar("A", bound="AsyncBaseTransport") + + +class BaseTransport: + def __enter__(self: T) -> T: + return self + + def __exit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + pass + + def request( + self, + method: bytes, + url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], + headers: typing.List[typing.Tuple[bytes, bytes]] = None, + stream: typing.Iterator[bytes] = None, + ext: dict = None, + ) -> typing.Tuple[ + int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict + ]: + pass + + def close(self) -> None: + pass + + +class AsyncBaseTransport: + async def __aenter__(self: A) -> A: + return self + + async def __aexit__( + self, + exc_type: typing.Type[BaseException] = None, + exc_value: BaseException = None, + traceback: TracebackType = None, + ) -> None: + pass + + async def arequest( + self, + method: bytes, + url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], + headers: typing.List[typing.Tuple[bytes, bytes]] = None, + stream: typing.AsyncIterator[bytes] = None, + ext: dict = None, + ) -> typing.Tuple[ + int, typing.List[typing.Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict + ]: + pass + + async def aclose(self) -> None: + pass diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index 84aeb26be8..daf08ed48c 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -31,6 +31,7 @@ from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context from .._types import CertTypes, VerifyTypes +from .base import AsyncBaseTransport, BaseTransport T = typing.TypeVar("T", bound="HTTPTransport") A = typing.TypeVar("A", bound="AsyncHTTPTransport") @@ -38,7 +39,7 @@ URL = typing.Tuple[bytes, bytes, typing.Optional[int], bytes] -class HTTPTransport(httpcore.SyncHTTPTransport): +class HTTPTransport(BaseTransport): def __init__( self, verify: VerifyTypes = True, @@ -96,16 +97,16 @@ def request( method: bytes, url: URL, headers: Headers = None, - stream: httpcore.SyncByteStream = None, + stream: typing.Iterator[bytes] = None, ext: dict = None, - ) -> typing.Tuple[int, Headers, httpcore.SyncByteStream, dict]: - return self._pool.request(method, url, headers=headers, stream=stream, ext=ext) + ) -> typing.Tuple[int, Headers, typing.Iterator[bytes], dict]: + return self._pool.request(method, url, headers=headers, stream=stream, ext=ext) # type: ignore def close(self) -> None: self._pool.close() -class AsyncHTTPTransport(httpcore.AsyncHTTPTransport): +class AsyncHTTPTransport(AsyncBaseTransport): def __init__( self, verify: VerifyTypes = True, @@ -163,11 +164,11 @@ async def arequest( method: bytes, url: URL, headers: Headers = None, - stream: httpcore.AsyncByteStream = None, + stream: typing.AsyncIterator[bytes] = None, ext: dict = None, - ) -> typing.Tuple[int, Headers, httpcore.AsyncByteStream, dict]: - return await self._pool.arequest( - method, url, headers=headers, stream=stream, ext=ext + ) -> typing.Tuple[int, Headers, typing.AsyncIterator[bytes], dict]: + return await self._pool.arequest( # type: ignore + method, url, headers=headers, stream=stream, ext=ext # type: ignore ) async def aclose(self) -> None: diff --git a/httpx/_transports/mock.py b/httpx/_transports/mock.py index a55a88b7a2..472bfc6dff 100644 --- a/httpx/_transports/mock.py +++ b/httpx/_transports/mock.py @@ -1,12 +1,12 @@ import asyncio +import typing from typing import Callable, List, Optional, Tuple -import httpcore - from .._models import Request +from .base import AsyncBaseTransport, BaseTransport -class MockTransport(httpcore.SyncHTTPTransport, httpcore.AsyncHTTPTransport): +class MockTransport(AsyncBaseTransport, BaseTransport): def __init__(self, handler: Callable) -> None: self.handler = handler @@ -15,9 +15,9 @@ def request( method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, - stream: httpcore.SyncByteStream = None, + stream: typing.Iterator[bytes] = None, ext: dict = None, - ) -> Tuple[int, List[Tuple[bytes, bytes]], httpcore.SyncByteStream, dict]: + ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.Iterator[bytes], dict]: request = Request( method=method, url=url, @@ -38,9 +38,9 @@ async def arequest( method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, - stream: httpcore.AsyncByteStream = None, + stream: typing.AsyncIterator[bytes] = None, ext: dict = None, - ) -> Tuple[int, List[Tuple[bytes, bytes]], httpcore.AsyncByteStream, dict]: + ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict]: request = Request( method=method, url=url, diff --git a/httpx/_transports/wsgi.py b/httpx/_transports/wsgi.py index 67b44bde42..d4a5ddb316 100644 --- a/httpx/_transports/wsgi.py +++ b/httpx/_transports/wsgi.py @@ -3,7 +3,7 @@ import typing from urllib.parse import unquote -import httpcore +from .base import BaseTransport def _skip_leading_empty_chunks(body: typing.Iterable) -> typing.Iterable: @@ -14,7 +14,7 @@ def _skip_leading_empty_chunks(body: typing.Iterable) -> typing.Iterable: return [] -class WSGITransport(httpcore.SyncHTTPTransport): +class WSGITransport(BaseTransport): """ A custom transport that handles sending requests directly to an WSGI app. The simplest way to use this functionality is to use the `app` argument. @@ -64,13 +64,13 @@ def request( method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], headers: typing.List[typing.Tuple[bytes, bytes]] = None, - stream: httpcore.SyncByteStream = None, + stream: typing.Iterator[bytes] = None, ext: dict = None, ) -> typing.Tuple[ - int, typing.List[typing.Tuple[bytes, bytes]], httpcore.SyncByteStream, dict + int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: headers = [] if headers is None else headers - stream = httpcore.PlainByteStream(content=b"") if stream is None else stream + wsgi_input = io.BytesIO() if stream is None else io.BytesIO(b"".join(stream)) scheme, host, port, full_path = url path, _, query = full_path.partition(b"?") @@ -80,7 +80,7 @@ def request( environ = { "wsgi.version": (1, 0), "wsgi.url_scheme": scheme.decode("ascii"), - "wsgi.input": io.BytesIO(b"".join(stream)), + "wsgi.input": wsgi_input, "wsgi.errors": io.BytesIO(), "wsgi.multithread": True, "wsgi.multiprocess": False, @@ -126,7 +126,6 @@ def start_response( (key.encode("ascii"), value.encode("ascii")) for key, value in seen_response_headers ] - stream = httpcore.IteratorByteStream(iterator=result) ext = {} - return (status_code, headers, stream, ext) + return (status_code, headers, result, ext) diff --git a/tests/client/test_async_client.py b/tests/client/test_async_client.py index 1d3f4ccafa..42b612bfa7 100644 --- a/tests/client/test_async_client.py +++ b/tests/client/test_async_client.py @@ -169,12 +169,12 @@ async def test_100_continue(server): @pytest.mark.usefixtures("async_environment") async def test_context_managed_transport(): - class Transport(httpcore.AsyncHTTPTransport): + class Transport(httpx.AsyncBaseTransport): def __init__(self): self.events = [] async def aclose(self): - # The base implementation of httpcore.AsyncHTTPTransport just + # The base implementation of httpx.AsyncBaseTransport just # calls into `.aclose`, so simple transport cases can just override # this method for any cleanup, where more complex cases # might want to additionally override `__aenter__`/`__aexit__`. @@ -201,13 +201,13 @@ async def __aexit__(self, *args): @pytest.mark.usefixtures("async_environment") async def test_context_managed_transport_and_mount(): - class Transport(httpcore.AsyncHTTPTransport): + class Transport(httpx.AsyncBaseTransport): def __init__(self, name: str): self.name: str = name self.events: typing.List[str] = [] async def aclose(self): - # The base implementation of httpcore.AsyncHTTPTransport just + # The base implementation of httpx.AsyncBaseTransport just # calls into `.aclose`, so simple transport cases can just override # this method for any cleanup, where more complex cases # might want to additionally override `__aenter__`/`__aexit__`. diff --git a/tests/client/test_client.py b/tests/client/test_client.py index 13bb7f03ad..ad367a96a7 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -1,7 +1,6 @@ import typing from datetime import timedelta -import httpcore import pytest import httpx @@ -218,12 +217,12 @@ def test_pool_limits_deprecated(): def test_context_managed_transport(): - class Transport(httpcore.SyncHTTPTransport): + class Transport(httpx.BaseTransport): def __init__(self): self.events = [] def close(self): - # The base implementation of httpcore.SyncHTTPTransport just + # The base implementation of httpx.BaseTransport just # calls into `.close`, so simple transport cases can just override # this method for any cleanup, where more complex cases # might want to additionally override `__enter__`/`__exit__`. @@ -249,13 +248,13 @@ def __exit__(self, *args): def test_context_managed_transport_and_mount(): - class Transport(httpcore.SyncHTTPTransport): + class Transport(httpx.BaseTransport): def __init__(self, name: str): self.name: str = name self.events: typing.List[str] = [] def close(self): - # The base implementation of httpcore.SyncHTTPTransport just + # The base implementation of httpx.BaseTransport just # calls into `.close`, so simple transport cases can just override # this method for any cleanup, where more complex cases # might want to additionally override `__enter__`/`__exit__`. diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index f1c7005bba..1e18e5960b 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -41,7 +41,7 @@ def test_httpcore_exception_mapping(server) -> None: stream.read() # Make sure it also works with custom transports. - class MockTransport(httpcore.SyncHTTPTransport): + class MockTransport(httpx.BaseTransport): def request(self, *args: Any, **kwargs: Any) -> Any: raise httpcore.ProtocolError() From 38fd8051ddfa5c3a31611a9f33ce98e596cb2d08 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 18 Mar 2021 11:34:06 +0000 Subject: [PATCH 2/9] Test coverage and default transports to calling .close on __exit__ --- httpx/_transports/base.py | 12 ++++++++---- tests/test_asgi.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py index 860c7df49d..adefad9243 100644 --- a/httpx/_transports/base.py +++ b/httpx/_transports/base.py @@ -15,7 +15,7 @@ def __exit__( exc_value: BaseException = None, traceback: TracebackType = None, ) -> None: - pass + self.close() def request( self, @@ -27,7 +27,9 @@ def request( ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: - pass + raise NotImplementedError( + "The 'request' method must be implemented." + ) # pragma: nocover def close(self) -> None: pass @@ -43,7 +45,7 @@ async def __aexit__( exc_value: BaseException = None, traceback: TracebackType = None, ) -> None: - pass + await self.aclose() async def arequest( self, @@ -55,7 +57,9 @@ async def arequest( ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict ]: - pass + raise NotImplementedError( + "The 'arequest' method must be implemented." + ) # pragma: nocover async def aclose(self) -> None: pass diff --git a/tests/test_asgi.py b/tests/test_asgi.py index b16f68246c..7b750b8cc2 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -70,6 +70,30 @@ async def raise_exc_after_response(scope, receive, send): raise RuntimeError() +@pytest.mark.usefixtures("async_environment") +async def test_asgi_transport(): + async with httpx.ASGITransport(app=hello_world) as transport: + status_code, headers, stream, ext = await transport.arequest( + b"GET", (b"http", b"www.example.org", 80, b"/") + ) + body = b"".join([part async for part in stream]) + + assert status_code == 200 + assert body == b"Hello, World!" + + +@pytest.mark.usefixtures("async_environment") +async def test_asgi_transport_no_body(): + async with httpx.ASGITransport(app=echo_body) as transport: + status_code, headers, stream, ext = await transport.arequest( + b"GET", (b"http", b"www.example.org", 80, b"/") + ) + body = b"".join([part async for part in stream]) + + assert status_code == 200 + assert body == b"" + + @pytest.mark.usefixtures("async_environment") async def test_asgi(): async with httpx.AsyncClient(app=hello_world) as client: From 9d413e7bba49f0a386e5a9bad6c0f0b334f0ab46 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Thu, 18 Mar 2021 11:34:19 +0000 Subject: [PATCH 3/9] BaseTransport documentation --- docs/advanced.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index c023a8a12d..2a829851d2 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -1084,10 +1084,9 @@ which transport an outgoing request should be routed via, with [the same style used for specifying proxy routing](#routing). ```python -import httpcore import httpx -class HTTPSRedirectTransport(httpcore.SyncHTTPTransport): +class HTTPSRedirectTransport(httpx.BaseTransport): """ A transport that always redirects to HTTPS. """ @@ -1098,7 +1097,7 @@ class HTTPSRedirectTransport(httpcore.SyncHTTPTransport): location = b"https://%s%s" % (host, path) else: location = b"https://%s:%d%s" % (host, port, path) - stream = httpcore.PlainByteStream(b"") + stream = [b""] headers = [(b"location", location)] ext = {"http_version": b"HTTP/1.1"} return 303, headers, stream, ext From 09e17eb4dc2962e70904313d191d6dc26e3759a3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 22 Mar 2021 10:43:29 +0000 Subject: [PATCH 4/9] Use 'handle_request' for the transport API. --- docs/advanced.md | 16 +++++++++--- httpx/_client.py | 17 ++++++++----- httpx/_models.py | 8 +++--- httpx/_transports/asgi.py | 8 +++--- httpx/_transports/base.py | 49 +++++++++++++++++++++++++++++++----- httpx/_transports/default.py | 30 +++++++++++----------- httpx/_transports/mock.py | 12 ++++----- httpx/_transports/wsgi.py | 8 +++--- tests/test_asgi.py | 4 +-- tests/test_exceptions.py | 2 +- 10 files changed, 103 insertions(+), 51 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 2a829851d2..51773365e1 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -1021,6 +1021,14 @@ subclass `httpx.BaseTransport` to implement a transport to use with `Client`, or subclass `httpx.AsyncBaseTransport` to implement a transport to use with `AsyncClient`. +At the layer of the transport API we're simply using plain primitives. +No `Request` or `Response` models, no fancy `URL` or `Header` handling. +This strict point of cut-off provides a clear design separation between the +HTTPX API, and the low-level network handling. + +See the `handle_request` and `handle_async_request` docstrings for more details +on the specifics of the Transport API. + A complete example of a custom transport implementation would be: ```python @@ -1033,12 +1041,12 @@ class HelloWorldTransport(httpx.BaseTransport): A mock transport that always returns a JSON "Hello, world!" response. """ - def request(self, method, url, headers=None, stream=None, ext=None): + def handle_request(self, method, url, headers=None, stream=None, ext=None): message = {"text": "Hello, world!"} content = json.dumps(message).encode("utf-8") stream = [content] headers = [(b"content-type", b"application/json")] - ext = {"http_version": b"HTTP/1.1"} + ext = {"http_version": "HTTP/1.1"} return 200, headers, stream, ext ``` @@ -1091,7 +1099,7 @@ class HTTPSRedirectTransport(httpx.BaseTransport): A transport that always redirects to HTTPS. """ - def request(self, method, url, headers=None, stream=None, ext=None): + def handle_request(self, method, url, headers=None, stream=None, ext=None): scheme, host, port, path = url if port is None: location = b"https://%s%s" % (host, path) @@ -1099,7 +1107,7 @@ class HTTPSRedirectTransport(httpx.BaseTransport): location = b"https://%s:%d%s" % (host, port, path) stream = [b""] headers = [(b"location", location)] - ext = {"http_version": b"HTTP/1.1"} + ext = {"http_version": "HTTP/1.1"} return 303, headers, stream, ext diff --git a/httpx/_client.py b/httpx/_client.py index d8ba38f499..906eb7dad8 100644 --- a/httpx/_client.py +++ b/httpx/_client.py @@ -850,12 +850,12 @@ def _send_single_request(self, request: Request, timeout: Timeout) -> Response: timer.sync_start() with map_exceptions(HTTPCORE_EXC_MAP, request=request): - (status_code, headers, stream, ext) = transport.request( + (status_code, headers, stream, extensions) = transport.handle_request( request.method.encode(), request.url.raw, headers=request.headers.raw, stream=request.stream, # type: ignore - ext={"timeout": timeout.as_dict()}, + extensions={"timeout": timeout.as_dict()}, ) def on_close(response: Response) -> None: @@ -867,7 +867,7 @@ def on_close(response: Response) -> None: status_code, headers=headers, stream=stream, # type: ignore - ext=ext, + extensions=extensions, request=request, on_close=on_close, ) @@ -1484,12 +1484,17 @@ async def _send_single_request( await timer.async_start() with map_exceptions(HTTPCORE_EXC_MAP, request=request): - (status_code, headers, stream, ext) = await transport.arequest( + ( + status_code, + headers, + stream, + extensions, + ) = await transport.handle_async_request( request.method.encode(), request.url.raw, headers=request.headers.raw, stream=request.stream, # type: ignore - ext={"timeout": timeout.as_dict()}, + extensions={"timeout": timeout.as_dict()}, ) async def on_close(response: Response) -> None: @@ -1502,7 +1507,7 @@ async def on_close(response: Response) -> None: status_code, headers=headers, stream=stream, # type: ignore - ext=ext, + extensions=extensions, request=request, on_close=on_close, ) diff --git a/httpx/_models.py b/httpx/_models.py index 2d11888254..be6d4c27c9 100644 --- a/httpx/_models.py +++ b/httpx/_models.py @@ -902,7 +902,7 @@ def __init__( json: typing.Any = None, stream: ByteStream = None, request: Request = None, - ext: dict = None, + extensions: dict = None, history: typing.List["Response"] = None, on_close: typing.Callable = None, ): @@ -917,7 +917,7 @@ def __init__( self.call_next: typing.Optional[typing.Callable] = None - self.ext = {} if ext is None else ext + self.extensions = {} if extensions is None else extensions self.history = [] if history is None else list(history) self._on_close = on_close @@ -988,11 +988,11 @@ def request(self, value: Request) -> None: @property def http_version(self) -> str: - return self.ext.get("http_version", "HTTP/1.1") + return self.extensions.get("http_version", "HTTP/1.1") @property def reason_phrase(self) -> str: - return self.ext.get("reason", codes.get_reason_phrase(self.status_code)) + return self.extensions.get("reason", codes.get_reason_phrase(self.status_code)) @property def url(self) -> typing.Optional[URL]: diff --git a/httpx/_transports/asgi.py b/httpx/_transports/asgi.py index a30495eccd..23691502e5 100644 --- a/httpx/_transports/asgi.py +++ b/httpx/_transports/asgi.py @@ -70,13 +70,13 @@ def __init__( self.root_path = root_path self.client = client - async def arequest( + async def handle_async_request( self, method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, stream: typing.AsyncIterator[bytes] = None, - ext: dict = None, + extensions: dict = None, ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict]: headers = [] if headers is None else headers @@ -163,6 +163,6 @@ async def send(message: dict) -> None: async def response_stream() -> typing.AsyncIterator[bytes]: yield b"".join(body_parts) - ext = {} + extensions = {} - return (status_code, response_headers, response_stream(), ext) + return (status_code, response_headers, response_stream(), extensions) diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py index adefad9243..ef76bfc314 100644 --- a/httpx/_transports/base.py +++ b/httpx/_transports/base.py @@ -17,18 +17,55 @@ def __exit__( ) -> None: self.close() - def request( + def handle_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], headers: typing.List[typing.Tuple[bytes, bytes]] = None, stream: typing.Iterator[bytes] = None, - ext: dict = None, + extensions: dict = None, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: + """ + Send a single HTTP request and return a response. + + At this layer of API we're simply using plain primitives. No `Request` or + `Response` models, no fancy `URL` or `Header` handling. This strict point + of cut-off provides a clear design seperation between the HTTPX API, + and the low-level network handling. + + method: The request method as bytes. Eg. b'GET'. + url: The components of the request URL, as a tuple of `(scheme, host, port, target)`. + The target will usually be the URL path, but also allows for alternative + formulations, such as proxy requests which include the complete URL in + the target portion of the HTTP request, or for "OPTIONS *" requests, which + cannot be expressed in a URL string. + headers: The request headers as a list of byte pairs. + stream: The request body as a bytes iterator. + extensions: An open ended dictionary, including optional extensions to the + core request/response API. Keys may include: + timeout: A dictionary of str:Optional[float] timeout values. + May include values for 'connect', 'read', 'write', or 'pool'. + + Returns a tuple of: + + status_code: The response status code as an integer. Should be in the range 1xx-5xx. + headers: The response headers as a list of byte pairs. + stream: The response body as a bytes iterator. + extensions: An open ended dictionary, including optional extensions to the + core request/response API. Keys are plain strings, and may include: + reason: The textual portion of the status code, as a string. Eg 'OK'. + HTTP/2 onwards does not include a reason phrase on the wire. + When no reason key is included, a default based on the status code + may be used. An empty-string reason phrase should not be substituted + for a default, as it indicates the server left the portion blank + eg. the leading response bytes were b"HTTP/1.1 200 ". + http_version: The HTTP version, as a string. Eg. "HTTP/1.1". + When no http_version key is included, "HTTP/1.1" may be assumed. + """ raise NotImplementedError( - "The 'request' method must be implemented." + "The 'handle_request' method must be implemented." ) # pragma: nocover def close(self) -> None: @@ -47,18 +84,18 @@ async def __aexit__( ) -> None: await self.aclose() - async def arequest( + async def handle_async_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], headers: typing.List[typing.Tuple[bytes, bytes]] = None, stream: typing.AsyncIterator[bytes] = None, - ext: dict = None, + extensions: dict = None, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict ]: raise NotImplementedError( - "The 'arequest' method must be implemented." + "The 'handle_async_request' method must be implemented." ) # pragma: nocover async def aclose(self) -> None: diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index daf08ed48c..f74ba5f9a8 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -35,8 +35,6 @@ T = typing.TypeVar("T", bound="HTTPTransport") A = typing.TypeVar("A", bound="AsyncHTTPTransport") -Headers = typing.List[typing.Tuple[bytes, bytes]] -URL = typing.Tuple[bytes, bytes, typing.Optional[int], bytes] class HTTPTransport(BaseTransport): @@ -92,15 +90,17 @@ def __exit__( ) -> None: self._pool.__exit__(exc_type, exc_value, traceback) - def request( + def handle_request( self, method: bytes, - url: URL, - headers: Headers = None, + url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], + headers: typing.List[typing.Tuple[bytes, bytes]] = None, stream: typing.Iterator[bytes] = None, - ext: dict = None, - ) -> typing.Tuple[int, Headers, typing.Iterator[bytes], dict]: - return self._pool.request(method, url, headers=headers, stream=stream, ext=ext) # type: ignore + extensions: dict = None, + ) -> typing.Tuple[ + int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict + ]: + return self._pool.request(method, url, headers=headers, stream=stream, ext=extensions) # type: ignore def close(self) -> None: self._pool.close() @@ -159,16 +159,18 @@ async def __aexit__( ) -> None: await self._pool.__aexit__(exc_type, exc_value, traceback) - async def arequest( + async def handle_async_request( self, method: bytes, - url: URL, - headers: Headers = None, + url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], + headers: typing.List[typing.Tuple[bytes, bytes]] = None, stream: typing.AsyncIterator[bytes] = None, - ext: dict = None, - ) -> typing.Tuple[int, Headers, typing.AsyncIterator[bytes], dict]: + extensions: dict = None, + ) -> typing.Tuple[ + int, typing.List[typing.Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict + ]: return await self._pool.arequest( # type: ignore - method, url, headers=headers, stream=stream, ext=ext # type: ignore + method, url, headers=headers, stream=stream, ext=extensions # type: ignore ) async def aclose(self) -> None: diff --git a/httpx/_transports/mock.py b/httpx/_transports/mock.py index 472bfc6dff..ee821af3c4 100644 --- a/httpx/_transports/mock.py +++ b/httpx/_transports/mock.py @@ -10,13 +10,13 @@ class MockTransport(AsyncBaseTransport, BaseTransport): def __init__(self, handler: Callable) -> None: self.handler = handler - def request( + def handle_request( self, method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, stream: typing.Iterator[bytes] = None, - ext: dict = None, + extensions: dict = None, ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.Iterator[bytes], dict]: request = Request( method=method, @@ -30,16 +30,16 @@ def request( response.status_code, response.headers.raw, response.stream, - response.ext, + response.extensions, ) - async def arequest( + async def handle_async_request( self, method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], headers: List[Tuple[bytes, bytes]] = None, stream: typing.AsyncIterator[bytes] = None, - ext: dict = None, + extensions: dict = None, ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict]: request = Request( method=method, @@ -63,5 +63,5 @@ async def arequest( response.status_code, response.headers.raw, response.stream, - response.ext, + response.extensions, ) diff --git a/httpx/_transports/wsgi.py b/httpx/_transports/wsgi.py index d4a5ddb316..b8eaa2c997 100644 --- a/httpx/_transports/wsgi.py +++ b/httpx/_transports/wsgi.py @@ -59,13 +59,13 @@ def __init__( self.script_name = script_name self.remote_addr = remote_addr - def request( + def handle_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], headers: typing.List[typing.Tuple[bytes, bytes]] = None, stream: typing.Iterator[bytes] = None, - ext: dict = None, + extensions: dict = None, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: @@ -126,6 +126,6 @@ def start_response( (key.encode("ascii"), value.encode("ascii")) for key, value in seen_response_headers ] - ext = {} + extensions = {} - return (status_code, headers, result, ext) + return (status_code, headers, result, extensions) diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 7b750b8cc2..13d503bf21 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -73,7 +73,7 @@ async def raise_exc_after_response(scope, receive, send): @pytest.mark.usefixtures("async_environment") async def test_asgi_transport(): async with httpx.ASGITransport(app=hello_world) as transport: - status_code, headers, stream, ext = await transport.arequest( + status_code, headers, stream, ext = await transport.handle_async_request( b"GET", (b"http", b"www.example.org", 80, b"/") ) body = b"".join([part async for part in stream]) @@ -85,7 +85,7 @@ async def test_asgi_transport(): @pytest.mark.usefixtures("async_environment") async def test_asgi_transport_no_body(): async with httpx.ASGITransport(app=echo_body) as transport: - status_code, headers, stream, ext = await transport.arequest( + status_code, headers, stream, ext = await transport.handle_async_request( b"GET", (b"http", b"www.example.org", 80, b"/") ) body = b"".join([part async for part in stream]) diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index 1e18e5960b..4b136dfcd1 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -42,7 +42,7 @@ def test_httpcore_exception_mapping(server) -> None: # Make sure it also works with custom transports. class MockTransport(httpx.BaseTransport): - def request(self, *args: Any, **kwargs: Any) -> Any: + def handle_request(self, *args: Any, **kwargs: Any) -> Any: raise httpcore.ProtocolError() client = httpx.Client(transport=MockTransport()) From ad36aef7cb5696e894f605444618ba03f0aa60ef Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 22 Mar 2021 10:44:13 +0000 Subject: [PATCH 5/9] Docs tweaks --- docs/advanced.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index 51773365e1..a70fa5b8e5 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -1041,13 +1041,13 @@ class HelloWorldTransport(httpx.BaseTransport): A mock transport that always returns a JSON "Hello, world!" response. """ - def handle_request(self, method, url, headers=None, stream=None, ext=None): + def handle_request(self, method, url, headers=None, stream=None, extensions=None): message = {"text": "Hello, world!"} content = json.dumps(message).encode("utf-8") stream = [content] headers = [(b"content-type", b"application/json")] - ext = {"http_version": "HTTP/1.1"} - return 200, headers, stream, ext + extensions = {"http_version": "HTTP/1.1"} + return 200, headers, stream, extensions ``` Which we can use in the same way: @@ -1099,7 +1099,7 @@ class HTTPSRedirectTransport(httpx.BaseTransport): A transport that always redirects to HTTPS. """ - def handle_request(self, method, url, headers=None, stream=None, ext=None): + def handle_request(self, method, url, headers=None, stream=None, extensions=None): scheme, host, port, path = url if port is None: location = b"https://%s%s" % (host, path) @@ -1107,7 +1107,7 @@ class HTTPSRedirectTransport(httpx.BaseTransport): location = b"https://%s:%d%s" % (host, port, path) stream = [b""] headers = [(b"location", location)] - ext = {"http_version": "HTTP/1.1"} + extensions = {"http_version": "HTTP/1.1"} return 303, headers, stream, ext From 9b01ff5ec5618eaee06542eadba61f28c6168188 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 22 Mar 2021 10:46:11 +0000 Subject: [PATCH 6/9] Docs tweaks --- docs/advanced.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced.md b/docs/advanced.md index a70fa5b8e5..0d5819dc4e 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -1046,7 +1046,7 @@ class HelloWorldTransport(httpx.BaseTransport): content = json.dumps(message).encode("utf-8") stream = [content] headers = [(b"content-type", b"application/json")] - extensions = {"http_version": "HTTP/1.1"} + extensions = {} return 200, headers, stream, extensions ``` @@ -1107,7 +1107,7 @@ class HTTPSRedirectTransport(httpx.BaseTransport): location = b"https://%s:%d%s" % (host, port, path) stream = [b""] headers = [(b"location", location)] - extensions = {"http_version": "HTTP/1.1"} + extensions = {} return 303, headers, stream, ext From 2c1c3aafdad9730bab017815d34024f500ee03a3 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 22 Mar 2021 11:02:37 +0000 Subject: [PATCH 7/9] Minor docstring tweak --- httpx/_transports/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py index ef76bfc314..a0498b8e5d 100644 --- a/httpx/_transports/base.py +++ b/httpx/_transports/base.py @@ -55,10 +55,10 @@ def handle_request( stream: The response body as a bytes iterator. extensions: An open ended dictionary, including optional extensions to the core request/response API. Keys are plain strings, and may include: - reason: The textual portion of the status code, as a string. Eg 'OK'. + reason: The reason-phrase of the HTTP response, as bytes. Eg 'OK'. HTTP/2 onwards does not include a reason phrase on the wire. - When no reason key is included, a default based on the status code - may be used. An empty-string reason phrase should not be substituted + When no key is included, a default based on the status code may + be used. An empty-string reason phrase should not be substituted for a default, as it indicates the server left the portion blank eg. the leading response bytes were b"HTTP/1.1 200 ". http_version: The HTTP version, as a string. Eg. "HTTP/1.1". From 1edc9c33db2b7ee1190d59633943a77b315d5381 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 22 Mar 2021 11:43:18 +0000 Subject: [PATCH 8/9] Transport API docs --- CHANGELOG.md | 13 +++++++++++-- httpx/_transports/base.py | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 992f4e4e38..17aa5aafd5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,14 +4,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## 0.17.1 +## Master + +The 0.18.x release series formalises our low-level Transport API, introducing the +base classes `httpx.BaseTransport` and `httpx.AsyncBaseTransport`. + +* Transport instances now inherit from `httpx.BaseTransport` or `httpx.AsyncBaseTransport`, + and should implement either the `handle_request` method or `handle_async_request` method. +* The `response.ext` property and `Response(ext=...)` argument are now named `extensions`. + +## 0.17.1 (March 15th, 2021) ### Fixed * Type annotation on `CertTypes` allows `keyfile` and `password` to be optional. (Pull #1503) * Fix httpcore pinned version. (Pull #1495) -## 0.17.0 +## 0.17.0 (Februray 28th, 2021) ### Added diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py index a0498b8e5d..658106c853 100644 --- a/httpx/_transports/base.py +++ b/httpx/_transports/base.py @@ -35,6 +35,29 @@ def handle_request( of cut-off provides a clear design seperation between the HTTPX API, and the low-level network handling. + Developers shouldn't typically ever need to call into this API directly, + since the Client class provides all the higher level user-facing API + niceties. + + Example usage: + + with httpx.HTTPTransport() as transport: + status_code, headers, stream, extensions = transport.handle_request( + method=b'GET', + url=(b'https', b'www.example.com', 443, b'/'), + headers=[(b'Host', b'www.example.com')], + stream=[], + extensions={} + ) + try: + body = b''.join([part for part in stream]) + finally: + if hasattr(stream 'close'): + stream.close() + print(status_code, headers, body) + + Arguments: + method: The request method as bytes. Eg. b'GET'. url: The components of the request URL, as a tuple of `(scheme, host, port, target)`. The target will usually be the URL path, but also allows for alternative From 0a6220a71af84e5e8b418054e10a48a491b4b207 Mon Sep 17 00:00:00 2001 From: Tom Christie Date: Mon, 22 Mar 2021 11:49:56 +0000 Subject: [PATCH 9/9] Drop 'Optional' on Transport API --- httpx/_transports/asgi.py | 14 ++++---------- httpx/_transports/base.py | 12 ++++++------ httpx/_transports/default.py | 12 ++++++------ httpx/_transports/mock.py | 12 ++++++------ httpx/_transports/wsgi.py | 9 ++++----- tests/test_asgi.py | 16 ++++++++++++++-- 6 files changed, 40 insertions(+), 35 deletions(-) diff --git a/httpx/_transports/asgi.py b/httpx/_transports/asgi.py index 23691502e5..a030a7f078 100644 --- a/httpx/_transports/asgi.py +++ b/httpx/_transports/asgi.py @@ -74,12 +74,10 @@ async def handle_async_request( self, method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], - headers: List[Tuple[bytes, bytes]] = None, - stream: typing.AsyncIterator[bytes] = None, - extensions: dict = None, + headers: List[Tuple[bytes, bytes]], + stream: typing.AsyncIterator[bytes], + extensions: dict, ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict]: - headers = [] if headers is None else headers - # ASGI scope. scheme, host, port, full_path = url path, _, query = full_path.partition(b"?") @@ -99,7 +97,7 @@ async def handle_async_request( } # Request. - request_body_chunks = None if stream is None else stream.__aiter__() + request_body_chunks = stream.__aiter__() request_complete = False # Response. @@ -118,10 +116,6 @@ async def receive() -> dict: await response_complete.wait() return {"type": "http.disconnect"} - if request_body_chunks is None: - request_complete = True - return {"type": "http.request", "body": b"", "more_body": False} - try: body = await request_body_chunks.__anext__() except StopAsyncIteration: diff --git a/httpx/_transports/base.py b/httpx/_transports/base.py index 658106c853..bea96d014b 100644 --- a/httpx/_transports/base.py +++ b/httpx/_transports/base.py @@ -21,9 +21,9 @@ def handle_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], - headers: typing.List[typing.Tuple[bytes, bytes]] = None, - stream: typing.Iterator[bytes] = None, - extensions: dict = None, + headers: typing.List[typing.Tuple[bytes, bytes]], + stream: typing.Iterator[bytes], + extensions: dict, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: @@ -111,9 +111,9 @@ async def handle_async_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], - headers: typing.List[typing.Tuple[bytes, bytes]] = None, - stream: typing.AsyncIterator[bytes] = None, - extensions: dict = None, + headers: typing.List[typing.Tuple[bytes, bytes]], + stream: typing.AsyncIterator[bytes], + extensions: dict, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict ]: diff --git a/httpx/_transports/default.py b/httpx/_transports/default.py index f74ba5f9a8..f687269585 100644 --- a/httpx/_transports/default.py +++ b/httpx/_transports/default.py @@ -94,9 +94,9 @@ def handle_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], - headers: typing.List[typing.Tuple[bytes, bytes]] = None, - stream: typing.Iterator[bytes] = None, - extensions: dict = None, + headers: typing.List[typing.Tuple[bytes, bytes]], + stream: typing.Iterator[bytes], + extensions: dict, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: @@ -163,9 +163,9 @@ async def handle_async_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], - headers: typing.List[typing.Tuple[bytes, bytes]] = None, - stream: typing.AsyncIterator[bytes] = None, - extensions: dict = None, + headers: typing.List[typing.Tuple[bytes, bytes]], + stream: typing.AsyncIterator[bytes], + extensions: dict, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict ]: diff --git a/httpx/_transports/mock.py b/httpx/_transports/mock.py index ee821af3c4..fec127c123 100644 --- a/httpx/_transports/mock.py +++ b/httpx/_transports/mock.py @@ -14,9 +14,9 @@ def handle_request( self, method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], - headers: List[Tuple[bytes, bytes]] = None, - stream: typing.Iterator[bytes] = None, - extensions: dict = None, + headers: List[Tuple[bytes, bytes]], + stream: typing.Iterator[bytes], + extensions: dict, ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.Iterator[bytes], dict]: request = Request( method=method, @@ -37,9 +37,9 @@ async def handle_async_request( self, method: bytes, url: Tuple[bytes, bytes, Optional[int], bytes], - headers: List[Tuple[bytes, bytes]] = None, - stream: typing.AsyncIterator[bytes] = None, - extensions: dict = None, + headers: List[Tuple[bytes, bytes]], + stream: typing.AsyncIterator[bytes], + extensions: dict, ) -> Tuple[int, List[Tuple[bytes, bytes]], typing.AsyncIterator[bytes], dict]: request = Request( method=method, diff --git a/httpx/_transports/wsgi.py b/httpx/_transports/wsgi.py index b8eaa2c997..e773c8c6d6 100644 --- a/httpx/_transports/wsgi.py +++ b/httpx/_transports/wsgi.py @@ -63,14 +63,13 @@ def handle_request( self, method: bytes, url: typing.Tuple[bytes, bytes, typing.Optional[int], bytes], - headers: typing.List[typing.Tuple[bytes, bytes]] = None, - stream: typing.Iterator[bytes] = None, - extensions: dict = None, + headers: typing.List[typing.Tuple[bytes, bytes]], + stream: typing.Iterator[bytes], + extensions: dict, ) -> typing.Tuple[ int, typing.List[typing.Tuple[bytes, bytes]], typing.Iterator[bytes], dict ]: - headers = [] if headers is None else headers - wsgi_input = io.BytesIO() if stream is None else io.BytesIO(b"".join(stream)) + wsgi_input = io.BytesIO(b"".join(stream)) scheme, host, port, full_path = url path, _, query = full_path.partition(b"?") diff --git a/tests/test_asgi.py b/tests/test_asgi.py index 13d503bf21..d7cf9412af 100644 --- a/tests/test_asgi.py +++ b/tests/test_asgi.py @@ -70,11 +70,19 @@ async def raise_exc_after_response(scope, receive, send): raise RuntimeError() +async def empty_stream(): + yield b"" + + @pytest.mark.usefixtures("async_environment") async def test_asgi_transport(): async with httpx.ASGITransport(app=hello_world) as transport: status_code, headers, stream, ext = await transport.handle_async_request( - b"GET", (b"http", b"www.example.org", 80, b"/") + method=b"GET", + url=(b"http", b"www.example.org", 80, b"/"), + headers=[(b"Host", b"www.example.org")], + stream=empty_stream(), + extensions={}, ) body = b"".join([part async for part in stream]) @@ -86,7 +94,11 @@ async def test_asgi_transport(): async def test_asgi_transport_no_body(): async with httpx.ASGITransport(app=echo_body) as transport: status_code, headers, stream, ext = await transport.handle_async_request( - b"GET", (b"http", b"www.example.org", 80, b"/") + method=b"GET", + url=(b"http", b"www.example.org", 80, b"/"), + headers=[(b"Host", b"www.example.org")], + stream=empty_stream(), + extensions={}, ) body = b"".join([part async for part in stream])