From 573ae3fb4fce4524ddae56bd7fb21fb032b15e4e Mon Sep 17 00:00:00 2001 From: Teg Khanna Date: Wed, 1 Apr 2020 02:07:55 +0530 Subject: [PATCH 1/6] single input multiple files enabled --- httpx/_content_streams.py | 8 +++++-- tests/test_content_streams.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/httpx/_content_streams.py b/httpx/_content_streams.py index e6a3881f4e..2ddeead79b 100644 --- a/httpx/_content_streams.py +++ b/httpx/_content_streams.py @@ -276,7 +276,7 @@ def __init__(self, data: dict, files: dict, boundary: bytes = None) -> None: body = BytesIO() if boundary is None: boundary = binascii.hexlify(os.urandom(16)) - + for field in self.iter_fields(data, files): body.write(b"--%s\r\n" % boundary) body.write(field.render_headers()) @@ -301,7 +301,11 @@ def iter_fields( yield self.DataField(name=name, value=value) for name, value in files.items(): - yield self.FileField(name=name, value=value) + if isinstance(value, list): + for item in value: + yield self.FileField(name=name, value=item) + else: + yield self.FileField(name=name, value=value) def get_headers(self) -> typing.Dict[str, str]: content_length = str(len(self.body)) diff --git a/tests/test_content_streams.py b/tests/test_content_streams.py index 84999efc2a..5c33506403 100644 --- a/tests/test_content_streams.py +++ b/tests/test_content_streams.py @@ -148,6 +148,50 @@ async def test_multipart_files_content(): ) +@pytest.mark.asyncio +async def test_multipart_multiple_files_single_input_content(): + files = {"file": [io.BytesIO(b""), io.BytesIO(b"")]} + stream = encode(files=files, boundary=b"+++") + sync_content = b"".join([part for part in stream]) + async_content = b"".join([part async for part in stream]) + + assert stream.can_replay() + assert stream.get_headers() == { + "Content-Length": "271", + "Content-Type": "multipart/form-data; boundary=+++", + } + assert sync_content == b"".join( + [ + b"--+++\r\n", + b'Content-Disposition: form-data; name="file"; filename="upload"\r\n', + b"Content-Type: application/octet-stream\r\n", + b"\r\n", + b"\r\n", + b"--+++\r\n", + b'Content-Disposition: form-data; name="file"; filename="upload"\r\n', + b"Content-Type: application/octet-stream\r\n", + b"\r\n", + b"\r\n", + b"--+++--\r\n", + ] + ) + assert async_content == b"".join( + [ + b"--+++\r\n", + b'Content-Disposition: form-data; name="file"; filename="upload"\r\n', + b"Content-Type: application/octet-stream\r\n", + b"\r\n", + b"\r\n", + b"--+++\r\n", + b'Content-Disposition: form-data; name="file"; filename="upload"\r\n', + b"Content-Type: application/octet-stream\r\n", + b"\r\n", + b"\r\n", + b"--+++--\r\n", + ] + ) + + @pytest.mark.asyncio async def test_multipart_data_and_files_content(): data = {"message": "Hello, world!"} From ed875d30373cce33866fe355420a5afa4c19330d Mon Sep 17 00:00:00 2001 From: Teg Khanna Date: Wed, 8 Apr 2020 18:07:06 +0530 Subject: [PATCH 2/6] multiples files as list of tuples instead of dict --- httpx/_content_streams.py | 64 +++++++++++++++++++++++------------ tests/test_content_streams.py | 5 ++- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/httpx/_content_streams.py b/httpx/_content_streams.py index 2ddeead79b..5b2070213f 100644 --- a/httpx/_content_streams.py +++ b/httpx/_content_streams.py @@ -15,21 +15,44 @@ dict, str, bytes, typing.Iterator[bytes], typing.AsyncIterator[bytes] ] -RequestFiles = typing.Dict[ - str, - typing.Union[ - # file (or str) - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - # (filename, file (or str)) - typing.Tuple[ - typing.Optional[str], +RequestFiles = typing.Union[ + typing.Dict[ + str, + typing.Union[ + # file (or str) typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + # (filename, file (or str)) + typing.Tuple[ + typing.Optional[str], + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + ], + # (filename, file (or str), content_type) + typing.Tuple[ + typing.Optional[str], + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + typing.Optional[str], + ], ], - # (filename, file (or str), content_type) + ], + typing.List[ + # (input_name, file_details) typing.Tuple[ - typing.Optional[str], - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - typing.Optional[str], + str, + typing.Union[ + # file (or str) + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + # (filename, file (or str)) + typing.Tuple[ + typing.Optional[str], + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + ], + # (filename, file (or str), content_type) + typing.Tuple[ + typing.Optional[str], + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + typing.Optional[str], + ], + ], ], ], ] @@ -272,11 +295,13 @@ def render_data(self) -> bytes: content = self.file.read() return content.encode("utf-8") if isinstance(content, str) else content - def __init__(self, data: dict, files: dict, boundary: bytes = None) -> None: + def __init__( + self, data: dict, files: typing.Union[dict, list], boundary: bytes = None + ) -> None: body = BytesIO() if boundary is None: boundary = binascii.hexlify(os.urandom(16)) - + for field in self.iter_fields(data, files): body.write(b"--%s\r\n" % boundary) body.write(field.render_headers()) @@ -291,7 +316,7 @@ def __init__(self, data: dict, files: dict, boundary: bytes = None) -> None: self.body = body.getvalue() def iter_fields( - self, data: dict, files: dict + self, data: dict, files: typing.Union[dict, list] ) -> typing.Iterator[typing.Union["FileField", "DataField"]]: for name, value in data.items(): if isinstance(value, list): @@ -300,12 +325,9 @@ def iter_fields( else: yield self.DataField(name=name, value=value) - for name, value in files.items(): - if isinstance(value, list): - for item in value: - yield self.FileField(name=name, value=item) - else: - yield self.FileField(name=name, value=value) + file_items = files.items() if isinstance(files, dict) else files + for name, value in file_items: + yield self.FileField(name=name, value=value) def get_headers(self) -> typing.Dict[str, str]: content_length = str(len(self.body)) diff --git a/tests/test_content_streams.py b/tests/test_content_streams.py index 5c33506403..6e512f84d2 100644 --- a/tests/test_content_streams.py +++ b/tests/test_content_streams.py @@ -150,7 +150,10 @@ async def test_multipart_files_content(): @pytest.mark.asyncio async def test_multipart_multiple_files_single_input_content(): - files = {"file": [io.BytesIO(b""), io.BytesIO(b"")]} + files = [ + ("file", io.BytesIO(b"")), + ("file", io.BytesIO(b"")), + ] stream = encode(files=files, boundary=b"+++") sync_content = b"".join([part for part in stream]) async_content = b"".join([part async for part in stream]) From 465ada71ec721addffd546cbcaa559005ab6115e Mon Sep 17 00:00:00 2001 From: Teg Khanna Date: Wed, 8 Apr 2020 23:41:40 +0530 Subject: [PATCH 3/6] File type made reusable --- httpx/_content_streams.py | 54 ++++++++++++--------------------------- 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/httpx/_content_streams.py b/httpx/_content_streams.py index 5b2070213f..0c18cdbc59 100644 --- a/httpx/_content_streams.py +++ b/httpx/_content_streams.py @@ -15,48 +15,26 @@ dict, str, bytes, typing.Iterator[bytes], typing.AsyncIterator[bytes] ] -RequestFiles = typing.Union[ - typing.Dict[ - str, - typing.Union[ - # file (or str) - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - # (filename, file (or str)) - typing.Tuple[ - typing.Optional[str], - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - ], - # (filename, file (or str), content_type) - typing.Tuple[ - typing.Optional[str], - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - typing.Optional[str], - ], - ], +FileEntry = typing.Union[ + # file (or str) + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + # (filename, file (or str)) + typing.Tuple[ + typing.Optional[str], + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], ], - typing.List[ - # (input_name, file_details) - typing.Tuple[ - str, - typing.Union[ - # file (or str) - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - # (filename, file (or str)) - typing.Tuple[ - typing.Optional[str], - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - ], - # (filename, file (or str), content_type) - typing.Tuple[ - typing.Optional[str], - typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], - typing.Optional[str], - ], - ], - ], + # (filename, file (or str), content_type) + typing.Tuple[ + typing.Optional[str], + typing.Union[typing.IO[str], typing.IO[bytes], StrOrBytes], + typing.Optional[str], ], ] +RequestFiles = typing.Union[ + typing.Dict[str, FileEntry], typing.List[typing.Tuple[str, FileEntry]] +] + class ContentStream: def get_headers(self) -> typing.Dict[str, str]: From 6cc7c3aa17161295d073c4229c641baeb3950855 Mon Sep 17 00:00:00 2001 From: Teg Khanna Date: Thu, 9 Apr 2020 16:02:06 +0530 Subject: [PATCH 4/6] data now accepts a list of tuple as well --- httpx/_content_streams.py | 16 ++++++++++------ tests/test_content_streams.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/httpx/_content_streams.py b/httpx/_content_streams.py index 0c18cdbc59..4685f75d10 100644 --- a/httpx/_content_streams.py +++ b/httpx/_content_streams.py @@ -12,7 +12,7 @@ from ._utils import format_form_param RequestData = typing.Union[ - dict, str, bytes, typing.Iterator[bytes], typing.AsyncIterator[bytes] + dict, list, str, bytes, typing.Iterator[bytes], typing.AsyncIterator[bytes] ] FileEntry = typing.Union[ @@ -175,7 +175,7 @@ class URLEncodedStream(ContentStream): Request content as URL encoded form data. """ - def __init__(self, data: dict) -> None: + def __init__(self, data: typing.Union[dict, list]) -> None: self.body = urlencode(data, doseq=True).encode("utf-8") def get_headers(self) -> typing.Dict[str, str]: @@ -274,7 +274,10 @@ def render_data(self) -> bytes: return content.encode("utf-8") if isinstance(content, str) else content def __init__( - self, data: dict, files: typing.Union[dict, list], boundary: bytes = None + self, + data: typing.Union[dict, list], + files: typing.Union[dict, list], + boundary: bytes = None, ) -> None: body = BytesIO() if boundary is None: @@ -294,9 +297,10 @@ def __init__( self.body = body.getvalue() def iter_fields( - self, data: dict, files: typing.Union[dict, list] + self, data: typing.Union[dict, list], files: typing.Union[dict, list] ) -> typing.Iterator[typing.Union["FileField", "DataField"]]: - for name, value in data.items(): + data_items = data.items() if isinstance(data, dict) else data + for name, value in data_items: if isinstance(value, list): for item in value: yield self.DataField(name=name, value=item) @@ -336,7 +340,7 @@ def encode( return MultipartStream(data={}, files=files, boundary=boundary) else: return ByteStream(body=b"") - elif isinstance(data, dict): + elif isinstance(data, (dict, list)): if files: return MultipartStream(data=data, files=files, boundary=boundary) else: diff --git a/tests/test_content_streams.py b/tests/test_content_streams.py index 6e512f84d2..56b4368184 100644 --- a/tests/test_content_streams.py +++ b/tests/test_content_streams.py @@ -114,6 +114,21 @@ async def test_urlencoded_content(): assert async_content == b"Hello=world%21" +@pytest.mark.asyncio +async def test_urlencoded_list_content(): + stream = encode(data=[("Hello", "world1!"), ("Hello", "world2!")]) + sync_content = b"".join([part for part in stream]) + async_content = b"".join([part async for part in stream]) + + assert stream.can_replay() + assert stream.get_headers() == { + "Content-Length": "31", + "Content-Type": "application/x-www-form-urlencoded", + } + assert sync_content == b"Hello=world1%21&Hello=world2%21" + assert async_content == b"Hello=world1%21&Hello=world2%21" + + @pytest.mark.asyncio async def test_multipart_files_content(): files = {"file": io.BytesIO(b"")} From 6b79b12d14fa3fd03d67d0e3b277a81de4d56489 Mon Sep 17 00:00:00 2001 From: Teg Khanna Date: Sat, 11 Apr 2020 15:46:34 +0530 Subject: [PATCH 5/6] resolved conflicts due to #857 merge --- httpx/_content_streams.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/httpx/_content_streams.py b/httpx/_content_streams.py index 88264a1cbf..1bd64bed74 100644 --- a/httpx/_content_streams.py +++ b/httpx/_content_streams.py @@ -328,7 +328,10 @@ def render(self) -> typing.Iterator[bytes]: yield from self.render_data() def __init__( - self, data: typing.Mapping, files: typing.Mapping, boundary: bytes = None + self, + data: typing.Union[typing.Mapping, list], + files: typing.Union[typing.Mapping, list], + boundary: bytes = None, ) -> None: if boundary is None: boundary = binascii.hexlify(os.urandom(16)) @@ -340,7 +343,9 @@ def __init__( self.fields = list(self._iter_fields(data, files)) def _iter_fields( - self, data: typing.Mapping, files: typing.Mapping + self, + data: typing.Union[typing.Mapping, list], + files: typing.Union[typing.Mapping, list], ) -> typing.Iterator[typing.Union["FileField", "DataField"]]: data_items = data.items() if isinstance(data, dict) else data for name, value in data_items: From 87fd4c49298f2b5d6e5f4c8b1dc5a828dbc55a34 Mon Sep 17 00:00:00 2001 From: Teg Khanna Date: Sat, 11 Apr 2020 15:54:05 +0530 Subject: [PATCH 6/6] untracked settings.json --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 95af2ea266..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "/home/tegkhanna/.virtualenvs/httpx/bin/python" -} \ No newline at end of file