From 295f2f99210fc04890e3f9516710a7f71a0c3bbf Mon Sep 17 00:00:00 2001 From: Lanqing Huang Date: Wed, 19 Apr 2023 19:23:23 +0800 Subject: [PATCH 1/5] Implement `_structure_deque` and related tests, docs --- HISTORY.md | 2 ++ docs/structuring.md | 23 +++++++++++++ docs/unstructuring.md | 4 +-- src/cattrs/_compat.py | 18 +++++++--- src/cattrs/converters.py | 38 +++++++++++++++++++-- tests/test_converter.py | 62 +++++++++++++++++++++++++++++++++++ tests/test_generics.py | 15 ++++++++- tests/test_structure_attrs.py | 18 +++++++++- 8 files changed, 170 insertions(+), 10 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 294f12c8..6927bbf5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -18,6 +18,8 @@ ([#350](https://github.com/python-attrs/cattrs/issues/350) [#353](https://github.com/python-attrs/cattrs/pull/353)) - Subclasses structuring and unstructuring is now supported via a custom `include_subclasses` strategy. ([#312](https://github.com/python-attrs/cattrs/pull/312)) +- Add unstructuring and structuring support to `deque` in standard lib. + ([#355](https://github.com/python-attrs/cattrs/issues/355)) ## 22.2.0 (2022-10-03) diff --git a/docs/structuring.md b/docs/structuring.md index 5d9254c7..65ad578e 100644 --- a/docs/structuring.md +++ b/docs/structuring.md @@ -146,6 +146,29 @@ These generic types are composable with all other converters. ['1', None, '3'] ``` +### Deques + +Deques can be produced from any iterable object. Types converting +to deques are: + +- `Deque[T]` +- `deque[T]` + +In all cases, a new unbounded deque (`maxlen=None`) will be returned, so this operation can be used to +copy an iterable into a deque. + +```{doctest} +>>> cattrs.structure((1, 2, 3), deque[int]) +[1, 2, 3] +``` + +These generic types are composable with all other converters. + +```{doctest} +>>> cattrs.structure((1, None, 3), deque[Optional[str]]) +['1', None, '3'] +``` + ### Sets and Frozensets Sets and frozensets can be produced from any iterable object. Types converting diff --git a/docs/unstructuring.md b/docs/unstructuring.md index 474cc82a..99e1f6ce 100644 --- a/docs/unstructuring.md +++ b/docs/unstructuring.md @@ -80,9 +80,9 @@ unstructure all sets into lists, try the following: Going even further, the Converter contains heuristics to support the following Python types, in order of decreasing generality: -- `Sequence`, `MutableSequence`, `list`, `tuple` +- `Sequence`, `MutableSequence`, `list`, `deque`, `tuple` - `Set`, `frozenset`, `MutableSet`, `set` -- `Mapping`, `MutableMapping`, `dict`, `Counter` +- `Mapping`, `MutableMapping`, `dict`, `defaultdict`, `OrderedDict`, `Counter` For example, if you override the unstructure type for `Sequence`, but not for `MutableSequence`, `list` or `tuple`, the override will also affect those diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index 18106eff..3c3d19c9 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -1,12 +1,13 @@ import builtins import sys +from collections import deque from collections.abc import MutableSet as AbcMutableSet from collections.abc import Set as AbcSet from dataclasses import MISSING from dataclasses import fields as dataclass_fields from dataclasses import is_dataclass from typing import AbstractSet as TypingAbstractSet -from typing import Any, Dict, FrozenSet, List +from typing import Any, Deque, Dict, FrozenSet, List from typing import Mapping as TypingMapping from typing import MutableMapping as TypingMutableMapping from typing import MutableSequence as TypingMutableSequence @@ -327,8 +328,10 @@ def is_sequence(type: Any) -> bool: TypingSequence, TypingMutableSequence, AbcMutableSequence, - Tuple, tuple, + Tuple, + deque, + Deque, ) or ( type.__class__ is _GenericAlias @@ -339,10 +342,17 @@ def is_sequence(type: Any) -> bool: and type.__args__[1] is ... ) ) - or (origin in (list, AbcMutableSequence, AbcSequence)) + or (origin in (list, deque, AbcMutableSequence, AbcSequence)) or (origin is tuple and type.__args__[1] is ...) ) + def is_deque(type): + return ( + type in (deque, Deque) + or (type.__class__ is _GenericAlias and issubclass(type.__origin__, Deque)) + or (getattr(type, "__origin__", None) is deque) + ) + def is_mutable_set(type): return ( type in (TypingSet, TypingMutableSet, set) @@ -370,7 +380,7 @@ def is_bare(type): def is_mapping(type): return ( - type in (TypingMapping, Dict, TypingMutableMapping, dict, AbcMutableMapping) + type in (dict, Dict, TypingMapping, TypingMutableMapping, AbcMutableMapping) or ( type.__class__ is _GenericAlias and issubclass(type.__origin__, TypingMapping) diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 64016c73..e770b406 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -1,4 +1,4 @@ -from collections import Counter +from collections import Counter, deque from collections.abc import MutableSet as AbcMutableSet from dataclasses import Field from enum import Enum @@ -7,6 +7,7 @@ from typing import ( Any, Callable, + Deque, Dict, Iterable, List, @@ -46,6 +47,7 @@ is_annotated, is_bare, is_counter, + is_deque, is_frozenset, is_generic, is_generic_attrs, @@ -193,6 +195,7 @@ def __init__( (is_literal, self._structure_simple_literal), (is_literal_containing_enums, self._structure_enum_literal), (is_sequence, self._structure_list), + (is_deque, self._structure_deque), (is_mutable_set, self._structure_set), (is_frozenset, self._structure_frozenset), (is_tuple, self._structure_tuple), @@ -326,7 +329,6 @@ def register_structure_hook_factory( def structure(self, obj: Any, cl: Type[T]) -> T: """Convert unstructured Python data structures to structured data.""" - return self._structure_func.dispatch(cl)(obj, cl) # Classes to Python primitives. @@ -545,6 +547,36 @@ def _structure_list(self, obj: Iterable[T], cl: Any) -> List[T]: res = [handler(e, elem_type) for e in obj] return res + def _structure_deque(self, obj: Iterable[T], cl: Any) -> Deque[T]: + """Convert an iterable to a potentially generic deque.""" + if is_bare(cl) or cl.__args__[0] is Any: + res = deque(e for e in obj) + else: + elem_type = cl.__args__[0] + handler = self._structure_func.dispatch(elem_type) + if self.detailed_validation: + errors = [] + res = deque() + ix = 0 # Avoid `enumerate` for performance. + for e in obj: + try: + res.append(handler(e, elem_type)) + except Exception as e: + msg = IterableValidationNote( + f"Structuring {cl} @ index {ix}", ix, elem_type + ) + e.__notes__ = getattr(e, "__notes__", []) + [msg] + errors.append(e) + finally: + ix += 1 + if errors: + raise IterableValidationError( + f"While structuring {cl!r}", errors, cl + ) + else: + res = deque(handler(e, elem_type) for e in obj) + return res + def _structure_set( self, obj: Iterable[T], cl: Any, structure_to: type = set ) -> Set[T]: @@ -823,6 +855,8 @@ def __init__( if MutableSequence in co: if list not in co: co[list] = co[MutableSequence] + if deque not in co: + co[deque] = co[MutableSequence] # abc.Mapping overrides, if defined, can apply to MutableMappings if Mapping in co: diff --git a/tests/test_converter.py b/tests/test_converter.py index fc441017..017cb42b 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -1,5 +1,7 @@ """Test both structuring and unstructuring.""" +from collections import deque from typing import ( + Deque, FrozenSet, List, MutableSequence, @@ -524,20 +526,24 @@ class Outer: (tuple, tuple), (list, list), (list, List), + (deque, Deque), (set, Set), (set, set), (frozenset, frozenset), (frozenset, FrozenSet), (list, MutableSequence), + (deque, MutableSequence), (tuple, Sequence), ] if is_py39_plus else [ (tuple, Tuple), (list, List), + (deque, Deque), (set, Set), (frozenset, FrozenSet), (list, MutableSequence), + (deque, MutableSequence), (tuple, Sequence), ] ), @@ -563,6 +569,62 @@ def test_seq_of_simple_classes_unstructure(cls_and_vals, seq_type_and_annotation assert all(e == test_val for e in outputs) +@given( + sampled_from( + [ + (tuple, Tuple), + (tuple, tuple), + (list, list), + (list, List), + (deque, deque), + (deque, Deque), + (set, Set), + (set, set), + (frozenset, frozenset), + (frozenset, FrozenSet), + ] + if is_py39_plus + else [ + (tuple, Tuple), + (list, List), + (deque, Deque), + (set, Set), + (frozenset, FrozenSet), + ] + ) +) +def test_seq_of_bare_classes_structure(seq_type_and_annotation): + """Structure iterable of values to a sequence of primitives.""" + converter = Converter() + + cls_and_vals = ( + (int, (1,), {}), + (float, (1.0,), {}), + (str, ("test",), {}), + (bool, (True,), {}), + ) + + for cl, vals, kwargs in cls_and_vals: + + @define(frozen=True) + class C: + a: cl + + seq_type, annotation = seq_type_and_annotation + + inputs = [{"a": cl(*vals, **kwargs)} for _ in range(20)] + outputs = converter.structure( + inputs, + cl=annotation[C] + if annotation not in (Tuple, tuple) + else annotation[C, ...], + ) + expected = seq_type(C(a=cl(*vals, **kwargs)) for _ in range(20)) + + assert type(outputs) == seq_type + assert outputs == expected + + @pytest.mark.skipif(not is_py39_plus, reason="3.9+ only") def test_annotated_attrs(): """Annotation support works for attrs classes.""" diff --git a/tests/test_generics.py b/tests/test_generics.py index a7801b6a..c9c3d252 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -1,4 +1,5 @@ -from typing import Dict, Generic, List, Optional, TypeVar, Union +from collections import deque +from typing import Deque, Dict, Generic, List, Optional, TypeVar, Union import pytest from attr import asdict, attrs, define @@ -145,6 +146,18 @@ class TClass2(Generic[T]): assert res == data +def test_structure_deque_of_generic_unions(converter): + @attrs(auto_attribs=True) + class TClass2(Generic[T]): + c: T + + data = deque((TClass2(c="string"), TClass(1, 2))) + res = converter.structure( + [asdict(x) for x in data], Deque[Union[TClass[int, int], TClass2[str]]] + ) + assert res == data + + def test_raises_if_no_generic_params_supplied( converter: Union[Converter, BaseConverter] ): diff --git a/tests/test_structure_attrs.py b/tests/test_structure_attrs.py index 2a08c662..65686113 100644 --- a/tests/test_structure_attrs.py +++ b/tests/test_structure_attrs.py @@ -1,7 +1,9 @@ """Loading of attrs classes.""" +from collections import deque from enum import Enum from ipaddress import IPv4Address, IPv6Address, ip_address -from typing import Union +from random import randint +from typing import Deque, Union from unittest.mock import Mock import pytest @@ -80,6 +82,20 @@ def test_structure_tuple(cl_and_vals): assert obj == loaded +@given(simple_classes(kw_only=False)) +def test_structure_deque(cl_and_vals): + """Structuring of deque works.""" + converter = BaseConverter() + cl, vals, kwargs = cl_and_vals + converter.register_structure_hook(cl, converter._structure_deque) + obj = deque(cl(*vals, **kwargs) for _ in range(randint(1, 5))) + + dumped = astuple(obj) + loaded = converter.structure(dumped, Deque[cl]) + + assert obj == loaded + + @given(simple_classes(defaults=False), simple_classes(defaults=False)) def test_structure_union(cl_and_vals_a, cl_and_vals_b): """Structuring of automatically-disambiguable unions works.""" From 24537b3993c732948263c2337b2cefff8245c178 Mon Sep 17 00:00:00 2001 From: Lanqing Huang Date: Wed, 19 Apr 2023 22:09:23 +0800 Subject: [PATCH 2/5] Adjust location of `test_seqs_deque` --- tests/test_structure.py | 11 +++++++++++ tests/test_structure_attrs.py | 18 +----------------- tests/untyped.py | 15 ++++++++++++++- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/tests/test_structure.py b/tests/test_structure.py index 9d10f78f..9f460af8 100644 --- a/tests/test_structure.py +++ b/tests/test_structure.py @@ -30,6 +30,7 @@ lists_of_primitives, primitive_strategies, seqs_of_primitives, + deque_seqs_of_primitives, ) NoneType = type(None) @@ -85,6 +86,16 @@ def test_structuring_seqs(seq_and_type): assert x == y +@given(deque_seqs_of_primitives) +def test_structuring_seqs_to_deque(seq_and_type): + """Test structuring sequence generic types.""" + converter = BaseConverter() + iterable, t = seq_and_type + converted = converter.structure(iterable, t) + for x, y in zip(iterable, converted): + assert x == y + + @given(sets_of_primitives, set_types) def test_structuring_sets(set_and_type, set_type): """Test structuring generic sets.""" diff --git a/tests/test_structure_attrs.py b/tests/test_structure_attrs.py index 65686113..2a08c662 100644 --- a/tests/test_structure_attrs.py +++ b/tests/test_structure_attrs.py @@ -1,9 +1,7 @@ """Loading of attrs classes.""" -from collections import deque from enum import Enum from ipaddress import IPv4Address, IPv6Address, ip_address -from random import randint -from typing import Deque, Union +from typing import Union from unittest.mock import Mock import pytest @@ -82,20 +80,6 @@ def test_structure_tuple(cl_and_vals): assert obj == loaded -@given(simple_classes(kw_only=False)) -def test_structure_deque(cl_and_vals): - """Structuring of deque works.""" - converter = BaseConverter() - cl, vals, kwargs = cl_and_vals - converter.register_structure_hook(cl, converter._structure_deque) - obj = deque(cl(*vals, **kwargs) for _ in range(randint(1, 5))) - - dumped = astuple(obj) - loaded = converter.structure(dumped, Deque[cl]) - - assert obj == loaded - - @given(simple_classes(defaults=False), simple_classes(defaults=False)) def test_structure_union(cl_and_vals_a, cl_and_vals_b): """Structuring of automatically-disambiguable unions works.""" diff --git a/tests/untyped.py b/tests/untyped.py index 98ed7d85..aab308f5 100644 --- a/tests/untyped.py +++ b/tests/untyped.py @@ -5,6 +5,7 @@ from enum import Enum from typing import ( Any, + Deque, Dict, List, Mapping, @@ -57,6 +58,7 @@ def enums_of_primitives(draw): list_types = st.sampled_from([List, Sequence, MutableSequence]) +deque_types = st.sampled_from([Deque, Sequence, MutableSequence]) set_types = st.sampled_from([Set, MutableSet]) @@ -71,6 +73,17 @@ def lists_of_primitives(draw): return draw(st.lists(prim_strat)), list_t +@st.composite +def deques_of_primitives(draw): + """Generate a strategy that yields tuples of list of primitives and types. + + For example, a sample value might be ([1,2], List[int]). + """ + prim_strat, t = draw(primitive_strategies) + deque_t = draw(deque_types.map(lambda deque_t: deque_t[t]) | deque_types) + return draw(st.lists(prim_strat)), deque_t + + @st.composite def mut_sets_of_primitives(draw): """A strategy that generates mutable sets of primitives.""" @@ -98,7 +111,7 @@ def frozen_sets_of_primitives(draw): dict_types = st.sampled_from([Dict, MutableMapping, Mapping]) seqs_of_primitives = st.one_of(lists_of_primitives(), h_tuples_of_primitives) - +deque_seqs_of_primitives = st.one_of(deques_of_primitives(), h_tuples_of_primitives) sets_of_primitives = st.one_of(mut_sets_of_primitives(), frozen_sets_of_primitives()) From ead213464472e96fc3d36839d04aae7aa5479117 Mon Sep 17 00:00:00 2001 From: Lanqing Huang Date: Thu, 20 Apr 2023 17:25:09 +0800 Subject: [PATCH 3/5] Fix compatibility issue for py3.7/py3.8 and improve docs --- docs/structuring.md | 6 +++++- src/cattrs/_compat.py | 9 ++++++++- tests/test_converter.py | 17 ++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/structuring.md b/docs/structuring.md index 65ad578e..c9467952 100644 --- a/docs/structuring.md +++ b/docs/structuring.md @@ -154,8 +154,9 @@ to deques are: - `Deque[T]` - `deque[T]` -In all cases, a new unbounded deque (`maxlen=None`) will be returned, so this operation can be used to +In all cases, a new **unbounded** deque (`maxlen=None`) will be returned, so this operation can be used to copy an iterable into a deque. +If you want to convert into bounded `deque`, registering a custom structuring hook is a good approach. ```{doctest} >>> cattrs.structure((1, 2, 3), deque[int]) @@ -169,6 +170,9 @@ These generic types are composable with all other converters. ['1', None, '3'] ``` +```{versionadded} 23.1.0 +``` + ### Sets and Frozensets Sets and frozensets can be produced from any iterable object. Types converting diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index 3c3d19c9..17c6784d 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -178,6 +178,13 @@ def is_sequence(type: Any) -> bool: or (type.__origin__ in (Tuple, tuple) and type.__args__[1] is ...) ) + def is_deque(type: Any) -> bool: + return ( + type in (deque, Deque) + or (type.__class__ is _GenericAlias and issubclass(type.__origin__, deque)) + or type.__origin__ is deque + ) + def is_mutable_set(type): return type is set or ( type.__class__ is _GenericAlias and issubclass(type.__origin__, MutableSet) @@ -349,7 +356,7 @@ def is_sequence(type: Any) -> bool: def is_deque(type): return ( type in (deque, Deque) - or (type.__class__ is _GenericAlias and issubclass(type.__origin__, Deque)) + or (type.__class__ is _GenericAlias and issubclass(type.__origin__, deque)) or (getattr(type, "__origin__", None) is deque) ) diff --git a/tests/test_converter.py b/tests/test_converter.py index 017cb42b..eeff70d5 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -597,29 +597,24 @@ def test_seq_of_bare_classes_structure(seq_type_and_annotation): """Structure iterable of values to a sequence of primitives.""" converter = Converter() - cls_and_vals = ( - (int, (1,), {}), - (float, (1.0,), {}), - (str, ("test",), {}), - (bool, (True,), {}), - ) + bare_classes = ((int, (1,)), (float, (1.0,)), (str, ("test",)), (bool, (True,))) + seq_type, annotation = seq_type_and_annotation - for cl, vals, kwargs in cls_and_vals: + for cl, vals in bare_classes: @define(frozen=True) class C: a: cl + b: cl - seq_type, annotation = seq_type_and_annotation - - inputs = [{"a": cl(*vals, **kwargs)} for _ in range(20)] + inputs = [{"a": cl(*vals), "b": cl(*vals)} for _ in range(5)] outputs = converter.structure( inputs, cl=annotation[C] if annotation not in (Tuple, tuple) else annotation[C, ...], ) - expected = seq_type(C(a=cl(*vals, **kwargs)) for _ in range(20)) + expected = seq_type(C(a=cl(*vals), b=cl(*vals)) for _ in range(5)) assert type(outputs) == seq_type assert outputs == expected From b3804ed627917bed93b94db096de724c737ad866 Mon Sep 17 00:00:00 2001 From: Lanqing Huang Date: Fri, 19 May 2023 11:11:40 +0800 Subject: [PATCH 4/5] Fix typo in docstring of `deques_of_primitives` --- tests/untyped.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/untyped.py b/tests/untyped.py index aab308f5..073ba910 100644 --- a/tests/untyped.py +++ b/tests/untyped.py @@ -77,7 +77,7 @@ def lists_of_primitives(draw): def deques_of_primitives(draw): """Generate a strategy that yields tuples of list of primitives and types. - For example, a sample value might be ([1,2], List[int]). + For example, a sample value might be ([1,2], Deque[int]). """ prim_strat, t = draw(primitive_strategies) deque_t = draw(deque_types.map(lambda deque_t: deque_t[t]) | deque_types) From baee96b88ddad46285583c0d6b6bd8ac66d81e98 Mon Sep 17 00:00:00 2001 From: Lanqing Huang Date: Mon, 22 May 2023 10:26:32 +0800 Subject: [PATCH 5/5] Fix incorrect doctest output for example of deques --- docs/structuring.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/structuring.md b/docs/structuring.md index c9467952..57b35848 100644 --- a/docs/structuring.md +++ b/docs/structuring.md @@ -154,20 +154,20 @@ to deques are: - `Deque[T]` - `deque[T]` -In all cases, a new **unbounded** deque (`maxlen=None`) will be returned, so this operation can be used to -copy an iterable into a deque. +In all cases, a new **unbounded** deque (`maxlen=None`) will be returned, +so this operation can be used to copy an iterable into a deque. If you want to convert into bounded `deque`, registering a custom structuring hook is a good approach. ```{doctest} >>> cattrs.structure((1, 2, 3), deque[int]) -[1, 2, 3] +deque([1, 2, 3]) ``` These generic types are composable with all other converters. ```{doctest} >>> cattrs.structure((1, None, 3), deque[Optional[str]]) -['1', None, '3'] +deque(['1', None, '3']) ``` ```{versionadded} 23.1.0