From ee4e0e09d4ebea08f659ba889e62d95e37fa88f0 Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Wed, 24 Mar 2021 02:25:41 +0100 Subject: [PATCH 1/4] Implement easy collection unstructuring --- docs/converters.rst | 4 +- docs/unstructuring.rst | 51 ++++++++ src/cattr/_compat.py | 48 ++++--- src/cattr/converters.py | 60 ++++++++- tests/test_unstructure_collections.py | 177 ++++++++++++++++++++++++++ 5 files changed, 315 insertions(+), 25 deletions(-) create mode 100644 tests/test_unstructure_collections.py diff --git a/docs/converters.rst b/docs/converters.rst index 8641ce5a..01147bcb 100644 --- a/docs/converters.rst +++ b/docs/converters.rst @@ -51,7 +51,7 @@ classes. * structuring and unstructuring can be customized * support for ``attrs`` classes with PEP563 (postponed) annotations * support for generic ``attrs`` classes +* support for easy overriding collection unstructuring - -The ``GenConverter`` will become the default converter type in a later release. \ No newline at end of file +The ``GenConverter`` will become the default converter type in a later release. diff --git a/docs/unstructuring.rst b/docs/unstructuring.rst index fa66cda0..745e3108 100644 --- a/docs/unstructuring.rst +++ b/docs/unstructuring.rst @@ -35,6 +35,57 @@ a complex or recursive collection. >>> data is copy False +Customizing collection unstructuring +------------------------------------ + +Sometimes it's useful to be able to override collection unstructuring in a +generic way. A common example is using a JSON library that doesn't support +sets, but expects lists and tuples instead. + +Using ordinary unstructuring hooks for this is unwieldy due to the semantics of +``singledispatch``; in other words, you'd need to register hooks for all +specific types of set you're using (``set[int]``, ``set[float]``, +``set[str]``...), which is not useful. + +Function-based hooks can be used instead, but come with their own set of +challenges - they're complicated to write efficiently. + +The ``GenConverter`` supports easy customizations of collection unstructuring +using its ``unstruct_collection_overrides`` parameter. For example, to +unstructure all sets into lists, try the following: + +.. doctest:: + + >>> from collections.abc import Set + >>> converter = cattr.GenConverter(unstruct_collection_overrides={Set: list}) + >>> + >>> converter.unstructure({1, 2, 3}) + [1, 2, 3] + +Going even further, the ``GenConverter`` contains heuristics to support the +following Python types, in order of decreasing generality: + + * ``Sequence``, ``MutableSequence``, ``list``, ``tuple`` + * ``Set``, ``MutableSet``, ``set`` + +For example, if you override the unstructure type for ``Sequence``, but not for +``MutableSequence``, ``list`` or ``tuple``, the override will also affect those +types. An easy way to remember the rule: + + * all ``MutableSequence`` s are ``Sequence`` s, so the override will apply + * all ``list`` s are ``MutableSequence`` s, so the override will apply + * all ``tuple`` s are ``Sequence`` s, so the override will apply + +If, however, you override only ``MutableSequence``, fields annotated as +``Sequence`` will not be affected (since not all sequences are mutable +sequences), and fields annotated as tuples will not be affected (since tuples +are not mutable sequences in the first place). + +Similar logic applies to the set hierarchy. + +Make sure you're using the types from ``collections.abc`` on Python 3.9+, and +from ``typing`` on older Python versions. + ``typing.Annotated`` -------------------- diff --git a/src/cattr/_compat.py b/src/cattr/_compat.py index 0913787d..9c16763f 100644 --- a/src/cattr/_compat.py +++ b/src/cattr/_compat.py @@ -15,12 +15,12 @@ Dict, FrozenSet, List, - Mapping, - MutableMapping, - MutableSequence, - MutableSet, - Sequence, - Set, + Mapping as TypingMapping, + MutableMapping as TypingMutableMapping, + MutableSequence as TypingMutableSequence, + MutableSet as TypingMutableSet, + Sequence as TypingSequence, + Set as TypingSet, Tuple, ) @@ -91,6 +91,11 @@ def adapted_fields(type) -> List[Attribute]: if is_py37 or is_py38: + Set = TypingSet + MutableSet = TypingMutableSet + Sequence = TypingSequence + MutableSequence = TypingMutableSequence + from typing import Union, _GenericAlias def is_annotated(_): @@ -113,7 +118,7 @@ def is_sequence(type: Any) -> bool: return type in (List, list, Tuple, tuple) or ( type.__class__ is _GenericAlias and type.__origin__ is not Union - and issubclass(type.__origin__, Sequence) + and issubclass(type.__origin__, TypingSequence) ) def is_mutable_set(type): @@ -129,16 +134,16 @@ def is_frozenset(type): ) def is_mapping(type): - return type in (Mapping, dict) or ( + return type in (TypingMapping, dict) or ( type.__class__ is _GenericAlias - and issubclass(type.__origin__, Mapping) + and issubclass(type.__origin__, TypingMapping) ) bare_list_args = List.__args__ - bare_seq_args = Sequence.__args__ - bare_mapping_args = Mapping.__args__ + bare_seq_args = TypingSequence.__args__ + bare_mapping_args = TypingMapping.__args__ bare_dict_args = Dict.__args__ - bare_mutable_seq_args = MutableSequence.__args__ + bare_mutable_seq_args = TypingMutableSequence.__args__ def is_bare(type): args = type.__args__ @@ -167,6 +172,11 @@ def is_bare(type): Set as AbcSet, ) + Set = AbcSet + MutableSet = AbcMutableSet + Sequence = AbcSequence + MutableSequence = AbcMutableSequence + def is_annotated(type) -> bool: return getattr(type, "__class__", None) is _AnnotatedAlias @@ -194,8 +204,8 @@ def is_sequence(type: Any) -> bool: in ( List, list, - Sequence, - MutableSequence, + TypingSequence, + TypingMutableSequence, AbcMutableSequence, tuple, ) @@ -205,7 +215,7 @@ def is_sequence(type: Any) -> bool: (origin is not tuple) and issubclass( origin, - Sequence, + TypingSequence, ) or origin is tuple and type.__args__[1] is ... @@ -217,10 +227,10 @@ def is_sequence(type: Any) -> bool: def is_mutable_set(type): return ( - type in (Set, MutableSet, set) + type in (TypingSet, TypingMutableSet, set) or ( type.__class__ is _GenericAlias - and issubclass(type.__origin__, MutableSet) + and issubclass(type.__origin__, TypingMutableSet) ) or ( getattr(type, "__origin__", None) @@ -245,10 +255,10 @@ def is_bare(type): def is_mapping(type): return ( - type in (Mapping, Dict, MutableMapping, dict) + type in (TypingMapping, Dict, TypingMutableMapping, dict) or ( type.__class__ is _GenericAlias - and issubclass(type.__origin__, Mapping) + and issubclass(type.__origin__, TypingMapping) ) or (getattr(type, "__origin__", None) is dict) ) diff --git a/src/cattr/converters.py b/src/cattr/converters.py index f943fc20..e26e348f 100644 --- a/src/cattr/converters.py +++ b/src/cattr/converters.py @@ -1,6 +1,15 @@ from enum import Enum from functools import lru_cache -from typing import Any, Callable, Dict, Mapping, Optional, Tuple, Type, TypeVar +from typing import ( + Any, + Callable, + Dict, + Mapping, + Optional, + Tuple, + Type, + TypeVar, +) from attr import resolve_types, has as attrs_has @@ -18,6 +27,10 @@ has, fields, has_with_generic, + Set, + MutableSet, + Sequence, + MutableSequence, ) from .disambiguators import create_uniq_field_dis_func from .dispatch import MultiStrategyDispatch @@ -471,7 +484,11 @@ def _get_dis_func(union): class GenConverter(Converter): """A converter which generates specialized un/structuring functions.""" - __slots__ = ("omit_if_default", "type_overrides") + __slots__ = ( + "omit_if_default", + "type_overrides", + "_unstruct_collection_overrides", + ) def __init__( self, @@ -479,12 +496,44 @@ def __init__( unstruct_strat: UnstructureStrategy = UnstructureStrategy.AS_DICT, omit_if_default: bool = False, type_overrides: Mapping[Type, AttributeOverride] = {}, + unstruct_collection_overrides: Mapping[Type, Callable] = {}, ): super().__init__( dict_factory=dict_factory, unstruct_strat=unstruct_strat ) self.omit_if_default = omit_if_default - self.type_overrides = type_overrides + self.type_overrides = dict(type_overrides) + + self._unstruct_collection_overrides = unstruct_collection_overrides + + # Do a little post-processing magic to make things easier for users. + co = unstruct_collection_overrides + + # abc.MutableSet overrrides, if defined, apply to sets + if MutableSet in co: + if set not in co: + co[set] = co[MutableSet] + + # abc.Set overrides, if defined, apply to abc.MutableSets and sets + if Set in co: + if MutableSet not in co: + co[MutableSet] = co[Set] + if set not in co: + co[set] = co[Set] + + # abc.MutableSequence overrides, if defined, can apply to lists + if MutableSequence in co: + if list not in co: + co[list] = co[MutableSequence] + + # abc.Sequence overrides, if defined, can apply to MutableSequences, lists and tuples + if Sequence in co: + if MutableSequence not in co: + co[MutableSequence] = co[Sequence] + if list not in co: + co[list] = co[Sequence] + if tuple not in co: + co[tuple] = co[Sequence] if unstruct_strat is UnstructureStrategy.AS_DICT: # Override the attrs handler. @@ -576,7 +625,10 @@ def gen_structure_attrs_fromdict(self, cl: Type[T]) -> T: # only direct dispatch so that subclasses get separately generated return h - def gen_unstructure_iterable(self, cl: Any, unstructure_to=list): + def gen_unstructure_iterable(self, cl: Any, unstructure_to=None): + unstructure_to = self._unstruct_collection_overrides.get( + get_origin(cl) or cl, unstructure_to or list + ) h = make_iterable_unstructure_fn( cl, self, unstructure_to=unstructure_to ) diff --git a/tests/test_unstructure_collections.py b/tests/test_unstructure_collections.py new file mode 100644 index 00000000..c601a39e --- /dev/null +++ b/tests/test_unstructure_collections.py @@ -0,0 +1,177 @@ +from typing import Set + +import attr +from cattr import GenConverter +from cattr.converters import is_mutable_set, is_sequence, is_mapping +from functools import partial +from cattr._compat import is_py39_plus + +if is_py39_plus: + from collections.abc import MutableSet, Set, Sequence, MutableSequence +else: + from typing import Set, MutableSet, Sequence, MutableSequence + + +def test_collection_unstructure_override_set(): + """Test overriding unstructuring sets.""" + + # First approach, predicate hook with is_mutable_set + c = GenConverter() + + c._unstructure_func.register_func_list( + [ + ( + is_mutable_set, + partial(c.gen_unstructure_iterable, unstructure_to=list), + True, + ) + ] + ) + + assert c.unstructure({1, 2, 3}, unstructure_as=Set[int]) == [1, 2, 3] + + # Second approach, using __builtins__.set + c = GenConverter(unstruct_collection_overrides={set: list}) + + assert c.unstructure({1, 2, 3}, unstructure_as=Set[int]) == {1, 2, 3} + assert c.unstructure({1, 2, 3}, unstructure_as=MutableSet[int]) == { + 1, + 2, + 3, + } + assert c.unstructure({1, 2, 3}) == [1, 2, 3] + + # Second approach, using abc.MutableSet + c = GenConverter(unstruct_collection_overrides={MutableSet: list}) + + assert c.unstructure({1, 2, 3}, unstructure_as=Set[int]) == {1, 2, 3} + assert c.unstructure({1, 2, 3}, unstructure_as=MutableSet[int]) == [ + 1, + 2, + 3, + ] + assert c.unstructure({1, 2, 3}) == [1, 2, 3] + + # Second approach, using abc.Set + c = GenConverter(unstruct_collection_overrides={Set: list}) + + assert c.unstructure({1, 2, 3}, unstructure_as=Set[int]) == [1, 2, 3] + assert c.unstructure({1, 2, 3}, unstructure_as=MutableSet[int]) == [ + 1, + 2, + 3, + ] + assert c.unstructure({1, 2, 3}) == [1, 2, 3] + + +def test_collection_unstructure_override_seq(): + """Test overriding unstructuring seq.""" + + # First approach, predicate hook + c = GenConverter() + + c._unstructure_func.register_func_list( + [ + ( + is_sequence, + partial(c.gen_unstructure_iterable, unstructure_to=tuple), + True, + ) + ] + ) + + assert c.unstructure([1, 2, 3], unstructure_as=Sequence[int]) == (1, 2, 3) + + @attr.define + class MyList: + args = attr.ib(converter=list) + + # Second approach, using abc.MutableSequence + c = GenConverter(unstruct_collection_overrides={MutableSequence: MyList}) + + assert c.unstructure([1, 2, 3], unstructure_as=Sequence[int]) == [1, 2, 3] + assert c.unstructure( + [1, 2, 3], unstructure_as=MutableSequence[int] + ) == MyList( + [ + 1, + 2, + 3, + ] + ) + assert c.unstructure([1, 2, 3]) == MyList( + [ + 1, + 2, + 3, + ] + ) + assert c.unstructure((1, 2, 3)) == [ + 1, + 2, + 3, + ] + + # Second approach, using abc.Sequence + c = GenConverter(unstruct_collection_overrides={Sequence: MyList}) + + assert c.unstructure([1, 2, 3], unstructure_as=Sequence[int]) == MyList( + [1, 2, 3] + ) + assert c.unstructure( + [1, 2, 3], unstructure_as=MutableSequence[int] + ) == MyList([1, 2, 3]) + + assert c.unstructure([1, 2, 3]) == MyList([1, 2, 3]) + + assert c.unstructure((1, 2, 3), unstructure_as=tuple[int, ...]) == MyList( + [ + 1, + 2, + 3, + ] + ) + + # Second approach, using __builtins__.list + c = GenConverter(unstruct_collection_overrides={list: MyList}) + + assert c.unstructure([1, 2, 3], unstructure_as=Sequence[int]) == [1, 2, 3] + assert c.unstructure([1, 2, 3], unstructure_as=MutableSequence[int]) == [ + 1, + 2, + 3, + ] + assert c.unstructure([1, 2, 3]) == MyList( + [ + 1, + 2, + 3, + ] + ) + assert c.unstructure((1, 2, 3)) == [ + 1, + 2, + 3, + ] + + # Second approach, using __builtins__.tuple + c = GenConverter(unstruct_collection_overrides={tuple: MyList}) + + assert c.unstructure([1, 2, 3], unstructure_as=Sequence[int]) == [1, 2, 3] + assert c.unstructure([1, 2, 3], unstructure_as=MutableSequence[int]) == [ + 1, + 2, + 3, + ] + assert c.unstructure([1, 2, 3]) == [ + 1, + 2, + 3, + ] + assert c.unstructure((1, 2, 3)) == MyList( + [ + 1, + 2, + 3, + ] + ) From 4923295f8cd727e89c2250b1957910bf074e7600 Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Wed, 24 Mar 2021 02:28:23 +0100 Subject: [PATCH 2/4] Update changelog --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 74eeef0c..0349a740 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -6,6 +6,8 @@ History ------------------ * Fix an issue with ``GenConverter`` unstructuring ``attrs`` classes and dataclasses with generic fields. (`#65 `_) +* ``GenConverter`` has support for easy overriding of collection unstructuring types (for example, unstructure all sets to lists) through its ``unstruct_collection_overrides`` argument. + (`#137 `_) 1.4.0 (2021-03-21) ------------------ From 84fd913b15846ad75378673b45bc14b6b39255c3 Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Thu, 25 Mar 2021 00:59:37 +0100 Subject: [PATCH 3/4] Skip some tests on old Pythons --- docs/unstructuring.rst | 3 +++ tests/test_unstructure_collections.py | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/unstructuring.rst b/docs/unstructuring.rst index 745e3108..0fbffcaa 100644 --- a/docs/unstructuring.rst +++ b/docs/unstructuring.rst @@ -38,6 +38,9 @@ a complex or recursive collection. Customizing collection unstructuring ------------------------------------ +.. important:: + This feature is supported for Python 3.9 and later. + Sometimes it's useful to be able to override collection unstructuring in a generic way. A common example is using a JSON library that doesn't support sets, but expects lists and tuples instead. diff --git a/tests/test_unstructure_collections.py b/tests/test_unstructure_collections.py index c601a39e..e7118f5f 100644 --- a/tests/test_unstructure_collections.py +++ b/tests/test_unstructure_collections.py @@ -1,17 +1,16 @@ +import pytest from typing import Set import attr from cattr import GenConverter -from cattr.converters import is_mutable_set, is_sequence, is_mapping +from cattr.converters import is_mutable_set, is_sequence from functools import partial from cattr._compat import is_py39_plus -if is_py39_plus: - from collections.abc import MutableSet, Set, Sequence, MutableSequence -else: - from typing import Set, MutableSet, Sequence, MutableSequence +from collections.abc import MutableSet, Set, Sequence, MutableSequence +@pytest.mark.skipif(not is_py39_plus, reason="Requires Python 3.9+") def test_collection_unstructure_override_set(): """Test overriding unstructuring sets.""" @@ -64,6 +63,7 @@ def test_collection_unstructure_override_set(): assert c.unstructure({1, 2, 3}) == [1, 2, 3] +@pytest.mark.skipif(not is_py39_plus, reason="Requires Python 3.9+") def test_collection_unstructure_override_seq(): """Test overriding unstructuring seq.""" From 0c2cc1b1c4d8ad153bcec001fb0031f249155c6e Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Thu, 25 Mar 2021 01:24:59 +0100 Subject: [PATCH 4/4] Fix lint --- tests/test_unstructure_collections.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_unstructure_collections.py b/tests/test_unstructure_collections.py index e7118f5f..e3d45c09 100644 --- a/tests/test_unstructure_collections.py +++ b/tests/test_unstructure_collections.py @@ -1,5 +1,4 @@ import pytest -from typing import Set import attr from cattr import GenConverter