diff --git a/HISTORY.md b/HISTORY.md index 1728f61e..518b5734 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -27,6 +27,8 @@ Our backwards-compatibility policy can be found [here](https://github.com/python can now be used as decorators and have gained new features. See [here](https://catt.rs/en/latest/customizing.html#use-as-decorators) and [here](https://catt.rs/en/latest/customizing.html#id1) for more details. ([#487](https://github.com/python-attrs/cattrs/pull/487)) +- Introduce and [document](https://catt.rs/en/latest/customizing.html#customizing-collections) the {mod}`cattrs.cols` module for better collection customizations. + ([#504](https://github.com/python-attrs/cattrs/issues/504) [#540](https://github.com/python-attrs/cattrs/pull/540)) - Introduce the [_msgspec_](https://jcristharif.com/msgspec/) {mod}`preconf converter `. Only JSON is supported for now, with other formats supported by _msgspec_ to come later. ([#481](https://github.com/python-attrs/cattrs/pull/481)) @@ -46,8 +48,10 @@ can now be used as decorators and have gained new features. ([#481](https://github.com/python-attrs/cattrs/pull/481)) - The {class}`orjson preconf converter ` now passes through dates and datetimes to orjson while unstructuring, greatly improving speed. ([#463](https://github.com/python-attrs/cattrs/pull/463)) -- `cattrs.gen` generators now attach metadata to the generated functions, making them introspectable. +- {mod}`cattrs.gen` generators now attach metadata to the generated functions, making them introspectable. ([#472](https://github.com/python-attrs/cattrs/pull/472)) +- Structure hook factories in {mod}`cattrs.gen` now handle recursive classes better. + ([#540](https://github.com/python-attrs/cattrs/pull/540)) - The [tagged union strategy](https://catt.rs/en/stable/strategies.html#tagged-unions-strategy) now leaves the tags in the payload unless `forbid_extra_keys` is set. ([#533](https://github.com/python-attrs/cattrs/issues/533) [#534](https://github.com/python-attrs/cattrs/pull/534)) - More robust support for `Annotated` and `NotRequired` in TypedDicts. diff --git a/docs/basics.md b/docs/basics.md index f40559ab..8465dfc7 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -14,7 +14,7 @@ To create a private converter, instantiate a {class}`cattrs.Converter`. Converte The two main methods, {meth}`structure ` and {meth}`unstructure `, are used to convert between _structured_ and _unstructured_ data. -```python +```{doctest} basics >>> from cattrs import structure, unstructure >>> from attrs import define @@ -23,7 +23,7 @@ The two main methods, {meth}`structure ` and {me ... a: int >>> unstructure(Model(1)) -{"a": 1} +{'a': 1} >>> structure({"a": 1}, Model) Model(a=1) ``` @@ -31,32 +31,31 @@ Model(a=1) _cattrs_ comes with a rich library of un/structuring hooks by default but it excels at composing custom hooks with built-in ones. The simplest approach to customization is writing a new hook from scratch. -For example, we can write our own hook for the `int` class. +For example, we can write our own hook for the `int` class and register it to a converter. -```python ->>> def int_hook(value, type): +```{doctest} basics +>>> from cattrs import Converter + +>>> converter = Converter() + +>>> @converter.register_structure_hook +... def int_hook(value, type) -> int: ... if not isinstance(value, int): ... raise ValueError('not an int!') ... return value ``` -We can then register this hook to a converter and any other hook converting an `int` will use it. - -```python ->>> from cattrs import Converter - ->>> converter = Converter() ->>> converter.register_structure_hook(int, int_hook) -``` +Now, any other hook converting an `int` will use it. -Another approach to customization is wrapping an existing hook with your own function. +Another approach to customization is wrapping (composing) an existing hook with your own function. A base hook can be obtained from a converter and then be subjected to the very rich machinery of function composition that Python offers. -```python +```{doctest} basics >>> base_hook = converter.get_structure_hook(Model) ->>> def my_model_hook(value, type): +>>> @converter.register_structure_hook +... def my_model_hook(value, type) -> Model: ... # Apply any preprocessing to the value. ... result = base_hook(value, type) ... # Apply any postprocessing to the model. @@ -65,13 +64,6 @@ A base hook can be obtained from a converter and then be subjected to the very r (`cattrs.structure({}, Model)` is equivalent to `cattrs.get_structure_hook(Model)({}, Model)`.) -This new hook can be used directly or registered to a converter (the original instance, or a different one): - -```python ->>> converter.register_structure_hook(Model, my_model_hook) -``` - - Now if we use this hook to structure a `Model`, through ✨the magic of function composition✨ that hook will use our old `int_hook`. ```python diff --git a/docs/cattrs.rst b/docs/cattrs.rst index 5170d264..13bd1f0a 100644 --- a/docs/cattrs.rst +++ b/docs/cattrs.rst @@ -19,6 +19,14 @@ Subpackages Submodules ---------- +cattrs.cols module +------------------ + +.. automodule:: cattrs.cols + :members: + :undoc-members: + :show-inheritance: + cattrs.disambiguators module ---------------------------- diff --git a/docs/conf.py b/docs/conf.py old mode 100755 new mode 100644 diff --git a/docs/customizing.md b/docs/customizing.md index a1f009c6..07802b83 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -155,6 +155,76 @@ Here's an example of using an unstructure hook factory to handle unstructuring [ [1, 2] ``` +## Customizing Collections + +The {mod}`cattrs.cols` module contains predicates and hook factories useful for customizing collection handling. +These hook factories can be wrapped to apply complex customizations. + +Available predicates are: + +* {meth}`is_any_set ` +* {meth}`is_frozenset ` +* {meth}`is_set ` +* {meth}`is_sequence ` +* {meth}`is_namedtuple ` + +````{tip} +These predicates aren't _cattrs_-specific and may be useful in other contexts. +```{doctest} predicates +>>> from cattrs.cols import is_sequence + +>>> is_sequence(list[str]) +True +``` +```` + + +Available hook factories are: + +* {meth}`iterable_unstructure_factory ` +* {meth}`list_structure_factory ` +* {meth}`namedtuple_structure_factory ` +* {meth}`namedtuple_unstructure_factory ` + +Additional predicates and hook factories will be added as requested. + +For example, by default sequences are structured from any iterable into lists. +This may be too lax, and additional validation may be applied by wrapping the default list structuring hook factory. + +```{testcode} list-customization +from cattrs.cols import is_sequence, list_structure_factory + +c = Converter() + +@c.register_structure_hook_factory(is_sequence) +def strict_list_hook_factory(type, converter): + + # First, we generate the default hook... + list_hook = list_structure_factory(type, converter) + + # Then, we wrap it with a function of our own... + def strict_list_hook(value, type): + if not isinstance(value, list): + raise ValueError("Not a list!") + return list_hook(value, type) + + # And finally, we return our own composite hook. + return strict_list_hook +``` + +Now, all sequence structuring will be stricter: + +```{doctest} list-customization +>>> c.structure({"a", "b", "c"}, list[str]) +Traceback (most recent call last): + ... +ValueError: Not a list! +``` + +```{versionadded} 24.1.0 + +``` + ## Using `cattrs.gen` Generators The {mod}`cattrs.gen` module allows for generating and compiling specialized hooks for unstructuring _attrs_ classes, dataclasses and typed dicts. diff --git a/docs/indepth.md b/docs/indepth.md index 94048349..a0700af4 100644 --- a/docs/indepth.md +++ b/docs/indepth.md @@ -23,6 +23,10 @@ The new copy may be changed through the `copy` arguments, but will retain all ma This feature is supported for Python 3.9 and later. ``` +```{tip} +See [](customizing.md#customizing-collections) for a more modern and more powerful way of customizing collection handling. +``` + Overriding collection unstructuring in a generic way can be a very useful feature. A common example is using a JSON library that doesn't support sets, but expects lists and tuples instead. diff --git a/docs/index.md b/docs/index.md index 323ef2d0..24cd50d4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,6 +37,7 @@ caption: Dev Guide history benchmarking contributing +modindex ``` ```{include} ../README.md diff --git a/docs/strategies.md b/docs/strategies.md index 4a9540a0..e4ba639a 100644 --- a/docs/strategies.md +++ b/docs/strategies.md @@ -376,6 +376,7 @@ This strategy has been preapplied to the following preconfigured converters: - {py:class}`Cbor2Converter ` - {py:class}`JsonConverter ` - {py:class}`MsgpackConverter ` +- {py:class}`MsgspecJsonConverter ` - {py:class}`OrjsonConverter ` - {py:class}`PyyamlConverter ` - {py:class}`TomlkitConverter ` diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index bad9d037..0eda9947 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -332,6 +332,11 @@ def get_notrequired_base(type) -> "Union[Any, Literal[NOTHING]]": return NOTHING def is_sequence(type: Any) -> bool: + """A predicate function for sequences. + + Matches lists, sequences, mutable sequences, deques and homogenous + tuples. + """ origin = getattr(type, "__origin__", None) return ( type @@ -366,7 +371,11 @@ def is_deque(type): or (getattr(type, "__origin__", None) is deque) ) - def is_mutable_set(type): + def is_mutable_set(type: Any) -> bool: + """A predicate function for (mutable) sets. + + Matches built-in sets and sets from the typing module. + """ return ( type in (TypingSet, TypingMutableSet, set) or ( @@ -376,7 +385,11 @@ def is_mutable_set(type): or (getattr(type, "__origin__", None) in (set, AbcMutableSet, AbcSet)) ) - def is_frozenset(type): + def is_frozenset(type: Any) -> bool: + """A predicate function for frozensets. + + Matches built-in frozensets and frozensets from the typing module. + """ return ( type in (FrozenSet, frozenset) or ( @@ -491,9 +504,10 @@ def is_deque(type: Any) -> bool: or type.__origin__ is deque ) - def is_mutable_set(type): - return type is set or ( - type.__class__ is _GenericAlias and is_subclass(type.__origin__, MutableSet) + def is_mutable_set(type) -> bool: + return type in (set, TypingAbstractSet) or ( + type.__class__ is _GenericAlias + and is_subclass(type.__origin__, (MutableSet, TypingAbstractSet)) ) def is_frozenset(type): diff --git a/src/cattrs/cols.py b/src/cattrs/cols.py new file mode 100644 index 00000000..c8d093ea --- /dev/null +++ b/src/cattrs/cols.py @@ -0,0 +1,189 @@ +"""Utility functions for collections.""" + +from __future__ import annotations + +from sys import version_info +from typing import TYPE_CHECKING, Any, Iterable, NamedTuple, Tuple, TypeVar + +from ._compat import ANIES, is_bare, is_frozenset, is_sequence, is_subclass +from ._compat import is_mutable_set as is_set +from .dispatch import StructureHook, UnstructureHook +from .errors import IterableValidationError, IterableValidationNote +from .fns import identity +from .gen import make_hetero_tuple_unstructure_fn + +if TYPE_CHECKING: + from .converters import BaseConverter + +__all__ = [ + "is_any_set", + "is_frozenset", + "is_namedtuple", + "is_set", + "is_sequence", + "iterable_unstructure_factory", + "list_structure_factory", + "namedtuple_structure_factory", + "namedtuple_unstructure_factory", +] + + +def is_any_set(type) -> bool: + """A predicate function for both mutable and frozensets.""" + return is_set(type) or is_frozenset(type) + + +if version_info[:2] >= (3, 9): + + def is_namedtuple(type: Any) -> bool: + """A predicate function for named tuples.""" + + if is_subclass(type, tuple): + for cl in type.mro(): + orig_bases = cl.__dict__.get("__orig_bases__", ()) + if NamedTuple in orig_bases: + return True + return False + +else: + + def is_namedtuple(type: Any) -> bool: + """A predicate function for named tuples.""" + # This is tricky. It may not be possible for this function to be 100% + # accurate, since it doesn't seem like we can distinguish between tuple + # subclasses and named tuples reliably. + + if is_subclass(type, tuple): + for cl in type.mro(): + if cl is tuple: + # No point going further. + break + if "_fields" in cl.__dict__: + return True + return False + + +def _is_passthrough(type: type[tuple], converter: BaseConverter) -> bool: + """If all fields would be passed through, this class should not be processed + either. + """ + return all( + converter.get_unstructure_hook(t) == identity + for t in type.__annotations__.values() + ) + + +T = TypeVar("T") + + +def list_structure_factory(type: type, converter: BaseConverter) -> StructureHook: + """A hook factory for structuring lists. + + Converts any given iterable into a list. + """ + + if is_bare(type) or type.__args__[0] in ANIES: + + def structure_list(obj: Iterable[T], _: type = type) -> list[T]: + return list(obj) + + return structure_list + + elem_type = type.__args__[0] + + try: + handler = converter.get_structure_hook(elem_type) + except RecursionError: + # Break the cycle by using late binding. + handler = converter.structure + + if converter.detailed_validation: + + def structure_list( + obj: Iterable[T], _: type = type, _handler=handler, _elem_type=elem_type + ) -> list[T]: + errors = [] + res = [] + ix = 0 # Avoid `enumerate` for performance. + for e in obj: + try: + res.append(handler(e, _elem_type)) + except Exception as e: + msg = IterableValidationNote( + f"Structuring {type} @ index {ix}", ix, elem_type + ) + e.__notes__ = [*getattr(e, "__notes__", []), msg] + errors.append(e) + finally: + ix += 1 + if errors: + raise IterableValidationError( + f"While structuring {type!r}", errors, type + ) + + return res + + else: + + def structure_list( + obj: Iterable[T], _: type = type, _handler=handler, _elem_type=elem_type + ) -> list[T]: + return [_handler(e, _elem_type) for e in obj] + + return structure_list + + +def iterable_unstructure_factory( + cl: Any, converter: BaseConverter, unstructure_to: Any = None +) -> UnstructureHook: + """A hook factory for unstructuring iterables. + + :param unstructure_to: Force unstructuring to this type, if provided. + """ + handler = converter.unstructure + + # Let's try fishing out the type args + # Unspecified tuples have `__args__` as empty tuples, so guard + # against IndexError. + if getattr(cl, "__args__", None) not in (None, ()): + type_arg = cl.__args__[0] + if isinstance(type_arg, TypeVar): + type_arg = getattr(type_arg, "__default__", Any) + handler = converter.get_unstructure_hook(type_arg, cache_result=False) + if handler == identity: + # Save ourselves the trouble of iterating over it all. + return unstructure_to or cl + + def unstructure_iterable(iterable, _seq_cl=unstructure_to or cl, _hook=handler): + return _seq_cl(_hook(i) for i in iterable) + + return unstructure_iterable + + +def namedtuple_unstructure_factory( + type: type[tuple], converter: BaseConverter, unstructure_to: Any = None +) -> UnstructureHook: + """A hook factory for unstructuring namedtuples. + + :param unstructure_to: Force unstructuring to this type, if provided. + """ + + if unstructure_to is None and _is_passthrough(type, converter): + return identity + + return make_hetero_tuple_unstructure_fn( + type, + converter, + unstructure_to=tuple if unstructure_to is None else unstructure_to, + type_args=tuple(type.__annotations__.values()), + ) + + +def namedtuple_structure_factory( + type: type[tuple], converter: BaseConverter +) -> StructureHook: + """A hook factory for structuring namedtuples.""" + # We delegate to the existing infrastructure for heterogenous tuples. + hetero_tuple_type = Tuple[tuple(type.__annotations__.values())] + base_hook = converter.get_structure_hook(hetero_tuple_type) + return lambda v, _: type(*base_hook(v, hetero_tuple_type)) diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 5ca1d065..35a9ba59 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -51,6 +51,12 @@ is_union_type, signature, ) +from .cols import ( + is_namedtuple, + list_structure_factory, + namedtuple_structure_factory, + namedtuple_unstructure_factory, +) from .disambiguators import create_default_dis_func, is_supported_union from .dispatch import ( HookFactory, @@ -83,11 +89,6 @@ ) from .gen.typeddicts import make_dict_structure_fn as make_typeddict_dict_struct_fn from .gen.typeddicts import make_dict_unstructure_fn as make_typeddict_dict_unstruct_fn -from .tuples import ( - is_namedtuple, - namedtuple_structure_factory, - namedtuple_unstructure_factory, -) __all__ = ["UnstructureStrategy", "BaseConverter", "Converter", "GenConverter"] @@ -238,7 +239,7 @@ def __init__( ), (is_literal, self._structure_simple_literal), (is_literal_containing_enums, self._structure_enum_literal), - (is_sequence, self._structure_list), + (is_sequence, list_structure_factory, "extended"), (is_deque, self._structure_deque), (is_mutable_set, self._structure_set), (is_frozenset, self._structure_frozenset), @@ -738,36 +739,6 @@ def structure_attrs_fromdict(self, obj: Mapping[str, Any], cl: type[T]) -> T: return cl(**conv_obj) - def _structure_list(self, obj: Iterable[T], cl: Any) -> list[T]: - """Convert an iterable to a potentially generic list.""" - if is_bare(cl) or cl.__args__[0] in ANIES: - res = list(obj) - else: - elem_type = cl.__args__[0] - handler = self._structure_func.dispatch(elem_type) - if self.detailed_validation: - errors = [] - res = [] - ix = 0 # Avoid `enumerate` for performance. - for e in obj: - try: - res.append(handler(e, elem_type)) - except Exception as e: - msg = IterableValidationNote( - f"Structuring {cl} @ index {ix}", ix, elem_type - ) - e.__notes__ = [*getattr(e, "__notes__", []), msg] - errors.append(e) - finally: - ix += 1 - if errors: - raise IterableValidationError( - f"While structuring {cl!r}", errors, cl - ) - else: - res = [handler(e, elem_type) for e in obj] - return res - def _structure_deque(self, obj: Iterable[T], cl: Any) -> deque[T]: """Convert an iterable to a potentially generic deque.""" if is_bare(cl) or cl.__args__[0] in ANIES: diff --git a/src/cattrs/gen/__init__.py b/src/cattrs/gen/__init__.py index e8b40cf8..afabfa2b 100644 --- a/src/cattrs/gen/__init__.py +++ b/src/cattrs/gen/__init__.py @@ -356,219 +356,175 @@ def make_dict_structure_fn( globs["__c_a"] = allowed_fields globs["__c_feke"] = ForbiddenExtraKeysError - if _cattrs_detailed_validation: - lines.append(" res = {}") - lines.append(" errors = []") - invocation_lines.append("**res,") - internal_arg_parts["__c_cve"] = ClassValidationError - internal_arg_parts["__c_avn"] = AttributeValidationNote - for a in attrs: - an = a.name - override = kwargs.get(an, neutral) - if override.omit: - continue - if override.omit is None and not a.init and not _cattrs_include_init_false: - continue - t = a.type - if isinstance(t, TypeVar): - t = mapping.get(t.__name__, t) - elif is_generic(t) and not is_bare(t) and not is_annotated(t): - t = deep_copy_with(t, mapping) - - # For each attribute, we try resolving the type here and now. - # If a type is manually overwritten, this function should be - # regenerated. - if override.struct_hook is not None: - # If the user has requested an override, just use that. - handler = override.struct_hook - else: - handler = find_structure_handler( - a, t, converter, _cattrs_prefer_attrib_converters - ) + # We keep track of what we're generating to help with recursive + # class graphs. + try: + working_set = already_generating.working_set + except AttributeError: + working_set = set() + already_generating.working_set = working_set + else: + if cl in working_set: + raise RecursionError() - struct_handler_name = f"__c_structure_{an}" - if handler is not None: - internal_arg_parts[struct_handler_name] = handler + working_set.add(cl) - ian = a.alias - if override.rename is None: - kn = an if not _cattrs_use_alias else a.alias - else: - kn = override.rename + try: + if _cattrs_detailed_validation: + lines.append(" res = {}") + lines.append(" errors = []") + invocation_lines.append("**res,") + internal_arg_parts["__c_cve"] = ClassValidationError + internal_arg_parts["__c_avn"] = AttributeValidationNote + for a in attrs: + an = a.name + override = kwargs.get(an, neutral) + if override.omit: + continue + if ( + override.omit is None + and not a.init + and not _cattrs_include_init_false + ): + continue + t = a.type + if isinstance(t, TypeVar): + t = mapping.get(t.__name__, t) + elif is_generic(t) and not is_bare(t) and not is_annotated(t): + t = deep_copy_with(t, mapping) - allowed_fields.add(kn) - i = " " + # For each attribute, we try resolving the type here and now. + # If a type is manually overwritten, this function should be + # regenerated. + if override.struct_hook is not None: + # If the user has requested an override, just use that. + handler = override.struct_hook + else: + handler = find_structure_handler( + a, t, converter, _cattrs_prefer_attrib_converters + ) - if not a.init: - if a.default is not NOTHING: - pi_lines.append(f"{i}if '{kn}' in o:") - i = f"{i} " - pi_lines.append(f"{i}try:") - i = f"{i} " - type_name = f"__c_type_{an}" - internal_arg_parts[type_name] = t + struct_handler_name = f"__c_structure_{an}" if handler is not None: - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - pi_lines.append( - f"{i}instance.{an} = {struct_handler_name}(o['{kn}'])" - ) - else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - pi_lines.append( - f"{i}instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" - ) + internal_arg_parts[struct_handler_name] = handler + + ian = a.alias + if override.rename is None: + kn = an if not _cattrs_use_alias else a.alias else: - pi_lines.append(f"{i}instance.{an} = o['{kn}']") - i = i[:-2] - pi_lines.append(f"{i}except Exception as e:") - i = f"{i} " - pi_lines.append( - f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' - ) - pi_lines.append(f"{i}errors.append(e)") + kn = override.rename - else: - if a.default is not NOTHING: - lines.append(f"{i}if '{kn}' in o:") + allowed_fields.add(kn) + i = " " + + if not a.init: + if a.default is not NOTHING: + pi_lines.append(f"{i}if '{kn}' in o:") + i = f"{i} " + pi_lines.append(f"{i}try:") i = f"{i} " - lines.append(f"{i}try:") - i = f"{i} " - type_name = f"__c_type_{an}" - internal_arg_parts[type_name] = t - if handler: - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - lines.append( - f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'])" - ) + type_name = f"__c_type_{an}" + internal_arg_parts[type_name] = t + if handler is not None: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + pi_lines.append( + f"{i}instance.{an} = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + pi_lines.append( + f"{i}instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" + ) else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - lines.append( - f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'], {tn})" - ) - else: - lines.append(f"{i}res['{ian}'] = o['{kn}']") - i = i[:-2] - lines.append(f"{i}except Exception as e:") - i = f"{i} " - lines.append( - f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' - ) - lines.append(f"{i}errors.append(e)") + pi_lines.append(f"{i}instance.{an} = o['{kn}']") + i = i[:-2] + pi_lines.append(f"{i}except Exception as e:") + i = f"{i} " + pi_lines.append( + f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' + ) + pi_lines.append(f"{i}errors.append(e)") - if _cattrs_forbid_extra_keys: - post_lines += [ - " unknown_fields = set(o.keys()) - __c_a", - " if unknown_fields:", - " errors.append(__c_feke('', __cl, unknown_fields))", - ] + else: + if a.default is not NOTHING: + lines.append(f"{i}if '{kn}' in o:") + i = f"{i} " + lines.append(f"{i}try:") + i = f"{i} " + type_name = f"__c_type_{an}" + internal_arg_parts[type_name] = t + if handler: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + lines.append( + f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + lines.append( + f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'], {tn})" + ) + else: + lines.append(f"{i}res['{ian}'] = o['{kn}']") + i = i[:-2] + lines.append(f"{i}except Exception as e:") + i = f"{i} " + lines.append( + f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' + ) + lines.append(f"{i}errors.append(e)") - post_lines.append( - f" if errors: raise __c_cve('While structuring ' + {cl_name!r}, errors, __cl)" - ) - if not pi_lines: - instantiation_lines = ( - [" try:"] - + [" return __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - + [ - f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" + if _cattrs_forbid_extra_keys: + post_lines += [ + " unknown_fields = set(o.keys()) - __c_a", + " if unknown_fields:", + " errors.append(__c_feke('', __cl, unknown_fields))", ] - ) - else: - instantiation_lines = ( - [" try:"] - + [" instance = __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - + [ - f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" - ] - ) - pi_lines.append(" return instance") - else: - non_required = [] - # The first loop deals with required args. - for a in attrs: - an = a.name - override = kwargs.get(an, neutral) - if override.omit: - continue - if override.omit is None and not a.init and not _cattrs_include_init_false: - continue - if a.default is not NOTHING: - non_required.append(a) - continue - t = a.type - if isinstance(t, TypeVar): - t = mapping.get(t.__name__, t) - elif is_generic(t) and not is_bare(t) and not is_annotated(t): - t = deep_copy_with(t, mapping) - # For each attribute, we try resolving the type here and now. - # If a type is manually overwritten, this function should be - # regenerated. - if override.struct_hook is not None: - # If the user has requested an override, just use that. - handler = override.struct_hook - else: - handler = find_structure_handler( - a, t, converter, _cattrs_prefer_attrib_converters + post_lines.append( + f" if errors: raise __c_cve('While structuring ' + {cl_name!r}, errors, __cl)" + ) + if not pi_lines: + instantiation_lines = ( + [" try:"] + + [" return __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + + [ + f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" + ] ) - - if override.rename is None: - kn = an if not _cattrs_use_alias else a.alias - else: - kn = override.rename - allowed_fields.add(kn) - - if not a.init: - if handler is not None: - struct_handler_name = f"__c_structure_{an}" - internal_arg_parts[struct_handler_name] = handler - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - pi_line = f" instance.{an} = {struct_handler_name}(o['{kn}'])" - else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - pi_line = ( - f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" - ) - else: - pi_line = f" instance.{an} = o['{kn}']" - - pi_lines.append(pi_line) else: - if handler: - struct_handler_name = f"__c_structure_{an}" - internal_arg_parts[struct_handler_name] = handler - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - invocation_line = f"{struct_handler_name}(o['{kn}'])," - else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - invocation_line = f"{struct_handler_name}(o['{kn}'], {tn})," - else: - invocation_line = f"o['{kn}']," - - if a.kw_only: - invocation_line = f"{a.alias}={invocation_line}" - invocation_lines.append(invocation_line) - - # The second loop is for optional args. - if non_required: - invocation_lines.append("**res,") - lines.append(" res = {}") - - for a in non_required: + instantiation_lines = ( + [" try:"] + + [" instance = __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + + [ + f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" + ] + ) + pi_lines.append(" return instance") + else: + non_required = [] + # The first loop deals with required args. + for a in attrs: an = a.name override = kwargs.get(an, neutral) + if override.omit: + continue + if ( + override.omit is None + and not a.init + and not _cattrs_include_init_false + ): + continue + if a.default is not NOTHING: + non_required.append(a) + continue t = a.type if isinstance(t, TypeVar): t = mapping.get(t.__name__, t) @@ -586,66 +542,136 @@ def make_dict_structure_fn( a, t, converter, _cattrs_prefer_attrib_converters ) - struct_handler_name = f"__c_structure_{an}" - internal_arg_parts[struct_handler_name] = handler - if override.rename is None: kn = an if not _cattrs_use_alias else a.alias else: kn = override.rename allowed_fields.add(kn) + if not a.init: - pi_lines.append(f" if '{kn}' in o:") - if handler: + if handler is not None: + struct_handler_name = f"__c_structure_{an}" + internal_arg_parts[struct_handler_name] = handler if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t - pi_lines.append( - f" instance.{an} = {struct_handler_name}(o['{kn}'])" + pi_line = ( + f" instance.{an} = {struct_handler_name}(o['{kn}'])" ) else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t - pi_lines.append( - f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" - ) + pi_line = f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" else: - pi_lines.append(f" instance.{an} = o['{kn}']") + pi_line = f" instance.{an} = o['{kn}']" + + pi_lines.append(pi_line) else: - post_lines.append(f" if '{kn}' in o:") if handler: + struct_handler_name = f"__c_structure_{an}" + internal_arg_parts[struct_handler_name] = handler if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t - post_lines.append( - f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'])" - ) + invocation_line = f"{struct_handler_name}(o['{kn}'])," else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t - post_lines.append( - f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'], {tn})" - ) + invocation_line = f"{struct_handler_name}(o['{kn}'], {tn})," else: - post_lines.append(f" res['{a.alias}'] = o['{kn}']") - if not pi_lines: - instantiation_lines = ( - [" return __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - ) - else: - instantiation_lines = ( - [" instance = __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - ) - pi_lines.append(" return instance") - - if _cattrs_forbid_extra_keys: - post_lines += [ - " unknown_fields = set(o.keys()) - __c_a", - " if unknown_fields:", - " raise __c_feke('', __cl, unknown_fields)", - ] + invocation_line = f"o['{kn}']," + + if a.kw_only: + invocation_line = f"{a.alias}={invocation_line}" + invocation_lines.append(invocation_line) + + # The second loop is for optional args. + if non_required: + invocation_lines.append("**res,") + lines.append(" res = {}") + + for a in non_required: + an = a.name + override = kwargs.get(an, neutral) + t = a.type + if isinstance(t, TypeVar): + t = mapping.get(t.__name__, t) + elif is_generic(t) and not is_bare(t) and not is_annotated(t): + t = deep_copy_with(t, mapping) + + # For each attribute, we try resolving the type here and now. + # If a type is manually overwritten, this function should be + # regenerated. + if override.struct_hook is not None: + # If the user has requested an override, just use that. + handler = override.struct_hook + else: + handler = find_structure_handler( + a, t, converter, _cattrs_prefer_attrib_converters + ) + + struct_handler_name = f"__c_structure_{an}" + internal_arg_parts[struct_handler_name] = handler + + if override.rename is None: + kn = an if not _cattrs_use_alias else a.alias + else: + kn = override.rename + allowed_fields.add(kn) + if not a.init: + pi_lines.append(f" if '{kn}' in o:") + if handler: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + pi_lines.append( + f" instance.{an} = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + pi_lines.append( + f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" + ) + else: + pi_lines.append(f" instance.{an} = o['{kn}']") + else: + post_lines.append(f" if '{kn}' in o:") + if handler: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + post_lines.append( + f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + post_lines.append( + f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'], {tn})" + ) + else: + post_lines.append(f" res['{a.alias}'] = o['{kn}']") + if not pi_lines: + instantiation_lines = ( + [" return __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + ) + else: + instantiation_lines = ( + [" instance = __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + ) + pi_lines.append(" return instance") + + if _cattrs_forbid_extra_keys: + post_lines += [ + " unknown_fields = set(o.keys()) - __c_a", + " if unknown_fields:", + " raise __c_feke('', __cl, unknown_fields)", + ] + finally: + working_set.remove(cl) + if not working_set: + del already_generating.working_set # At the end, we create the function header. internal_arg_line = ", ".join([f"{i}={i}" for i in internal_arg_parts]) diff --git a/src/cattrs/gen/_shared.py b/src/cattrs/gen/_shared.py index 5a9e3aa7..4e631437 100644 --- a/src/cattrs/gen/_shared.py +++ b/src/cattrs/gen/_shared.py @@ -19,34 +19,40 @@ def find_structure_handler( Return `None` if no handler should be used. """ - if a.converter is not None and prefer_attrs_converters: - # If the user as requested to use attrib converters, use nothing - # so it falls back to that. - handler = None - elif a.converter is not None and not prefer_attrs_converters and type is not None: - handler = c.get_structure_hook(type, cache_result=False) - if handler == raise_error: + try: + if a.converter is not None and prefer_attrs_converters: + # If the user as requested to use attrib converters, use nothing + # so it falls back to that. handler = None - elif type is not None: - if ( - is_bare_final(type) - and a.default is not NOTHING - and not isinstance(a.default, Factory) + elif ( + a.converter is not None and not prefer_attrs_converters and type is not None ): - # This is a special case where we can use the - # type of the default to dispatch on. - type = a.default.__class__ handler = c.get_structure_hook(type, cache_result=False) - if handler == c._structure_call: - # Finals can't really be used with _structure_call, so - # we wrap it so the rest of the toolchain doesn't get - # confused. - - def handler(v, _, _h=handler): - return _h(v, type) - + if handler == raise_error: + handler = None + elif type is not None: + if ( + is_bare_final(type) + and a.default is not NOTHING + and not isinstance(a.default, Factory) + ): + # This is a special case where we can use the + # type of the default to dispatch on. + type = a.default.__class__ + handler = c.get_structure_hook(type, cache_result=False) + if handler == c._structure_call: + # Finals can't really be used with _structure_call, so + # we wrap it so the rest of the toolchain doesn't get + # confused. + + def handler(v, _, _h=handler): + return _h(v, type) + + else: + handler = c.get_structure_hook(type, cache_result=False) else: - handler = c.get_structure_hook(type, cache_result=False) - else: - handler = c.structure - return handler + handler = c.structure + return handler + except RecursionError: + # This means we're dealing with a reference cycle, so use late binding. + return c.structure diff --git a/src/cattrs/preconf/msgspec.py b/src/cattrs/preconf/msgspec.py index e58cbec8..6ef84d76 100644 --- a/src/cattrs/preconf/msgspec.py +++ b/src/cattrs/preconf/msgspec.py @@ -13,7 +13,7 @@ from msgspec import Struct, convert, to_builtins from msgspec.json import Encoder, decode -from cattrs._compat import ( +from .._compat import ( fields, get_args, get_origin, @@ -22,13 +22,12 @@ is_mapping, is_sequence, ) -from cattrs.dispatch import UnstructureHook -from cattrs.fns import identity - +from ..cols import is_namedtuple from ..converters import BaseConverter, Converter +from ..dispatch import UnstructureHook +from ..fns import identity from ..gen import make_hetero_tuple_unstructure_fn from ..strategies import configure_union_passthrough -from ..tuples import is_namedtuple from . import wrap T = TypeVar("T") diff --git a/src/cattrs/preconf/orjson.py b/src/cattrs/preconf/orjson.py index 60e46287..4b595bcf 100644 --- a/src/cattrs/preconf/orjson.py +++ b/src/cattrs/preconf/orjson.py @@ -9,10 +9,10 @@ from orjson import dumps, loads from .._compat import AbstractSet, is_mapping +from ..cols import is_namedtuple, namedtuple_unstructure_factory from ..converters import BaseConverter, Converter from ..fns import identity from ..strategies import configure_union_passthrough -from ..tuples import is_namedtuple, namedtuple_unstructure_factory from . import wrap T = TypeVar("T") diff --git a/src/cattrs/preconf/pyyaml.py b/src/cattrs/preconf/pyyaml.py index 45bc828a..73746257 100644 --- a/src/cattrs/preconf/pyyaml.py +++ b/src/cattrs/preconf/pyyaml.py @@ -7,9 +7,9 @@ from yaml import safe_dump, safe_load from .._compat import FrozenSetSubscriptable +from ..cols import is_namedtuple, namedtuple_unstructure_factory from ..converters import BaseConverter, Converter from ..strategies import configure_union_passthrough -from ..tuples import is_namedtuple, namedtuple_unstructure_factory from . import validate_datetime, wrap T = TypeVar("T") diff --git a/src/cattrs/tuples.py b/src/cattrs/tuples.py deleted file mode 100644 index 1cddd67c..00000000 --- a/src/cattrs/tuples.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -from sys import version_info -from typing import TYPE_CHECKING, Any, NamedTuple, Tuple - -from ._compat import is_subclass -from .dispatch import StructureHook, UnstructureHook -from .fns import identity -from .gen import make_hetero_tuple_unstructure_fn - -if TYPE_CHECKING: - from .converters import BaseConverter - -if version_info[:2] >= (3, 9): - - def is_namedtuple(type: Any) -> bool: - """A predicate function for named tuples.""" - - if is_subclass(type, tuple): - for cl in type.mro(): - orig_bases = cl.__dict__.get("__orig_bases__", ()) - if NamedTuple in orig_bases: - return True - return False - -else: - - def is_namedtuple(type: Any) -> bool: - """A predicate function for named tuples.""" - # This is tricky. It may not be possible for this function to be 100% - # accurate, since it doesn't seem like we can distinguish between tuple - # subclasses and named tuples reliably. - - if is_subclass(type, tuple): - for cl in type.mro(): - if cl is tuple: - # No point going further. - break - if "_fields" in cl.__dict__: - return True - return False - - -def is_passthrough(type: type[tuple], converter: BaseConverter) -> bool: - """If all fields would be passed through, this class should not be processed - either. - """ - return all( - converter.get_unstructure_hook(t) == identity - for t in type.__annotations__.values() - ) - - -def namedtuple_unstructure_factory( - type: type[tuple], converter: BaseConverter, unstructure_to: Any = None -) -> UnstructureHook: - """A hook factory for unstructuring namedtuples. - - :param unstructure_to: Force unstructuring to this type, if provided. - """ - - if unstructure_to is None and is_passthrough(type, converter): - return identity - - return make_hetero_tuple_unstructure_fn( - type, - converter, - unstructure_to=tuple if unstructure_to is None else unstructure_to, - type_args=tuple(type.__annotations__.values()), - ) - - -def namedtuple_structure_factory( - type: type[tuple], converter: BaseConverter -) -> StructureHook: - """A hook factory for structuring namedtuples.""" - # We delegate to the existing infrastructure for heterogenous tuples. - hetero_tuple_type = Tuple[tuple(type.__annotations__.values())] - base_hook = converter.get_structure_hook(hetero_tuple_type) - return lambda v, _: type(*base_hook(v, hetero_tuple_type)) diff --git a/tests/strategies/test_include_subclasses.py b/tests/strategies/test_include_subclasses.py index 29a61281..4ddf61b2 100644 --- a/tests/strategies/test_include_subclasses.py +++ b/tests/strategies/test_include_subclasses.py @@ -343,3 +343,26 @@ class A: include_subclasses(A, genconverter, union_strategy=configure_tagged_union) assert genconverter.structure({"a": 1}, A) == A(1) + + +def test_cyclic_classes(genconverter: Converter): + """A cyclic reference case from issue #542.""" + + @define + class Base: + pass + + @define + class Subclass1(Base): + b: str + a: Base + + @define + class Subclass2(Base): + b: str + + include_subclasses(Base, genconverter, union_strategy=configure_tagged_union) + + assert genconverter.structure( + {"b": "a", "_type": "Subclass1", "a": {"b": "c", "_type": "Subclass2"}}, Base + ) == Subclass1("a", Subclass2("c")) diff --git a/tests/test_cols.py b/tests/test_cols.py new file mode 100644 index 00000000..5c596011 --- /dev/null +++ b/tests/test_cols.py @@ -0,0 +1,21 @@ +"""Tests for the `cattrs.cols` module.""" + +from cattrs import BaseConverter +from cattrs._compat import AbstractSet, FrozenSet +from cattrs.cols import is_any_set, iterable_unstructure_factory + + +def test_set_overriding(converter: BaseConverter): + """Overriding abstract sets by wrapping the default factory works.""" + + converter.register_unstructure_hook_factory( + is_any_set, + lambda t, c: iterable_unstructure_factory(t, c, unstructure_to=sorted), + ) + + assert converter.unstructure({"c", "b", "a"}, AbstractSet[str]) == ["a", "b", "c"] + assert converter.unstructure(frozenset(["c", "b", "a"]), FrozenSet[str]) == [ + "a", + "b", + "c", + ] diff --git a/tests/test_tuples.py b/tests/test_tuples.py index 91c35a50..3b63af81 100644 --- a/tests/test_tuples.py +++ b/tests/test_tuples.py @@ -2,8 +2,8 @@ from typing import NamedTuple, Tuple +from cattrs.cols import is_namedtuple from cattrs.converters import Converter -from cattrs.tuples import is_namedtuple def test_simple_hetero_tuples(genconverter: Converter):