From 1d12ef6e353c4f13dd72868890fd3ab69867c998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sun, 26 May 2024 21:38:56 +0200 Subject: [PATCH 01/10] Initial list strategy work --- src/cattrs/{tuples.py => cols.py} | 76 ++++- src/cattrs/converters.py | 79 ++--- src/cattrs/dispatch.py | 6 +- src/cattrs/gen/__init__.py | 496 ++++++++++++++++-------------- src/cattrs/gen/_shared.py | 60 ++-- src/cattrs/preconf/msgspec.py | 9 +- src/cattrs/preconf/orjson.py | 2 +- src/cattrs/preconf/pyyaml.py | 2 +- tests/test_generics.py | 9 +- tests/test_tuples.py | 2 +- 10 files changed, 413 insertions(+), 328 deletions(-) rename src/cattrs/{tuples.py => cols.py} (51%) diff --git a/src/cattrs/tuples.py b/src/cattrs/cols.py similarity index 51% rename from src/cattrs/tuples.py rename to src/cattrs/cols.py index 1cddd67c..4918ea5c 100644 --- a/src/cattrs/tuples.py +++ b/src/cattrs/cols.py @@ -1,16 +1,24 @@ +"""Utility functions for collections.""" from __future__ import annotations from sys import version_info -from typing import TYPE_CHECKING, Any, NamedTuple, Tuple +from typing import TYPE_CHECKING, Any, Iterable, NamedTuple, Tuple, TypeVar -from ._compat import is_subclass +from ._compat import ANIES, is_bare, is_subclass from .dispatch import StructureHook, UnstructureHook +from .errors import IterableValidationError, IterableValidationNote from .fns import identity from .gen import make_hetero_tuple_unstructure_fn if TYPE_CHECKING: from .converters import BaseConverter +__all__ = [ + "is_namedtuple", + "namedtuple_unstructure_factory", + "namedtuple_structure_factory", +] + if version_info[:2] >= (3, 9): def is_namedtuple(type: Any) -> bool: @@ -41,7 +49,7 @@ def is_namedtuple(type: Any) -> bool: return False -def is_passthrough(type: type[tuple], converter: BaseConverter) -> bool: +def _is_passthrough(type: type[tuple], converter: BaseConverter) -> bool: """If all fields would be passed through, this class should not be processed either. """ @@ -51,6 +59,66 @@ def is_passthrough(type: type[tuple], converter: BaseConverter) -> bool: ) +T = TypeVar("T") + + +def list_structure_factory(type: type, converter: BaseConverter) -> StructureHook: + """A hook factory for structuring lists. + + Converts any given iterable into a list. + """ + + if is_bare(type) or type.__args__[0] in ANIES: + + def structure_list(obj: Iterable[T], _: type = type) -> list[T]: + return list(obj) + + return structure_list + + elem_type = type.__args__[0] + + try: + handler = converter.get_structure_hook(elem_type) + except RecursionError: + # Break the cycle by using late binding. + handler = converter.structure + + if converter.detailed_validation: + + def structure_list( + obj: Iterable[T], _: type = type, _handler=handler, _elem_type=elem_type + ) -> list[T]: + errors = [] + res = [] + ix = 0 # Avoid `enumerate` for performance. + for e in obj: + try: + res.append(handler(e, _elem_type)) + except Exception as e: + msg = IterableValidationNote( + f"Structuring {type} @ index {ix}", ix, elem_type + ) + e.__notes__ = [*getattr(e, "__notes__", []), msg] + errors.append(e) + finally: + ix += 1 + if errors: + raise IterableValidationError( + f"While structuring {type!r}", errors, type + ) + + return res + + else: + + def structure_list( + obj: Iterable[T], _: type = type, _handler=handler, _elem_type=elem_type + ) -> list[T]: + return [_handler(e, _elem_type) for e in obj] + + return structure_list + + def namedtuple_unstructure_factory( type: type[tuple], converter: BaseConverter, unstructure_to: Any = None ) -> UnstructureHook: @@ -59,7 +127,7 @@ def namedtuple_unstructure_factory( :param unstructure_to: Force unstructuring to this type, if provided. """ - if unstructure_to is None and is_passthrough(type, converter): + if unstructure_to is None and _is_passthrough(type, converter): return identity return make_hetero_tuple_unstructure_fn( diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 5ca1d065..dec49c62 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -51,6 +51,12 @@ is_union_type, signature, ) +from .cols import ( + is_namedtuple, + list_structure_factory, + namedtuple_structure_factory, + namedtuple_unstructure_factory, +) from .disambiguators import create_default_dis_func, is_supported_union from .dispatch import ( HookFactory, @@ -83,11 +89,6 @@ ) from .gen.typeddicts import make_dict_structure_fn as make_typeddict_dict_struct_fn from .gen.typeddicts import make_dict_unstructure_fn as make_typeddict_dict_unstruct_fn -from .tuples import ( - is_namedtuple, - namedtuple_structure_factory, - namedtuple_unstructure_factory, -) __all__ = ["UnstructureStrategy", "BaseConverter", "Converter", "GenConverter"] @@ -238,7 +239,7 @@ def __init__( ), (is_literal, self._structure_simple_literal), (is_literal_containing_enums, self._structure_enum_literal), - (is_sequence, self._structure_list), + (is_sequence, list_structure_factory, "extended"), (is_deque, self._structure_deque), (is_mutable_set, self._structure_set), (is_frozenset, self._structure_frozenset), @@ -287,10 +288,12 @@ def unstruct_strat(self) -> UnstructureStrategy: ) @overload - def register_unstructure_hook(self) -> Callable[[UnstructureHook], None]: ... + def register_unstructure_hook(self) -> Callable[[UnstructureHook], None]: + ... @overload - def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: ... + def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: + ... def register_unstructure_hook( self, cls: Any = None, func: UnstructureHook | None = None @@ -338,22 +341,26 @@ def register_unstructure_hook_func( @overload def register_unstructure_hook_factory( self, predicate: Predicate - ) -> Callable[[UnstructureHookFactory], UnstructureHookFactory]: ... + ) -> Callable[[UnstructureHookFactory], UnstructureHookFactory]: + ... @overload def register_unstructure_hook_factory( self, predicate: Predicate - ) -> Callable[[ExtendedUnstructureHookFactory], ExtendedUnstructureHookFactory]: ... + ) -> Callable[[ExtendedUnstructureHookFactory], ExtendedUnstructureHookFactory]: + ... @overload def register_unstructure_hook_factory( self, predicate: Predicate, factory: UnstructureHookFactory - ) -> UnstructureHookFactory: ... + ) -> UnstructureHookFactory: + ... @overload def register_unstructure_hook_factory( self, predicate: Predicate, factory: ExtendedUnstructureHookFactory - ) -> ExtendedUnstructureHookFactory: ... + ) -> ExtendedUnstructureHookFactory: + ... def register_unstructure_hook_factory(self, predicate, factory=None): """ @@ -422,10 +429,12 @@ def get_unstructure_hook( ) @overload - def register_structure_hook(self) -> Callable[[StructureHook], None]: ... + def register_structure_hook(self) -> Callable[[StructureHook], None]: + ... @overload - def register_structure_hook(self, cl: Any, func: StructuredValue) -> None: ... + def register_structure_hook(self, cl: Any, func: StructuredValue) -> None: + ... def register_structure_hook( self, cl: Any, func: StructureHook | None = None @@ -475,22 +484,26 @@ def register_structure_hook_func( @overload def register_structure_hook_factory( self, predicate: Predicate - ) -> Callable[[StructureHookFactory, StructureHookFactory]]: ... + ) -> Callable[[StructureHookFactory, StructureHookFactory]]: + ... @overload def register_structure_hook_factory( self, predicate: Predicate - ) -> Callable[[ExtendedStructureHookFactory, ExtendedStructureHookFactory]]: ... + ) -> Callable[[ExtendedStructureHookFactory, ExtendedStructureHookFactory]]: + ... @overload def register_structure_hook_factory( self, predicate: Predicate, factory: StructureHookFactory - ) -> StructureHookFactory: ... + ) -> StructureHookFactory: + ... @overload def register_structure_hook_factory( self, predicate: Predicate, factory: ExtendedStructureHookFactory - ) -> ExtendedStructureHookFactory: ... + ) -> ExtendedStructureHookFactory: + ... def register_structure_hook_factory(self, predicate, factory=None): """ @@ -738,36 +751,6 @@ def structure_attrs_fromdict(self, obj: Mapping[str, Any], cl: type[T]) -> T: return cl(**conv_obj) - def _structure_list(self, obj: Iterable[T], cl: Any) -> list[T]: - """Convert an iterable to a potentially generic list.""" - if is_bare(cl) or cl.__args__[0] in ANIES: - res = list(obj) - else: - elem_type = cl.__args__[0] - handler = self._structure_func.dispatch(elem_type) - if self.detailed_validation: - errors = [] - res = [] - ix = 0 # Avoid `enumerate` for performance. - for e in obj: - try: - res.append(handler(e, elem_type)) - except Exception as e: - msg = IterableValidationNote( - f"Structuring {cl} @ index {ix}", ix, elem_type - ) - e.__notes__ = [*getattr(e, "__notes__", []), msg] - errors.append(e) - finally: - ix += 1 - if errors: - raise IterableValidationError( - f"While structuring {cl!r}", errors, cl - ) - else: - res = [handler(e, elem_type) for e in obj] - return res - def _structure_deque(self, obj: Iterable[T], cl: Any) -> deque[T]: """Convert an iterable to a potentially generic deque.""" if is_bare(cl) or cl.__args__[0] in ANIES: diff --git a/src/cattrs/dispatch.py b/src/cattrs/dispatch.py index 3d746dbc..f82ae878 100644 --- a/src/cattrs/dispatch.py +++ b/src/cattrs/dispatch.py @@ -44,9 +44,9 @@ class FunctionDispatch: """ _converter: BaseConverter - _handler_pairs: list[tuple[Predicate, Callable[[Any, Any], Any], bool, bool]] = ( - Factory(list) - ) + _handler_pairs: list[ + tuple[Predicate, Callable[[Any, Any], Any], bool, bool] + ] = Factory(list) def register( self, diff --git a/src/cattrs/gen/__init__.py b/src/cattrs/gen/__init__.py index e8b40cf8..afabfa2b 100644 --- a/src/cattrs/gen/__init__.py +++ b/src/cattrs/gen/__init__.py @@ -356,219 +356,175 @@ def make_dict_structure_fn( globs["__c_a"] = allowed_fields globs["__c_feke"] = ForbiddenExtraKeysError - if _cattrs_detailed_validation: - lines.append(" res = {}") - lines.append(" errors = []") - invocation_lines.append("**res,") - internal_arg_parts["__c_cve"] = ClassValidationError - internal_arg_parts["__c_avn"] = AttributeValidationNote - for a in attrs: - an = a.name - override = kwargs.get(an, neutral) - if override.omit: - continue - if override.omit is None and not a.init and not _cattrs_include_init_false: - continue - t = a.type - if isinstance(t, TypeVar): - t = mapping.get(t.__name__, t) - elif is_generic(t) and not is_bare(t) and not is_annotated(t): - t = deep_copy_with(t, mapping) - - # For each attribute, we try resolving the type here and now. - # If a type is manually overwritten, this function should be - # regenerated. - if override.struct_hook is not None: - # If the user has requested an override, just use that. - handler = override.struct_hook - else: - handler = find_structure_handler( - a, t, converter, _cattrs_prefer_attrib_converters - ) + # We keep track of what we're generating to help with recursive + # class graphs. + try: + working_set = already_generating.working_set + except AttributeError: + working_set = set() + already_generating.working_set = working_set + else: + if cl in working_set: + raise RecursionError() - struct_handler_name = f"__c_structure_{an}" - if handler is not None: - internal_arg_parts[struct_handler_name] = handler + working_set.add(cl) - ian = a.alias - if override.rename is None: - kn = an if not _cattrs_use_alias else a.alias - else: - kn = override.rename + try: + if _cattrs_detailed_validation: + lines.append(" res = {}") + lines.append(" errors = []") + invocation_lines.append("**res,") + internal_arg_parts["__c_cve"] = ClassValidationError + internal_arg_parts["__c_avn"] = AttributeValidationNote + for a in attrs: + an = a.name + override = kwargs.get(an, neutral) + if override.omit: + continue + if ( + override.omit is None + and not a.init + and not _cattrs_include_init_false + ): + continue + t = a.type + if isinstance(t, TypeVar): + t = mapping.get(t.__name__, t) + elif is_generic(t) and not is_bare(t) and not is_annotated(t): + t = deep_copy_with(t, mapping) - allowed_fields.add(kn) - i = " " + # For each attribute, we try resolving the type here and now. + # If a type is manually overwritten, this function should be + # regenerated. + if override.struct_hook is not None: + # If the user has requested an override, just use that. + handler = override.struct_hook + else: + handler = find_structure_handler( + a, t, converter, _cattrs_prefer_attrib_converters + ) - if not a.init: - if a.default is not NOTHING: - pi_lines.append(f"{i}if '{kn}' in o:") - i = f"{i} " - pi_lines.append(f"{i}try:") - i = f"{i} " - type_name = f"__c_type_{an}" - internal_arg_parts[type_name] = t + struct_handler_name = f"__c_structure_{an}" if handler is not None: - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - pi_lines.append( - f"{i}instance.{an} = {struct_handler_name}(o['{kn}'])" - ) - else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - pi_lines.append( - f"{i}instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" - ) + internal_arg_parts[struct_handler_name] = handler + + ian = a.alias + if override.rename is None: + kn = an if not _cattrs_use_alias else a.alias else: - pi_lines.append(f"{i}instance.{an} = o['{kn}']") - i = i[:-2] - pi_lines.append(f"{i}except Exception as e:") - i = f"{i} " - pi_lines.append( - f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' - ) - pi_lines.append(f"{i}errors.append(e)") + kn = override.rename - else: - if a.default is not NOTHING: - lines.append(f"{i}if '{kn}' in o:") + allowed_fields.add(kn) + i = " " + + if not a.init: + if a.default is not NOTHING: + pi_lines.append(f"{i}if '{kn}' in o:") + i = f"{i} " + pi_lines.append(f"{i}try:") i = f"{i} " - lines.append(f"{i}try:") - i = f"{i} " - type_name = f"__c_type_{an}" - internal_arg_parts[type_name] = t - if handler: - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - lines.append( - f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'])" - ) + type_name = f"__c_type_{an}" + internal_arg_parts[type_name] = t + if handler is not None: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + pi_lines.append( + f"{i}instance.{an} = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + pi_lines.append( + f"{i}instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" + ) else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - lines.append( - f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'], {tn})" - ) - else: - lines.append(f"{i}res['{ian}'] = o['{kn}']") - i = i[:-2] - lines.append(f"{i}except Exception as e:") - i = f"{i} " - lines.append( - f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' - ) - lines.append(f"{i}errors.append(e)") + pi_lines.append(f"{i}instance.{an} = o['{kn}']") + i = i[:-2] + pi_lines.append(f"{i}except Exception as e:") + i = f"{i} " + pi_lines.append( + f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' + ) + pi_lines.append(f"{i}errors.append(e)") - if _cattrs_forbid_extra_keys: - post_lines += [ - " unknown_fields = set(o.keys()) - __c_a", - " if unknown_fields:", - " errors.append(__c_feke('', __cl, unknown_fields))", - ] + else: + if a.default is not NOTHING: + lines.append(f"{i}if '{kn}' in o:") + i = f"{i} " + lines.append(f"{i}try:") + i = f"{i} " + type_name = f"__c_type_{an}" + internal_arg_parts[type_name] = t + if handler: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + lines.append( + f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + lines.append( + f"{i}res['{ian}'] = {struct_handler_name}(o['{kn}'], {tn})" + ) + else: + lines.append(f"{i}res['{ian}'] = o['{kn}']") + i = i[:-2] + lines.append(f"{i}except Exception as e:") + i = f"{i} " + lines.append( + f'{i}e.__notes__ = getattr(e, \'__notes__\', []) + [__c_avn("Structuring class {cl.__qualname__} @ attribute {an}", "{an}", __c_type_{an})]' + ) + lines.append(f"{i}errors.append(e)") - post_lines.append( - f" if errors: raise __c_cve('While structuring ' + {cl_name!r}, errors, __cl)" - ) - if not pi_lines: - instantiation_lines = ( - [" try:"] - + [" return __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - + [ - f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" + if _cattrs_forbid_extra_keys: + post_lines += [ + " unknown_fields = set(o.keys()) - __c_a", + " if unknown_fields:", + " errors.append(__c_feke('', __cl, unknown_fields))", ] - ) - else: - instantiation_lines = ( - [" try:"] - + [" instance = __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - + [ - f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" - ] - ) - pi_lines.append(" return instance") - else: - non_required = [] - # The first loop deals with required args. - for a in attrs: - an = a.name - override = kwargs.get(an, neutral) - if override.omit: - continue - if override.omit is None and not a.init and not _cattrs_include_init_false: - continue - if a.default is not NOTHING: - non_required.append(a) - continue - t = a.type - if isinstance(t, TypeVar): - t = mapping.get(t.__name__, t) - elif is_generic(t) and not is_bare(t) and not is_annotated(t): - t = deep_copy_with(t, mapping) - # For each attribute, we try resolving the type here and now. - # If a type is manually overwritten, this function should be - # regenerated. - if override.struct_hook is not None: - # If the user has requested an override, just use that. - handler = override.struct_hook - else: - handler = find_structure_handler( - a, t, converter, _cattrs_prefer_attrib_converters + post_lines.append( + f" if errors: raise __c_cve('While structuring ' + {cl_name!r}, errors, __cl)" + ) + if not pi_lines: + instantiation_lines = ( + [" try:"] + + [" return __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + + [ + f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" + ] ) - - if override.rename is None: - kn = an if not _cattrs_use_alias else a.alias - else: - kn = override.rename - allowed_fields.add(kn) - - if not a.init: - if handler is not None: - struct_handler_name = f"__c_structure_{an}" - internal_arg_parts[struct_handler_name] = handler - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - pi_line = f" instance.{an} = {struct_handler_name}(o['{kn}'])" - else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - pi_line = ( - f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" - ) - else: - pi_line = f" instance.{an} = o['{kn}']" - - pi_lines.append(pi_line) else: - if handler: - struct_handler_name = f"__c_structure_{an}" - internal_arg_parts[struct_handler_name] = handler - if handler == converter._structure_call: - internal_arg_parts[struct_handler_name] = t - invocation_line = f"{struct_handler_name}(o['{kn}'])," - else: - tn = f"__c_type_{an}" - internal_arg_parts[tn] = t - invocation_line = f"{struct_handler_name}(o['{kn}'], {tn})," - else: - invocation_line = f"o['{kn}']," - - if a.kw_only: - invocation_line = f"{a.alias}={invocation_line}" - invocation_lines.append(invocation_line) - - # The second loop is for optional args. - if non_required: - invocation_lines.append("**res,") - lines.append(" res = {}") - - for a in non_required: + instantiation_lines = ( + [" try:"] + + [" instance = __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + + [ + f" except Exception as exc: raise __c_cve('While structuring ' + {cl_name!r}, [exc], __cl)" + ] + ) + pi_lines.append(" return instance") + else: + non_required = [] + # The first loop deals with required args. + for a in attrs: an = a.name override = kwargs.get(an, neutral) + if override.omit: + continue + if ( + override.omit is None + and not a.init + and not _cattrs_include_init_false + ): + continue + if a.default is not NOTHING: + non_required.append(a) + continue t = a.type if isinstance(t, TypeVar): t = mapping.get(t.__name__, t) @@ -586,66 +542,136 @@ def make_dict_structure_fn( a, t, converter, _cattrs_prefer_attrib_converters ) - struct_handler_name = f"__c_structure_{an}" - internal_arg_parts[struct_handler_name] = handler - if override.rename is None: kn = an if not _cattrs_use_alias else a.alias else: kn = override.rename allowed_fields.add(kn) + if not a.init: - pi_lines.append(f" if '{kn}' in o:") - if handler: + if handler is not None: + struct_handler_name = f"__c_structure_{an}" + internal_arg_parts[struct_handler_name] = handler if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t - pi_lines.append( - f" instance.{an} = {struct_handler_name}(o['{kn}'])" + pi_line = ( + f" instance.{an} = {struct_handler_name}(o['{kn}'])" ) else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t - pi_lines.append( - f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" - ) + pi_line = f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" else: - pi_lines.append(f" instance.{an} = o['{kn}']") + pi_line = f" instance.{an} = o['{kn}']" + + pi_lines.append(pi_line) else: - post_lines.append(f" if '{kn}' in o:") if handler: + struct_handler_name = f"__c_structure_{an}" + internal_arg_parts[struct_handler_name] = handler if handler == converter._structure_call: internal_arg_parts[struct_handler_name] = t - post_lines.append( - f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'])" - ) + invocation_line = f"{struct_handler_name}(o['{kn}'])," else: tn = f"__c_type_{an}" internal_arg_parts[tn] = t - post_lines.append( - f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'], {tn})" - ) + invocation_line = f"{struct_handler_name}(o['{kn}'], {tn})," else: - post_lines.append(f" res['{a.alias}'] = o['{kn}']") - if not pi_lines: - instantiation_lines = ( - [" return __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - ) - else: - instantiation_lines = ( - [" instance = __cl("] - + [f" {line}" for line in invocation_lines] - + [" )"] - ) - pi_lines.append(" return instance") - - if _cattrs_forbid_extra_keys: - post_lines += [ - " unknown_fields = set(o.keys()) - __c_a", - " if unknown_fields:", - " raise __c_feke('', __cl, unknown_fields)", - ] + invocation_line = f"o['{kn}']," + + if a.kw_only: + invocation_line = f"{a.alias}={invocation_line}" + invocation_lines.append(invocation_line) + + # The second loop is for optional args. + if non_required: + invocation_lines.append("**res,") + lines.append(" res = {}") + + for a in non_required: + an = a.name + override = kwargs.get(an, neutral) + t = a.type + if isinstance(t, TypeVar): + t = mapping.get(t.__name__, t) + elif is_generic(t) and not is_bare(t) and not is_annotated(t): + t = deep_copy_with(t, mapping) + + # For each attribute, we try resolving the type here and now. + # If a type is manually overwritten, this function should be + # regenerated. + if override.struct_hook is not None: + # If the user has requested an override, just use that. + handler = override.struct_hook + else: + handler = find_structure_handler( + a, t, converter, _cattrs_prefer_attrib_converters + ) + + struct_handler_name = f"__c_structure_{an}" + internal_arg_parts[struct_handler_name] = handler + + if override.rename is None: + kn = an if not _cattrs_use_alias else a.alias + else: + kn = override.rename + allowed_fields.add(kn) + if not a.init: + pi_lines.append(f" if '{kn}' in o:") + if handler: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + pi_lines.append( + f" instance.{an} = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + pi_lines.append( + f" instance.{an} = {struct_handler_name}(o['{kn}'], {tn})" + ) + else: + pi_lines.append(f" instance.{an} = o['{kn}']") + else: + post_lines.append(f" if '{kn}' in o:") + if handler: + if handler == converter._structure_call: + internal_arg_parts[struct_handler_name] = t + post_lines.append( + f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'])" + ) + else: + tn = f"__c_type_{an}" + internal_arg_parts[tn] = t + post_lines.append( + f" res['{a.alias}'] = {struct_handler_name}(o['{kn}'], {tn})" + ) + else: + post_lines.append(f" res['{a.alias}'] = o['{kn}']") + if not pi_lines: + instantiation_lines = ( + [" return __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + ) + else: + instantiation_lines = ( + [" instance = __cl("] + + [f" {line}" for line in invocation_lines] + + [" )"] + ) + pi_lines.append(" return instance") + + if _cattrs_forbid_extra_keys: + post_lines += [ + " unknown_fields = set(o.keys()) - __c_a", + " if unknown_fields:", + " raise __c_feke('', __cl, unknown_fields)", + ] + finally: + working_set.remove(cl) + if not working_set: + del already_generating.working_set # At the end, we create the function header. internal_arg_line = ", ".join([f"{i}={i}" for i in internal_arg_parts]) diff --git a/src/cattrs/gen/_shared.py b/src/cattrs/gen/_shared.py index 5a9e3aa7..4e631437 100644 --- a/src/cattrs/gen/_shared.py +++ b/src/cattrs/gen/_shared.py @@ -19,34 +19,40 @@ def find_structure_handler( Return `None` if no handler should be used. """ - if a.converter is not None and prefer_attrs_converters: - # If the user as requested to use attrib converters, use nothing - # so it falls back to that. - handler = None - elif a.converter is not None and not prefer_attrs_converters and type is not None: - handler = c.get_structure_hook(type, cache_result=False) - if handler == raise_error: + try: + if a.converter is not None and prefer_attrs_converters: + # If the user as requested to use attrib converters, use nothing + # so it falls back to that. handler = None - elif type is not None: - if ( - is_bare_final(type) - and a.default is not NOTHING - and not isinstance(a.default, Factory) + elif ( + a.converter is not None and not prefer_attrs_converters and type is not None ): - # This is a special case where we can use the - # type of the default to dispatch on. - type = a.default.__class__ handler = c.get_structure_hook(type, cache_result=False) - if handler == c._structure_call: - # Finals can't really be used with _structure_call, so - # we wrap it so the rest of the toolchain doesn't get - # confused. - - def handler(v, _, _h=handler): - return _h(v, type) - + if handler == raise_error: + handler = None + elif type is not None: + if ( + is_bare_final(type) + and a.default is not NOTHING + and not isinstance(a.default, Factory) + ): + # This is a special case where we can use the + # type of the default to dispatch on. + type = a.default.__class__ + handler = c.get_structure_hook(type, cache_result=False) + if handler == c._structure_call: + # Finals can't really be used with _structure_call, so + # we wrap it so the rest of the toolchain doesn't get + # confused. + + def handler(v, _, _h=handler): + return _h(v, type) + + else: + handler = c.get_structure_hook(type, cache_result=False) else: - handler = c.get_structure_hook(type, cache_result=False) - else: - handler = c.structure - return handler + handler = c.structure + return handler + except RecursionError: + # This means we're dealing with a reference cycle, so use late binding. + return c.structure diff --git a/src/cattrs/preconf/msgspec.py b/src/cattrs/preconf/msgspec.py index e58cbec8..6ef84d76 100644 --- a/src/cattrs/preconf/msgspec.py +++ b/src/cattrs/preconf/msgspec.py @@ -13,7 +13,7 @@ from msgspec import Struct, convert, to_builtins from msgspec.json import Encoder, decode -from cattrs._compat import ( +from .._compat import ( fields, get_args, get_origin, @@ -22,13 +22,12 @@ is_mapping, is_sequence, ) -from cattrs.dispatch import UnstructureHook -from cattrs.fns import identity - +from ..cols import is_namedtuple from ..converters import BaseConverter, Converter +from ..dispatch import UnstructureHook +from ..fns import identity from ..gen import make_hetero_tuple_unstructure_fn from ..strategies import configure_union_passthrough -from ..tuples import is_namedtuple from . import wrap T = TypeVar("T") diff --git a/src/cattrs/preconf/orjson.py b/src/cattrs/preconf/orjson.py index 60e46287..4b595bcf 100644 --- a/src/cattrs/preconf/orjson.py +++ b/src/cattrs/preconf/orjson.py @@ -9,10 +9,10 @@ from orjson import dumps, loads from .._compat import AbstractSet, is_mapping +from ..cols import is_namedtuple, namedtuple_unstructure_factory from ..converters import BaseConverter, Converter from ..fns import identity from ..strategies import configure_union_passthrough -from ..tuples import is_namedtuple, namedtuple_unstructure_factory from . import wrap T = TypeVar("T") diff --git a/src/cattrs/preconf/pyyaml.py b/src/cattrs/preconf/pyyaml.py index 45bc828a..73746257 100644 --- a/src/cattrs/preconf/pyyaml.py +++ b/src/cattrs/preconf/pyyaml.py @@ -7,9 +7,9 @@ from yaml import safe_dump, safe_load from .._compat import FrozenSetSubscriptable +from ..cols import is_namedtuple, namedtuple_unstructure_factory from ..converters import BaseConverter, Converter from ..strategies import configure_union_passthrough -from ..tuples import is_namedtuple, namedtuple_unstructure_factory from . import validate_datetime, wrap T = TypeVar("T") diff --git a/tests/test_generics.py b/tests/test_generics.py index d0898a5e..f5b6d813 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -323,14 +323,17 @@ class C(Generic[T, U]): def test_nongeneric_protocols(converter): """Non-generic protocols work.""" - class NongenericProtocol(Protocol): ... + class NongenericProtocol(Protocol): + ... @define - class Entity(NongenericProtocol): ... + class Entity(NongenericProtocol): + ... assert generate_mapping(Entity) == {} - class GenericProtocol(Protocol[T]): ... + class GenericProtocol(Protocol[T]): + ... @define class GenericEntity(GenericProtocol[int]): diff --git a/tests/test_tuples.py b/tests/test_tuples.py index 91c35a50..3b63af81 100644 --- a/tests/test_tuples.py +++ b/tests/test_tuples.py @@ -2,8 +2,8 @@ from typing import NamedTuple, Tuple +from cattrs.cols import is_namedtuple from cattrs.converters import Converter -from cattrs.tuples import is_namedtuple def test_simple_hetero_tuples(genconverter: Converter): From 671790449ed3a004a94a993473725177325a8da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sun, 26 May 2024 23:45:26 +0200 Subject: [PATCH 02/10] Black reformat --- src/cattrs/cols.py | 1 + src/cattrs/converters.py | 36 ++++++++++++------------------------ src/cattrs/dispatch.py | 6 +++--- tests/test_generics.py | 9 +++------ 4 files changed, 19 insertions(+), 33 deletions(-) diff --git a/src/cattrs/cols.py b/src/cattrs/cols.py index 4918ea5c..c4c77ddc 100644 --- a/src/cattrs/cols.py +++ b/src/cattrs/cols.py @@ -1,4 +1,5 @@ """Utility functions for collections.""" + from __future__ import annotations from sys import version_info diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index dec49c62..35a9ba59 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -288,12 +288,10 @@ def unstruct_strat(self) -> UnstructureStrategy: ) @overload - def register_unstructure_hook(self) -> Callable[[UnstructureHook], None]: - ... + def register_unstructure_hook(self) -> Callable[[UnstructureHook], None]: ... @overload - def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: - ... + def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: ... def register_unstructure_hook( self, cls: Any = None, func: UnstructureHook | None = None @@ -341,26 +339,22 @@ def register_unstructure_hook_func( @overload def register_unstructure_hook_factory( self, predicate: Predicate - ) -> Callable[[UnstructureHookFactory], UnstructureHookFactory]: - ... + ) -> Callable[[UnstructureHookFactory], UnstructureHookFactory]: ... @overload def register_unstructure_hook_factory( self, predicate: Predicate - ) -> Callable[[ExtendedUnstructureHookFactory], ExtendedUnstructureHookFactory]: - ... + ) -> Callable[[ExtendedUnstructureHookFactory], ExtendedUnstructureHookFactory]: ... @overload def register_unstructure_hook_factory( self, predicate: Predicate, factory: UnstructureHookFactory - ) -> UnstructureHookFactory: - ... + ) -> UnstructureHookFactory: ... @overload def register_unstructure_hook_factory( self, predicate: Predicate, factory: ExtendedUnstructureHookFactory - ) -> ExtendedUnstructureHookFactory: - ... + ) -> ExtendedUnstructureHookFactory: ... def register_unstructure_hook_factory(self, predicate, factory=None): """ @@ -429,12 +423,10 @@ def get_unstructure_hook( ) @overload - def register_structure_hook(self) -> Callable[[StructureHook], None]: - ... + def register_structure_hook(self) -> Callable[[StructureHook], None]: ... @overload - def register_structure_hook(self, cl: Any, func: StructuredValue) -> None: - ... + def register_structure_hook(self, cl: Any, func: StructuredValue) -> None: ... def register_structure_hook( self, cl: Any, func: StructureHook | None = None @@ -484,26 +476,22 @@ def register_structure_hook_func( @overload def register_structure_hook_factory( self, predicate: Predicate - ) -> Callable[[StructureHookFactory, StructureHookFactory]]: - ... + ) -> Callable[[StructureHookFactory, StructureHookFactory]]: ... @overload def register_structure_hook_factory( self, predicate: Predicate - ) -> Callable[[ExtendedStructureHookFactory, ExtendedStructureHookFactory]]: - ... + ) -> Callable[[ExtendedStructureHookFactory, ExtendedStructureHookFactory]]: ... @overload def register_structure_hook_factory( self, predicate: Predicate, factory: StructureHookFactory - ) -> StructureHookFactory: - ... + ) -> StructureHookFactory: ... @overload def register_structure_hook_factory( self, predicate: Predicate, factory: ExtendedStructureHookFactory - ) -> ExtendedStructureHookFactory: - ... + ) -> ExtendedStructureHookFactory: ... def register_structure_hook_factory(self, predicate, factory=None): """ diff --git a/src/cattrs/dispatch.py b/src/cattrs/dispatch.py index f82ae878..3d746dbc 100644 --- a/src/cattrs/dispatch.py +++ b/src/cattrs/dispatch.py @@ -44,9 +44,9 @@ class FunctionDispatch: """ _converter: BaseConverter - _handler_pairs: list[ - tuple[Predicate, Callable[[Any, Any], Any], bool, bool] - ] = Factory(list) + _handler_pairs: list[tuple[Predicate, Callable[[Any, Any], Any], bool, bool]] = ( + Factory(list) + ) def register( self, diff --git a/tests/test_generics.py b/tests/test_generics.py index f5b6d813..d0898a5e 100644 --- a/tests/test_generics.py +++ b/tests/test_generics.py @@ -323,17 +323,14 @@ class C(Generic[T, U]): def test_nongeneric_protocols(converter): """Non-generic protocols work.""" - class NongenericProtocol(Protocol): - ... + class NongenericProtocol(Protocol): ... @define - class Entity(NongenericProtocol): - ... + class Entity(NongenericProtocol): ... assert generate_mapping(Entity) == {} - class GenericProtocol(Protocol[T]): - ... + class GenericProtocol(Protocol[T]): ... @define class GenericEntity(GenericProtocol[int]): From 41f2132ff81daf5c60629c71f72b84889eeb2e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Mon, 27 May 2024 00:29:46 +0200 Subject: [PATCH 03/10] More docs --- docs/basics.md | 40 +++++++++++++------------------ docs/cattrs.rst | 8 +++++++ docs/conf.py | 0 docs/customizing.md | 55 +++++++++++++++++++++++++++++++++++++++++++ docs/indepth.md | 4 ++++ docs/index.md | 1 + src/cattrs/_compat.py | 5 ++++ src/cattrs/cols.py | 6 +++-- 8 files changed, 93 insertions(+), 26 deletions(-) mode change 100755 => 100644 docs/conf.py diff --git a/docs/basics.md b/docs/basics.md index f40559ab..87ef7083 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -14,7 +14,7 @@ To create a private converter, instantiate a {class}`cattrs.Converter`. Converte The two main methods, {meth}`structure ` and {meth}`unstructure `, are used to convert between _structured_ and _unstructured_ data. -```python +```{doctest} basics >>> from cattrs import structure, unstructure >>> from attrs import define @@ -23,7 +23,7 @@ The two main methods, {meth}`structure ` and {me ... a: int >>> unstructure(Model(1)) -{"a": 1} +{'a': 1} >>> structure({"a": 1}, Model) Model(a=1) ``` @@ -31,32 +31,31 @@ Model(a=1) _cattrs_ comes with a rich library of un/structuring hooks by default but it excels at composing custom hooks with built-in ones. The simplest approach to customization is writing a new hook from scratch. -For example, we can write our own hook for the `int` class. +For example, we can write our own hook for the `int` class and register it to a converter. -```python ->>> def int_hook(value, type): +```{doctest} basics +>>> from cattrs import Converter + +>>> converter = Converter() + +>>> @converter.register_structure_hook +... def int_hook(value, type) -> int: ... if not isinstance(value, int): ... raise ValueError('not an int!') ... return value ``` -We can then register this hook to a converter and any other hook converting an `int` will use it. +Now, any other hook converting an `int` will use it. -```python ->>> from cattrs import Converter - ->>> converter = Converter() ->>> converter.register_structure_hook(int, int_hook) -``` - -Another approach to customization is wrapping an existing hook with your own function. +Another approach to customization is wrapping (composing) an existing hook with your own function. A base hook can be obtained from a converter and then be subjected to the very rich machinery of function composition that Python offers. -```python +```{doctest} basics >>> base_hook = converter.get_structure_hook(Model) ->>> def my_model_hook(value, type): +>>> @converter.register_structure_hook +... def my_model_hook(value, type) -> Model: ... # Apply any preprocessing to the value. ... result = base_hook(value, type) ... # Apply any postprocessing to the model. @@ -65,16 +64,9 @@ A base hook can be obtained from a converter and then be subjected to the very r (`cattrs.structure({}, Model)` is equivalent to `cattrs.get_structure_hook(Model)({}, Model)`.) -This new hook can be used directly or registered to a converter (the original instance, or a different one): - -```python ->>> converter.register_structure_hook(Model, my_model_hook) -``` - - Now if we use this hook to structure a `Model`, through ✨the magic of function composition✨ that hook will use our old `int_hook`. -```python +```{python} >>> converter.structure({"a": "1"}, Model) + Exception Group Traceback (most recent call last): | File "...", line 22, in diff --git a/docs/cattrs.rst b/docs/cattrs.rst index 5170d264..13bd1f0a 100644 --- a/docs/cattrs.rst +++ b/docs/cattrs.rst @@ -19,6 +19,14 @@ Subpackages Submodules ---------- +cattrs.cols module +------------------ + +.. automodule:: cattrs.cols + :members: + :undoc-members: + :show-inheritance: + cattrs.disambiguators module ---------------------------- diff --git a/docs/conf.py b/docs/conf.py old mode 100755 new mode 100644 diff --git a/docs/customizing.md b/docs/customizing.md index a1f009c6..d9703c32 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -155,6 +155,61 @@ Here's an example of using an unstructure hook factory to handle unstructuring [ [1, 2] ``` +## Customizing Collections + +The {mod}`cattrs.cols` module contains predicates and hook factories useful for customizing collection handling. +These hook factories can be wrapped to apply complex customizations. + +Available predicates are: + +* {meth}`is_sequence ` +* {meth}`is_namedtuple ` + +Available hook factories are: + +* {meth}`list_structure_factory ` +* {meth}`namedtuple_structure_factory ` +* {meth}`namedtuple_unstructure_factory ` + +Additional predicates and hook factories will be added as requested. + +For example, by default sequences are structured from any iterable into lists. +This may be too lax, and additional validation may be applied by wrapping the default list structuring hook factory. + +```{testcode} list-customization +from cattrs.cols import is_sequence, list_structure_factory + +c = Converter() + +@c.register_structure_hook_factory(is_sequence) +def strict_list_hook_factory(type, converter): + + # First, we generate the default hook... + list_hook = list_structure_factory(type, converter) + + # Then, we wrap it with a function of our own... + def strict_list_hook(value, type): + if not isinstance(value, list): + raise ValueError("Not a list!") + return list_hook(value, type) + + # And finally, we return our own composite hook. + return strict_list_hook +``` + +Now, all sequence structuring will be stricter: + +```{doctest} list-customization +>>> c.structure({"a", "b", "c"}, list[str]) +Traceback (most recent call last): + ... +ValueError: Not a list! +``` + +```{versionadded} 24.1.0 + +``` + ## Using `cattrs.gen` Generators The {mod}`cattrs.gen` module allows for generating and compiling specialized hooks for unstructuring _attrs_ classes, dataclasses and typed dicts. diff --git a/docs/indepth.md b/docs/indepth.md index 94048349..a0700af4 100644 --- a/docs/indepth.md +++ b/docs/indepth.md @@ -23,6 +23,10 @@ The new copy may be changed through the `copy` arguments, but will retain all ma This feature is supported for Python 3.9 and later. ``` +```{tip} +See [](customizing.md#customizing-collections) for a more modern and more powerful way of customizing collection handling. +``` + Overriding collection unstructuring in a generic way can be a very useful feature. A common example is using a JSON library that doesn't support sets, but expects lists and tuples instead. diff --git a/docs/index.md b/docs/index.md index 323ef2d0..24cd50d4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,6 +37,7 @@ caption: Dev Guide history benchmarking contributing +modindex ``` ```{include} ../README.md diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index bad9d037..f4861a3e 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -332,6 +332,11 @@ def get_notrequired_base(type) -> "Union[Any, Literal[NOTHING]]": return NOTHING def is_sequence(type: Any) -> bool: + """A predicate function for sequences. + + Matches lists, sequences, mutable sequences, deques and homogenous + tuples. + """ origin = getattr(type, "__origin__", None) return ( type diff --git a/src/cattrs/cols.py b/src/cattrs/cols.py index c4c77ddc..df792230 100644 --- a/src/cattrs/cols.py +++ b/src/cattrs/cols.py @@ -5,7 +5,7 @@ from sys import version_info from typing import TYPE_CHECKING, Any, Iterable, NamedTuple, Tuple, TypeVar -from ._compat import ANIES, is_bare, is_subclass +from ._compat import ANIES, is_bare, is_sequence, is_subclass from .dispatch import StructureHook, UnstructureHook from .errors import IterableValidationError, IterableValidationNote from .fns import identity @@ -16,8 +16,10 @@ __all__ = [ "is_namedtuple", - "namedtuple_unstructure_factory", + "is_sequence", + "list_structure_factory", "namedtuple_structure_factory", + "namedtuple_unstructure_factory", ] if version_info[:2] >= (3, 9): From 1cbd36b70378db295f75a86f64c24dc154de43a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Mon, 27 May 2024 00:32:42 +0200 Subject: [PATCH 04/10] Changelog --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 1728f61e..991324eb 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -27,6 +27,8 @@ Our backwards-compatibility policy can be found [here](https://github.com/python can now be used as decorators and have gained new features. See [here](https://catt.rs/en/latest/customizing.html#use-as-decorators) and [here](https://catt.rs/en/latest/customizing.html#id1) for more details. ([#487](https://github.com/python-attrs/cattrs/pull/487)) +- Introduce and [document](https://catt.rs/en/latest/customizing.html#customizing-collections) the {mod}`cattrs.cols` module for better collection customizations. + ([#504](https://github.com/python-attrs/cattrs/issues/504) [#540][https://github.com/python-attrs/cattrs/pull/540]) - Introduce the [_msgspec_](https://jcristharif.com/msgspec/) {mod}`preconf converter `. Only JSON is supported for now, with other formats supported by _msgspec_ to come later. ([#481](https://github.com/python-attrs/cattrs/pull/481)) From 358454441fb4a8fe636ea20e71b05ac5920b0199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 1 Jun 2024 18:19:07 +0200 Subject: [PATCH 05/10] More sets for cols --- docs/basics.md | 2 +- docs/customizing.md | 4 ++++ src/cattrs/_compat.py | 12 ++++++++++-- src/cattrs/cols.py | 40 +++++++++++++++++++++++++++++++++++++++- tests/test_cols.py | 21 +++++++++++++++++++++ 5 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 tests/test_cols.py diff --git a/docs/basics.md b/docs/basics.md index 87ef7083..8465dfc7 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -66,7 +66,7 @@ A base hook can be obtained from a converter and then be subjected to the very r Now if we use this hook to structure a `Model`, through ✨the magic of function composition✨ that hook will use our old `int_hook`. -```{python} +```python >>> converter.structure({"a": "1"}, Model) + Exception Group Traceback (most recent call last): | File "...", line 22, in diff --git a/docs/customizing.md b/docs/customizing.md index d9703c32..c219e0ff 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -162,11 +162,15 @@ These hook factories can be wrapped to apply complex customizations. Available predicates are: +* {meth}`is_any_set ` +* {meth}`is_frozenset ` +* {meth}`is_set ` * {meth}`is_sequence ` * {meth}`is_namedtuple ` Available hook factories are: +* {meth}`iterable_unstructure_factory ` * {meth}`list_structure_factory ` * {meth}`namedtuple_structure_factory ` * {meth}`namedtuple_unstructure_factory ` diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index f4861a3e..575a1d45 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -371,7 +371,11 @@ def is_deque(type): or (getattr(type, "__origin__", None) is deque) ) - def is_mutable_set(type): + def is_mutable_set(type: Any) -> bool: + """A predicate function for (mutable) sets. + + Matches built-in sets and sets from the typing module. + """ return ( type in (TypingSet, TypingMutableSet, set) or ( @@ -381,7 +385,11 @@ def is_mutable_set(type): or (getattr(type, "__origin__", None) in (set, AbcMutableSet, AbcSet)) ) - def is_frozenset(type): + def is_frozenset(type: Any) -> bool: + """A predicate function for frozensets. + + Matches built-in frozensets and frozensets from the typing module. + """ return ( type in (FrozenSet, frozenset) or ( diff --git a/src/cattrs/cols.py b/src/cattrs/cols.py index df792230..c8d093ea 100644 --- a/src/cattrs/cols.py +++ b/src/cattrs/cols.py @@ -5,7 +5,8 @@ from sys import version_info from typing import TYPE_CHECKING, Any, Iterable, NamedTuple, Tuple, TypeVar -from ._compat import ANIES, is_bare, is_sequence, is_subclass +from ._compat import ANIES, is_bare, is_frozenset, is_sequence, is_subclass +from ._compat import is_mutable_set as is_set from .dispatch import StructureHook, UnstructureHook from .errors import IterableValidationError, IterableValidationNote from .fns import identity @@ -15,13 +16,23 @@ from .converters import BaseConverter __all__ = [ + "is_any_set", + "is_frozenset", "is_namedtuple", + "is_set", "is_sequence", + "iterable_unstructure_factory", "list_structure_factory", "namedtuple_structure_factory", "namedtuple_unstructure_factory", ] + +def is_any_set(type) -> bool: + """A predicate function for both mutable and frozensets.""" + return is_set(type) or is_frozenset(type) + + if version_info[:2] >= (3, 9): def is_namedtuple(type: Any) -> bool: @@ -122,6 +133,33 @@ def structure_list( return structure_list +def iterable_unstructure_factory( + cl: Any, converter: BaseConverter, unstructure_to: Any = None +) -> UnstructureHook: + """A hook factory for unstructuring iterables. + + :param unstructure_to: Force unstructuring to this type, if provided. + """ + handler = converter.unstructure + + # Let's try fishing out the type args + # Unspecified tuples have `__args__` as empty tuples, so guard + # against IndexError. + if getattr(cl, "__args__", None) not in (None, ()): + type_arg = cl.__args__[0] + if isinstance(type_arg, TypeVar): + type_arg = getattr(type_arg, "__default__", Any) + handler = converter.get_unstructure_hook(type_arg, cache_result=False) + if handler == identity: + # Save ourselves the trouble of iterating over it all. + return unstructure_to or cl + + def unstructure_iterable(iterable, _seq_cl=unstructure_to or cl, _hook=handler): + return _seq_cl(_hook(i) for i in iterable) + + return unstructure_iterable + + def namedtuple_unstructure_factory( type: type[tuple], converter: BaseConverter, unstructure_to: Any = None ) -> UnstructureHook: diff --git a/tests/test_cols.py b/tests/test_cols.py new file mode 100644 index 00000000..5c596011 --- /dev/null +++ b/tests/test_cols.py @@ -0,0 +1,21 @@ +"""Tests for the `cattrs.cols` module.""" + +from cattrs import BaseConverter +from cattrs._compat import AbstractSet, FrozenSet +from cattrs.cols import is_any_set, iterable_unstructure_factory + + +def test_set_overriding(converter: BaseConverter): + """Overriding abstract sets by wrapping the default factory works.""" + + converter.register_unstructure_hook_factory( + is_any_set, + lambda t, c: iterable_unstructure_factory(t, c, unstructure_to=sorted), + ) + + assert converter.unstructure({"c", "b", "a"}, AbstractSet[str]) == ["a", "b", "c"] + assert converter.unstructure(frozenset(["c", "b", "a"]), FrozenSet[str]) == [ + "a", + "b", + "c", + ] From 4c87a1a343b55c46c0dd6f284e771bdf9d1661c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 1 Jun 2024 18:24:46 +0200 Subject: [PATCH 06/10] More history --- HISTORY.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 991324eb..ef965429 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -28,7 +28,7 @@ can now be used as decorators and have gained new features. See [here](https://catt.rs/en/latest/customizing.html#use-as-decorators) and [here](https://catt.rs/en/latest/customizing.html#id1) for more details. ([#487](https://github.com/python-attrs/cattrs/pull/487)) - Introduce and [document](https://catt.rs/en/latest/customizing.html#customizing-collections) the {mod}`cattrs.cols` module for better collection customizations. - ([#504](https://github.com/python-attrs/cattrs/issues/504) [#540][https://github.com/python-attrs/cattrs/pull/540]) + ([#504](https://github.com/python-attrs/cattrs/issues/504) [#540](https://github.com/python-attrs/cattrs/pull/540)) - Introduce the [_msgspec_](https://jcristharif.com/msgspec/) {mod}`preconf converter `. Only JSON is supported for now, with other formats supported by _msgspec_ to come later. ([#481](https://github.com/python-attrs/cattrs/pull/481)) @@ -50,6 +50,8 @@ can now be used as decorators and have gained new features. ([#463](https://github.com/python-attrs/cattrs/pull/463)) - `cattrs.gen` generators now attach metadata to the generated functions, making them introspectable. ([#472](https://github.com/python-attrs/cattrs/pull/472)) +- Structure hook factories in {mod}`cattrs.gen` now handle recursive classes better. + ([#540](https://github.com/python-attrs/cattrs/pull/540)) - The [tagged union strategy](https://catt.rs/en/stable/strategies.html#tagged-unions-strategy) now leaves the tags in the payload unless `forbid_extra_keys` is set. ([#533](https://github.com/python-attrs/cattrs/issues/533) [#534](https://github.com/python-attrs/cattrs/pull/534)) - More robust support for `Annotated` and `NotRequired` in TypedDicts. From 17a8fd85bafb89fd801e14e1b89c49fc3ed9b83a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 1 Jun 2024 18:35:21 +0200 Subject: [PATCH 07/10] Add test for better recursive structuring --- tests/strategies/test_include_subclasses.py | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/strategies/test_include_subclasses.py b/tests/strategies/test_include_subclasses.py index 29a61281..4ddf61b2 100644 --- a/tests/strategies/test_include_subclasses.py +++ b/tests/strategies/test_include_subclasses.py @@ -343,3 +343,26 @@ class A: include_subclasses(A, genconverter, union_strategy=configure_tagged_union) assert genconverter.structure({"a": 1}, A) == A(1) + + +def test_cyclic_classes(genconverter: Converter): + """A cyclic reference case from issue #542.""" + + @define + class Base: + pass + + @define + class Subclass1(Base): + b: str + a: Base + + @define + class Subclass2(Base): + b: str + + include_subclasses(Base, genconverter, union_strategy=configure_tagged_union) + + assert genconverter.structure( + {"b": "a", "_type": "Subclass1", "a": {"b": "c", "_type": "Subclass2"}}, Base + ) == Subclass1("a", Subclass2("c")) From a2d6b0f048f57648fbc285c8869c9daca9b2a2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 1 Jun 2024 18:43:00 +0200 Subject: [PATCH 08/10] Improve set handling on 3.8 --- src/cattrs/_compat.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index 575a1d45..0eda9947 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -504,9 +504,10 @@ def is_deque(type: Any) -> bool: or type.__origin__ is deque ) - def is_mutable_set(type): - return type is set or ( - type.__class__ is _GenericAlias and is_subclass(type.__origin__, MutableSet) + def is_mutable_set(type) -> bool: + return type in (set, TypingAbstractSet) or ( + type.__class__ is _GenericAlias + and is_subclass(type.__origin__, (MutableSet, TypingAbstractSet)) ) def is_frozenset(type): From aeeb120c04a35b06e437b35d69836bd9f2f5b407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 1 Jun 2024 18:53:24 +0200 Subject: [PATCH 09/10] Docs --- docs/customizing.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/customizing.md b/docs/customizing.md index c219e0ff..07802b83 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -168,6 +168,17 @@ Available predicates are: * {meth}`is_sequence ` * {meth}`is_namedtuple ` +````{tip} +These predicates aren't _cattrs_-specific and may be useful in other contexts. +```{doctest} predicates +>>> from cattrs.cols import is_sequence + +>>> is_sequence(list[str]) +True +``` +```` + + Available hook factories are: * {meth}`iterable_unstructure_factory ` From 5b5b773d06386c157ec480a4b25308f0de046531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 1 Jun 2024 18:58:38 +0200 Subject: [PATCH 10/10] Docs --- HISTORY.md | 2 +- docs/strategies.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index ef965429..518b5734 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -48,7 +48,7 @@ can now be used as decorators and have gained new features. ([#481](https://github.com/python-attrs/cattrs/pull/481)) - The {class}`orjson preconf converter ` now passes through dates and datetimes to orjson while unstructuring, greatly improving speed. ([#463](https://github.com/python-attrs/cattrs/pull/463)) -- `cattrs.gen` generators now attach metadata to the generated functions, making them introspectable. +- {mod}`cattrs.gen` generators now attach metadata to the generated functions, making them introspectable. ([#472](https://github.com/python-attrs/cattrs/pull/472)) - Structure hook factories in {mod}`cattrs.gen` now handle recursive classes better. ([#540](https://github.com/python-attrs/cattrs/pull/540)) diff --git a/docs/strategies.md b/docs/strategies.md index 4a9540a0..e4ba639a 100644 --- a/docs/strategies.md +++ b/docs/strategies.md @@ -376,6 +376,7 @@ This strategy has been preapplied to the following preconfigured converters: - {py:class}`Cbor2Converter ` - {py:class}`JsonConverter ` - {py:class}`MsgpackConverter ` +- {py:class}`MsgspecJsonConverter ` - {py:class}`OrjsonConverter ` - {py:class}`PyyamlConverter ` - {py:class}`TomlkitConverter `