From 499735de3890f020070268295c4d81346905afc4 Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Mon, 13 Nov 2023 19:13:56 +0100 Subject: [PATCH 1/3] Un/structure fallback factories --- src/cattrs/converters.py | 69 ++++++------ src/cattrs/dispatch.py | 156 +++++++++++++++------------ src/cattrs/fns.py | 17 +++ src/cattrs/gen/__init__.py | 9 +- src/cattrs/gen/_shared.py | 5 +- src/cattrs/gen/typeddicts.py | 7 +- src/cattrs/preconf/orjson.py | 3 +- tests/test_converter.py | 38 ++++++- tests/test_multistrategy_dispatch.py | 6 +- tests/test_structure_attrs.py | 16 +-- 10 files changed, 204 insertions(+), 122 deletions(-) create mode 100644 src/cattrs/fns.py diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 22f88271..9f8c4f29 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -11,7 +11,6 @@ Dict, Iterable, List, - NoReturn, Optional, Tuple, Type, @@ -56,12 +55,13 @@ is_union_type, ) from .disambiguators import create_default_dis_func, is_supported_union -from .dispatch import MultiStrategyDispatch +from .dispatch import HookFactory, MultiStrategyDispatch, StructureHook, UnstructureHook from .errors import ( IterableValidationError, IterableValidationNote, StructureHandlerNotFoundError, ) +from .fns import identity, raise_error from .gen import ( AttributeOverride, DictStructureFn, @@ -79,6 +79,8 @@ from .gen.typeddicts import make_dict_structure_fn as make_typeddict_dict_struct_fn from .gen.typeddicts import make_dict_unstructure_fn as make_typeddict_dict_unstruct_fn +__all__ = ["UnstructureStrategy", "BaseConverter", "Converter", "GenConverter"] + NoneType = type(None) T = TypeVar("T") V = TypeVar("V") @@ -127,7 +129,18 @@ def __init__( unstruct_strat: UnstructureStrategy = UnstructureStrategy.AS_DICT, prefer_attrib_converters: bool = False, detailed_validation: bool = True, + unstructure_fallback_factory: HookFactory[UnstructureHook] = lambda _: identity, + structure_fallback_factory: HookFactory[StructureHook] = lambda _: raise_error, ) -> None: + """ + :param unstructure_fallback_factory: A hook factory to be called when no + registered unstructuring hooks match. + :param structure_fallback_factory: A hook factory to be called when no + registered structuring hooks match. + + .. versionadded:: 23.2.0 *unstructure_fallback_factory* + .. versionadded:: 23.2.0 *structure_fallback_factory* + """ unstruct_strat = UnstructureStrategy(unstruct_strat) self._prefer_attrib_converters = prefer_attrib_converters @@ -143,13 +156,9 @@ def __init__( self._dis_func_cache = lru_cache()(self._get_dis_func) - self._unstructure_func = MultiStrategyDispatch(self._unstructure_identity) + self._unstructure_func = MultiStrategyDispatch(unstructure_fallback_factory) self._unstructure_func.register_cls_list( - [ - (bytes, self._unstructure_identity), - (str, self._unstructure_identity), - (Path, str), - ] + [(bytes, identity), (str, identity), (Path, str)] ) self._unstructure_func.register_func_list( [ @@ -175,7 +184,7 @@ def __init__( # Per-instance register of to-attrs converters. # Singledispatch dispatches based on the first argument, so we # store the function and switch the arguments in self.loads. - self._structure_func = MultiStrategyDispatch(BaseConverter._structure_error) + self._structure_func = MultiStrategyDispatch(structure_fallback_factory) self._structure_func.register_func_list( [ (lambda cl: cl is Any or cl is Optional or cl is None, lambda v, _: v), @@ -237,7 +246,7 @@ def unstruct_strat(self) -> UnstructureStrategy: else UnstructureStrategy.AS_TUPLE ) - def register_unstructure_hook(self, cls: Any, func: Callable[[Any], Any]) -> None: + def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: """Register a class-to-primitive converter function for a class. The converter function should take an instance of the class and return @@ -254,7 +263,7 @@ def register_unstructure_hook(self, cls: Any, func: Callable[[Any], Any]) -> Non self._unstructure_func.register_cls_list([(cls, func)]) def register_unstructure_hook_func( - self, check_func: Callable[[Any], bool], func: Callable[[Any], Any] + self, check_func: Callable[[Any], bool], func: UnstructureHook ) -> None: """Register a class-to-primitive converter function for a class, using a function to check if it's a match. @@ -262,9 +271,7 @@ def register_unstructure_hook_func( self._unstructure_func.register_func_list([(check_func, func)]) def register_unstructure_hook_factory( - self, - predicate: Callable[[Any], bool], - factory: Callable[[Any], Callable[[Any], Any]], + self, predicate: Callable[[Any], bool], factory: HookFactory[UnstructureHook] ) -> None: """ Register a hook factory for a given predicate. @@ -276,9 +283,7 @@ def register_unstructure_hook_factory( """ self._unstructure_func.register_func_list([(predicate, factory, True)]) - def register_structure_hook( - self, cl: Any, func: Callable[[Any, Type[T]], T] - ) -> None: + def register_structure_hook(self, cl: Any, func: StructureHook) -> None: """Register a primitive-to-class converter function for a type. The converter function should take two arguments: @@ -300,7 +305,7 @@ def register_structure_hook( self._structure_func.register_cls_list([(cl, func)]) def register_structure_hook_func( - self, check_func: Callable[[Type[T]], bool], func: Callable[[Any, Type[T]], T] + self, check_func: Callable[[Type[T]], bool], func: StructureHook ) -> None: """Register a class-to-primitive converter function for a class, using a function to check if it's a match. @@ -308,9 +313,7 @@ def register_structure_hook_func( self._structure_func.register_func_list([(check_func, func)]) def register_structure_hook_factory( - self, - predicate: Callable[[Any], bool], - factory: Callable[[Any], Callable[[Any, Any], Any]], + self, predicate: Callable[[Any], bool], factory: HookFactory[StructureHook] ) -> None: """ Register a hook factory for a given predicate. @@ -353,11 +356,6 @@ def _unstructure_enum(self, obj: Enum) -> Any: """Convert an enum to its value.""" return obj.value - @staticmethod - def _unstructure_identity(obj: T) -> T: - """Just pass it through.""" - return obj - def _unstructure_seq(self, seq: Sequence[T]) -> Sequence[T]: """Convert a sequence to primitive equivalents.""" # We can reuse the sequence class, so tuples stay tuples. @@ -388,12 +386,6 @@ def _unstructure_union(self, obj: Any) -> Any: # Python primitives to classes. - @staticmethod - def _structure_error(_, cl: Type) -> NoReturn: - """At the bottom of the condition stack, we explode if we can't handle it.""" - msg = f"Unsupported type: {cl!r}. Register a structure hook for it." - raise StructureHandlerNotFoundError(msg, type_=cl) - def _gen_structure_generic(self, cl: Type[T]) -> DictStructureFn[T]: """Create and return a hook for structuring generics.""" return make_dict_structure_fn( @@ -786,12 +778,25 @@ def __init__( unstruct_collection_overrides: Mapping[Type, Callable] = {}, prefer_attrib_converters: bool = False, detailed_validation: bool = True, + unstructure_fallback_factory: HookFactory[UnstructureHook] = lambda _: identity, + structure_fallback_factory: HookFactory[StructureHook] = lambda _: raise_error, ): + """ + :param unstructure_fallback_factory: A hook factory to be called when no + registered unstructuring hooks match. + :param structure_fallback_factory: A hook factory to be called when no + registered structuring hooks match. + + .. versionadded:: 23.2.0 *unstructure_fallback_factory* + .. versionadded:: 23.2.0 *structure_fallback_factory* + """ super().__init__( dict_factory=dict_factory, unstruct_strat=unstruct_strat, prefer_attrib_converters=prefer_attrib_converters, detailed_validation=detailed_validation, + unstructure_fallback_factory=unstructure_fallback_factory, + structure_fallback_factory=structure_fallback_factory, ) self.omit_if_default = omit_if_default self.forbid_extra_keys = forbid_extra_keys diff --git a/src/cattrs/dispatch.py b/src/cattrs/dispatch.py index 625e0ec7..74502b80 100644 --- a/src/cattrs/dispatch.py +++ b/src/cattrs/dispatch.py @@ -1,7 +1,20 @@ -from functools import lru_cache, singledispatch -from typing import Any, Callable, List, Optional, Tuple, Union +from functools import lru_cache, partial, singledispatch +from typing import Any, Callable, Dict, Generic, List, Optional, Tuple, TypeVar, Union -from attrs import Factory, define +from attrs import Factory, define, field +from typing_extensions import TypeAlias + +T = TypeVar("T") + +TargetType: TypeAlias = Any +UnstructuredValue: TypeAlias = Any +StructuredValue: TypeAlias = Any + +StructureHook: TypeAlias = Callable[[UnstructuredValue, TargetType], StructuredValue] +UnstructureHook: TypeAlias = Callable[[StructuredValue], UnstructuredValue] + +Hook = TypeVar("Hook", StructureHook, UnstructureHook) +HookFactory: TypeAlias = Callable[[TargetType], Hook] @define @@ -9,29 +22,81 @@ class _DispatchNotFound: """A dummy object to help signify a dispatch not found.""" -class MultiStrategyDispatch: +@define +class FunctionDispatch: + """ + FunctionDispatch is similar to functools.singledispatch, but + instead dispatches based on functions that take the type of the + first argument in the method, and return True or False. + + objects that help determine dispatch should be instantiated objects. + """ + + _handler_pairs: List[ + Tuple[Callable[[Any], bool], Callable[[Any, Any], Any], bool] + ] = Factory(list) + + def register( + self, + can_handle: Callable[[Any], bool], + func: Callable[..., Any], + is_generator=False, + ) -> None: + self._handler_pairs.insert(0, (can_handle, func, is_generator)) + + def dispatch(self, typ: Any) -> Optional[Callable[..., Any]]: + """ + Return the appropriate handler for the object passed. + """ + for can_handle, handler, is_generator in self._handler_pairs: + # can handle could raise an exception here + # such as issubclass being called on an instance. + # it's easier to just ignore that case. + try: + ch = can_handle(typ) + except Exception: # noqa: S112 + continue + if ch: + if is_generator: + return handler(typ) + + return handler + return None + + def get_num_fns(self) -> int: + return len(self._handler_pairs) + + def copy_to(self, other: "FunctionDispatch", skip: int = 0) -> None: + other._handler_pairs = self._handler_pairs[:-skip] + other._handler_pairs + + +@define +class MultiStrategyDispatch(Generic[Hook]): """ MultiStrategyDispatch uses a combination of exact-match dispatch, singledispatch, and FunctionDispatch. + + :param fallback_factory: A hook factory to be called when a hook cannot be + produced. + + .. versionchanged:: 23.2.0 + Fallbacks are now factories. """ - __slots__ = ( - "_direct_dispatch", - "_function_dispatch", - "_single_dispatch", - "_generators", - "_fallback_func", - "dispatch", + _fallback_factory: HookFactory[Hook] + _direct_dispatch: Dict = field(init=False, factory=dict) + _function_dispatch: FunctionDispatch = field(init=False, factory=FunctionDispatch) + _single_dispatch: Any = field( + init=False, factory=partial(singledispatch, _DispatchNotFound) + ) + dispatch: Callable[[TargetType], Hook] = field( + init=False, + default=Factory( + lambda self: lru_cache(maxsize=None)(self._dispatch), takes_self=True + ), ) - def __init__(self, fallback_func: Callable[[Any, Any], Any]): - self._direct_dispatch = {} - self._function_dispatch = FunctionDispatch() - self._single_dispatch = singledispatch(_DispatchNotFound) - self.dispatch = lru_cache(maxsize=None)(self._dispatch) - self._fallback_func = fallback_func - - def _dispatch(self, typ: Any) -> Callable[[Any, Any], Any]: + def _dispatch(self, typ: TargetType) -> Hook: try: dispatch = self._single_dispatch.dispatch(typ) if dispatch is not _DispatchNotFound: @@ -44,7 +109,7 @@ def _dispatch(self, typ: Any) -> Callable[[Any, Any], Any]: return direct_dispatch res = self._function_dispatch.dispatch(typ) - return res if res is not None else self._fallback_func + return res if res is not None else self._fallback_factory(typ) def register_cls_list(self, cls_and_handler, direct: bool = False) -> None: """Register a class to direct or singledispatch.""" @@ -79,11 +144,11 @@ def register_func_list( self.clear_direct() self.dispatch.cache_clear() - def clear_direct(self): + def clear_direct(self) -> None: """Clear the direct dispatch.""" self._direct_dispatch.clear() - def clear_cache(self): + def clear_cache(self) -> None: """Clear all caches.""" self._direct_dispatch.clear() self.dispatch.cache_clear() @@ -91,53 +156,8 @@ def clear_cache(self): def get_num_fns(self) -> int: return self._function_dispatch.get_num_fns() - def copy_to(self, other: "MultiStrategyDispatch", skip: int = 0): + def copy_to(self, other: "MultiStrategyDispatch", skip: int = 0) -> None: self._function_dispatch.copy_to(other._function_dispatch, skip=skip) for cls, fn in self._single_dispatch.registry.items(): other._single_dispatch.register(cls, fn) other.clear_cache() - - -@define -class FunctionDispatch: - """ - FunctionDispatch is similar to functools.singledispatch, but - instead dispatches based on functions that take the type of the - first argument in the method, and return True or False. - - objects that help determine dispatch should be instantiated objects. - """ - - _handler_pairs: List[ - Tuple[Callable[[Any], bool], Callable[[Any, Any], Any], bool] - ] = Factory(list) - - def register( - self, can_handle: Callable[[Any], bool], func, is_generator=False - ) -> None: - self._handler_pairs.insert(0, (can_handle, func, is_generator)) - - def dispatch(self, typ: Any) -> Optional[Callable[[Any, Any], Any]]: - """ - Return the appropriate handler for the object passed. - """ - for can_handle, handler, is_generator in self._handler_pairs: - # can handle could raise an exception here - # such as issubclass being called on an instance. - # it's easier to just ignore that case. - try: - ch = can_handle(typ) - except Exception: # noqa: S112 - continue - if ch: - if is_generator: - return handler(typ) - - return handler - return None - - def get_num_fns(self) -> int: - return len(self._handler_pairs) - - def copy_to(self, other: "FunctionDispatch", skip: int = 0) -> None: - other._handler_pairs = self._handler_pairs[:-skip] + other._handler_pairs diff --git a/src/cattrs/fns.py b/src/cattrs/fns.py new file mode 100644 index 00000000..43d0ab0d --- /dev/null +++ b/src/cattrs/fns.py @@ -0,0 +1,17 @@ +"""Useful internal functions.""" +from typing import NoReturn, Type, TypeVar + +from .errors import StructureHandlerNotFoundError + +T = TypeVar("T") + + +def identity(obj: T) -> T: + """The identity function.""" + return obj + + +def raise_error(_, cl: Type) -> NoReturn: + """At the bottom of the condition stack, we explode if we can't handle it.""" + msg = f"Unsupported type: {cl!r}. Register a structure hook for it." + raise StructureHandlerNotFoundError(msg, type_=cl) diff --git a/src/cattrs/gen/__init__.py b/src/cattrs/gen/__init__.py index 3cf0ced2..ec7d01b1 100644 --- a/src/cattrs/gen/__init__.py +++ b/src/cattrs/gen/__init__.py @@ -24,6 +24,7 @@ IterableValidationNote, StructureHandlerNotFoundError, ) +from ..fns import identity from ._consts import AttributeOverride, already_generating, neutral from ._generics import generate_mapping from ._lc import generate_unique_filename @@ -160,7 +161,7 @@ def make_dict_unstructure_fn( else: handler = converter.unstructure - is_identity = handler == converter._unstructure_identity + is_identity = handler == identity if not is_identity: unstruct_handler_name = f"__c_unstr_{attr_name}" @@ -697,7 +698,7 @@ def make_hetero_tuple_unstructure_fn( else: lines.append(" res = (") for i in range(len(handlers)): - if handlers[i] == converter._unstructure_identity: + if handlers[i] == identity: lines.append(f" tup[{i}],") else: lines.append(f" __cattr_u_{i}(tup[{i}]),") @@ -739,11 +740,11 @@ def make_mapping_unstructure_fn( key_arg, val_arg = args, Any # We can do the dispatch here and now. kh = key_handler or converter._unstructure_func.dispatch(key_arg) - if kh == converter._unstructure_identity: + if kh == identity: kh = None val_handler = converter._unstructure_func.dispatch(val_arg) - if val_handler == converter._unstructure_identity: + if val_handler == identity: val_handler = None globs = { diff --git a/src/cattrs/gen/_shared.py b/src/cattrs/gen/_shared.py index 6b96c7f8..bbade22e 100644 --- a/src/cattrs/gen/_shared.py +++ b/src/cattrs/gen/_shared.py @@ -2,9 +2,10 @@ from typing import TYPE_CHECKING, Any, Callable -from attr import NOTHING, Attribute, Factory +from attrs import NOTHING, Attribute, Factory from .._compat import is_bare_final +from ..fns import raise_error if TYPE_CHECKING: # pragma: no cover from cattr.converters import BaseConverter @@ -23,7 +24,7 @@ def find_structure_handler( handler = None elif a.converter is not None and not prefer_attrs_converters and type is not None: handler = c._structure_func.dispatch(type) - if handler == c._structure_error: + if handler == raise_error: handler = None elif type is not None: if ( diff --git a/src/cattrs/gen/typeddicts.py b/src/cattrs/gen/typeddicts.py index dd2f5c85..ed02249d 100644 --- a/src/cattrs/gen/typeddicts.py +++ b/src/cattrs/gen/typeddicts.py @@ -44,6 +44,7 @@ def get_annots(cl) -> dict[str, Any]: ForbiddenExtraKeysError, StructureHandlerNotFoundError, ) +from ..fns import identity from . import AttributeOverride from ._consts import already_generating, neutral from ._generics import generate_mapping @@ -142,12 +143,12 @@ def make_dict_unstructure_fn( except RecursionError: # There's a circular reference somewhere down the line handler = converter.unstructure - is_identity = handler == converter._unstructure_identity + is_identity = handler == identity if not is_identity: break else: # We've not broken the loop. - return converter._unstructure_identity + return identity for ix, a in enumerate(attrs): attr_name = a.name @@ -189,7 +190,7 @@ def make_dict_unstructure_fn( # There's a circular reference somewhere down the line handler = converter.unstructure - is_identity = handler == converter._unstructure_identity + is_identity = handler == identity if not is_identity: unstruct_handler_name = f"__c_unstr_{ix}" diff --git a/src/cattrs/preconf/orjson.py b/src/cattrs/preconf/orjson.py index 5cc74729..664f92b4 100644 --- a/src/cattrs/preconf/orjson.py +++ b/src/cattrs/preconf/orjson.py @@ -9,6 +9,7 @@ from cattrs._compat import AbstractSet, is_mapping from ..converters import BaseConverter, Converter +from ..fns import identity from ..strategies import configure_union_passthrough T = TypeVar("T") @@ -57,7 +58,7 @@ def key_handler(v): # In that case, we want to use the override. kh = converter._unstructure_func.dispatch(args[0]) - if kh != converter._unstructure_identity: + if kh != identity: key_handler = kh return converter.gen_unstructure_mapping( diff --git a/tests/test_converter.py b/tests/test_converter.py index 7fd3129a..ce1259e2 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -9,6 +9,7 @@ Sequence, Set, Tuple, + Type, Union, ) @@ -18,7 +19,11 @@ from hypothesis.strategies import booleans, just, lists, one_of, sampled_from from cattrs import BaseConverter, Converter, UnstructureStrategy -from cattrs.errors import ClassValidationError, ForbiddenExtraKeysError +from cattrs.errors import ( + ClassValidationError, + ForbiddenExtraKeysError, + StructureHandlerNotFoundError, +) from cattrs.gen import make_dict_structure_fn, override from ._compat import is_py39_plus, is_py310_plus @@ -707,3 +712,34 @@ class Outer: structured = converter.structure(raw, Outer) assert structured == Outer(Inner(2), [Inner(2)], Inner(2)) + + +def test_unstructure_fallbacks(converter_cls: Type[BaseConverter]): + """Unstructure fallback factories work.""" + + class Test: + """Unsupported by default.""" + + c = converter_cls() + + assert isinstance(c.unstructure(Test()), Test) + + c = converter_cls( + unstructure_fallback_factory=lambda _: lambda v: v.__class__.__name__ + ) + assert c.unstructure(Test()) == "Test" + + +def test_structure_fallbacks(converter_cls: Type[BaseConverter]): + """Structure fallback factories work.""" + + class Test: + """Unsupported by default.""" + + c = converter_cls() + + with pytest.raises(StructureHandlerNotFoundError): + c.structure({}, Test) + + c = converter_cls(structure_fallback_factory=lambda _: lambda v, _: Test()) + assert isinstance(c.structure({}, Test), Test) diff --git a/tests/test_multistrategy_dispatch.py b/tests/test_multistrategy_dispatch.py index 7d51cd4d..6d9c3499 100644 --- a/tests/test_multistrategy_dispatch.py +++ b/tests/test_multistrategy_dispatch.py @@ -21,14 +21,14 @@ def test_multistrategy_dispatch_register_cls(): _fallback() _foo_func() _foo_cls() - dispatch = MultiStrategyDispatch(_fallback) + dispatch = MultiStrategyDispatch(lambda _: _fallback) assert dispatch.dispatch(Foo) == _fallback dispatch.register_cls_list([(Foo, _foo_cls)]) assert dispatch.dispatch(Foo) == _foo_cls def test_multistrategy_dispatch_register_func(): - dispatch = MultiStrategyDispatch(_fallback) + dispatch = MultiStrategyDispatch(lambda _: _fallback) assert dispatch.dispatch(Foo) == _fallback dispatch.register_func_list([(lambda cls: issubclass(cls, Foo), _foo_func)]) assert dispatch.dispatch(Foo) == _foo_func @@ -40,7 +40,7 @@ def test_multistrategy_dispatch_conflict_class_wins(): are registered which handle the same type, the class dispatch should return. """ - dispatch = MultiStrategyDispatch(_fallback) + dispatch = MultiStrategyDispatch(lambda _: _fallback) dispatch.register_func_list([(lambda cls: issubclass(cls, Foo), _foo_func)]) dispatch.register_cls_list([(Foo, _foo_cls)]) assert dispatch.dispatch(Foo) == _foo_cls diff --git a/tests/test_structure_attrs.py b/tests/test_structure_attrs.py index 874df86e..3b7b8ae3 100644 --- a/tests/test_structure_attrs.py +++ b/tests/test_structure_attrs.py @@ -5,7 +5,7 @@ from unittest.mock import Mock import pytest -from attr import NOTHING, Factory, asdict, astuple, attrib, define, fields, make_class +from attrs import NOTHING, Factory, asdict, astuple, define, field, fields, make_class from hypothesis import assume, given from hypothesis.strategies import data, lists, sampled_from @@ -244,11 +244,11 @@ def called_after_default_converter(val): "HasConverter", { # non-built-in type with custom converter - "ip": attrib(type=Union[IPv4Address, IPv6Address], converter=ip_address), + "ip": field(type=Union[IPv4Address, IPv6Address], converter=ip_address), # attribute without type - "x": attrib(converter=attrib_converter), + "x": field(converter=attrib_converter), # built-in types converters - "z": attrib(type=int, converter=called_after_default_converter), + "z": field(type=int, converter=called_after_default_converter), }, ) @@ -270,13 +270,13 @@ def test_structure_prefers_attrib_converters(converter_type): "HasConverter", { # non-built-in type with custom converter - "ip": attrib(type=Union[IPv4Address, IPv6Address], converter=ip_address), + "ip": field(type=Union[IPv4Address, IPv6Address], converter=ip_address), # attribute without type - "x": attrib(converter=attrib_converter), + "x": field(converter=attrib_converter), # built-in types converters - "y": attrib(type=int, converter=attrib_converter), + "y": field(type=int, converter=attrib_converter), # attribute with type and default value - "z": attrib(type=int, converter=attrib_converter, default=5), + "z": field(type=int, converter=attrib_converter, default=5), }, ) From 987aa8269e06147850d6df936919f311a34825ce Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Tue, 14 Nov 2023 13:20:53 +0100 Subject: [PATCH 2/3] Finish up fallback hook factories --- HISTORY.md | 2 ++ docs/cattrs.rst | 8 ++++++ docs/conf.py | 2 ++ docs/converters.md | 60 ++++++++++++++++++++++++++++++---------- docs/customizing.md | 2 +- docs/usage.md | 9 +++--- src/cattrs/converters.py | 16 +++++++++-- tests/test_converter.py | 24 ++++++++++++++++ 8 files changed, 101 insertions(+), 22 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index daabcdab..27e4c773 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -18,6 +18,8 @@ ([#405](https://github.com/python-attrs/cattrs/pull/405)) - The `omit` parameter of {py:func}`cattrs.override` is now of type `bool | None` (from `bool`). `None` is the new default and means to apply default _cattrs_ handling to the attribute, which is to omit the attribute if it's marked as `init=False`, and keep it otherwise. +- Converters can now be initialized with custom fallback hook factories for un/structuring. + ([#331](https://github.com/python-attrs/cattrs/issues/311) [#441](https://github.com/python-attrs/cattrs/pull/441)) - Fix {py:func}`format_exception() ` parameter working for recursive calls to {py:func}`transform_error `. ([#389](https://github.com/python-attrs/cattrs/issues/389)) - [_attrs_ aliases](https://www.attrs.org/en/stable/init.html#private-attributes-and-aliases) are now supported, although aliased fields still map to their attribute name instead of their alias by default when un/structuring. diff --git a/docs/cattrs.rst b/docs/cattrs.rst index 4c82a09c..df008424 100644 --- a/docs/cattrs.rst +++ b/docs/cattrs.rst @@ -46,6 +46,14 @@ cattrs.errors module :undoc-members: :show-inheritance: +cattrs.fns module +----------------- + +.. automodule:: cattrs.fns + :members: + :undoc-members: + :show-inheritance: + cattrs.v module --------------- diff --git a/docs/conf.py b/docs/conf.py index 323e6911..52c385ad 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -284,6 +284,7 @@ doctest_global_setup = ( "import attr, cattr, cattrs;" "from attr import Factory, define, field;" + "from cattrs import Converter;" "from typing import *;" "from enum import Enum, unique" ) @@ -292,3 +293,4 @@ copybutton_prompt_text = r">>> |\.\.\. " copybutton_prompt_is_regexp = True myst_heading_anchors = 3 +autoclass_content = "both" diff --git a/docs/converters.md b/docs/converters.md index 85c6e31e..7492c4bc 100644 --- a/docs/converters.md +++ b/docs/converters.md @@ -1,10 +1,8 @@ # Converters All _cattrs_ functionality is exposed through a {class}`cattrs.Converter` object. -Global _cattrs_ functions, such as {meth}`cattrs.unstructure`, use a single -global converter. Changes done to this global converter, such as registering new -structure and unstructure hooks, affect all code using the global -functions. +Global _cattrs_ functions, such as {meth}`cattrs.unstructure`, use a single global converter. +Changes done to this global converter, such as registering new structure and unstructure hooks, affect all code using the global functions. ## Global Converter @@ -18,8 +16,7 @@ The following functions implicitly use this global converter: Changes made to the global converter will affect the behavior of these functions. -Larger applications are strongly encouraged to create and customize a different, -private instance of {class}`cattrs.Converter`. +Larger applications are strongly encouraged to create and customize a different, private instance of {class}`cattrs.Converter`. ## Converter Objects @@ -32,14 +29,51 @@ Currently, a converter contains the following state: - a reference to an unstructuring strategy (either AS_DICT or AS_TUPLE). - a `dict_factory` callable, used for creating `dicts` when dumping _attrs_ classes using `AS_DICT`. -Converters may be cloned using the {meth}`cattrs.Converter.copy` method. +Converters may be cloned using the {meth}`Converter.copy() ` method. The new copy may be changed through the `copy` arguments, but will retain all manually registered hooks from the original. +### Fallback Hook Factories + +By default, when a {class}`converter ` cannot handle a type it will: + +* when unstructuring, pass the value through unchanged +* when structuring, raise a {class}`cattrs.errors.StructureHandlerNotFoundError` asking the user to add configuration + +These behaviors can be customized by providing custom [hook factories](usage.md#using-factory-hooks) when creating the converter. + +```python +>>> from pickle import dumps + +>>> class Unsupported: +... """An artisinal (non-attrs) class, unsupported by default.""" + +>>> converter = Converter(unstructure_fallback_factory=lambda _: dumps) +>>> instance = Unsupported() +>>> converter.unstructure(instance) +``` + +This also enables converters to be chained. + +```python +>>> parent = Converter() + +>>> child = Converter( +... unstructure_fallback_factory=parent._unstructure_func.dispatch, +... structure_fallback_factory=parent._structure_func.dispatch, +... ) +``` + +```{note} +`Converter._structure_func.dispatch` and `Converter._unstructure_func.dispatch` are slated to become public APIs in a future release. +``` + +```{versionadded} 23.2.0 + +``` + ## `cattrs.Converter` -The {class}`Converter ` is a converter variant that automatically generates, -compiles and caches specialized structuring and unstructuring hooks for _attrs_ -classes and dataclasses. +The {class}`Converter ` is a converter variant that automatically generates, compiles and caches specialized structuring and unstructuring hooks for _attrs_ classes, dataclasses and TypedDicts. `Converter` differs from the {class}`cattrs.BaseConverter` in the following ways: @@ -53,7 +87,5 @@ The `Converter` used to be called `GenConverter`, and that alias is still presen ## `cattrs.BaseConverter` -The {class}`BaseConverter ` is a simpler and slower Converter variant. It does no -code generation, so it may be faster on first-use which can be useful -in specific cases, like CLI applications where startup time is more -important than throughput. +The {class}`BaseConverter ` is a simpler and slower `Converter` variant. +It does no code generation, so it may be faster on first-use which can be useful in specific cases, like CLI applications where startup time is more important than throughput. diff --git a/docs/customizing.md b/docs/customizing.md index 894d4131..c54bc2ec 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -6,7 +6,7 @@ This section deals with customizing the unstructuring and structuring processes The default {class}`Converter `, upon first encountering an _attrs_ class, will use the generation functions mentioned here to generate the specialized hooks for it, register the hooks and use them. -## Manual un/structuring hooks +## Manual Un/structuring Hooks You can write your own structuring and unstructuring functions and register them for types using {meth}`Converter.register_structure_hook() ` and diff --git a/docs/usage.md b/docs/usage.md index eafce35d..9e6cf9a6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -95,10 +95,9 @@ MyRecord(a_string='test', a_datetime=DateTime(2018, 7, 28, 18, 24, 0, tzinfo=Tim MyRecord(a_string='test', a_datetime=DateTime(2018, 7, 28, 18, 24, 0, tzinfo=Timezone('+02:00'))) ``` -## Using factory hooks +## Using Factory Hooks -For this example, let's assume you have some attrs classes with snake case attributes, and you want to -un/structure them as camel case. +For this example, let's assume you have some attrs classes with snake case attributes, and you want to un/structure them as camel case. ```{warning} A simpler and better approach to this problem is to simply make your class attributes camel case. @@ -257,7 +256,7 @@ converter.register_structure_hook_factory( The `converter` instance will now un/structure every attrs class to camel case. Nothing has been omitted from this final example; it's complete. -## Using fallback key names +## Using Fallback Key Names Sometimes when structuring data, the input data may be in multiple formats that need to be converted into a common attribute. @@ -305,7 +304,7 @@ class MyInternalAttr: _cattrs_ will now structure both key names into `new_field` on your class. -``` +```python converter.structure({"new_field": "foo"}, MyInternalAttr) converter.structure({"old_field": "foo"}, MyInternalAttr) ``` diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 9f8c4f29..3ba1ecad 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -133,6 +133,8 @@ def __init__( structure_fallback_factory: HookFactory[StructureHook] = lambda _: raise_error, ) -> None: """ + :param detailed_validation: Whether to use a slightly slower mode for detailed + validation errors. :param unstructure_fallback_factory: A hook factory to be called when no registered unstructuring hooks match. :param structure_fallback_factory: A hook factory to be called when no @@ -734,7 +736,11 @@ def copy( prefer_attrib_converters: Optional[bool] = None, detailed_validation: Optional[bool] = None, ) -> "BaseConverter": - """Create a copy of the converter, keeping all existing custom hooks.""" + """Create a copy of the converter, keeping all existing custom hooks. + + :param detailed_validation: Whether to use a slightly slower mode for detailed + validation errors. + """ res = self.__class__( dict_factory if dict_factory is not None else self._dict_factory, unstruct_strat @@ -782,6 +788,8 @@ def __init__( structure_fallback_factory: HookFactory[StructureHook] = lambda _: raise_error, ): """ + :param detailed_validation: Whether to use a slightly slower mode for detailed + validation errors. :param unstructure_fallback_factory: A hook factory to be called when no registered unstructuring hooks match. :param structure_fallback_factory: A hook factory to be called when no @@ -1047,7 +1055,11 @@ def copy( prefer_attrib_converters: Optional[bool] = None, detailed_validation: Optional[bool] = None, ) -> "Converter": - """Create a copy of the converter, keeping all existing custom hooks.""" + """Create a copy of the converter, keeping all existing custom hooks. + + :param detailed_validation: Whether to use a slightly slower mode for detailed + validation errors. + """ res = self.__class__( dict_factory if dict_factory is not None else self._dict_factory, unstruct_strat diff --git a/tests/test_converter.py b/tests/test_converter.py index ce1259e2..6e0563b7 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -743,3 +743,27 @@ class Test: c = converter_cls(structure_fallback_factory=lambda _: lambda v, _: Test()) assert isinstance(c.structure({}, Test), Test) + + +def test_fallback_chaining(converter_cls: Type[BaseConverter]): + """Converters can be chained using fallback hooks.""" + + class Test: + """Unsupported by default.""" + + parent = converter_cls() + + parent.register_unstructure_hook(Test, lambda _: "Test") + parent.register_structure_hook(Test, lambda v, _: Test()) + + # No chaining first. + child = converter_cls() + with pytest.raises(StructureHandlerNotFoundError): + child.structure(child.unstructure(Test()), Test) + + child = converter_cls( + unstructure_fallback_factory=parent._unstructure_func.dispatch, + structure_fallback_factory=parent._structure_func.dispatch, + ) + + assert isinstance(child.structure(child.unstructure(Test()), Test), Test) From 034b3c0e636695e4af7b5742f0ed1a9eadce581d Mon Sep 17 00:00:00 2001 From: Tin Tvrtkovic Date: Tue, 14 Nov 2023 13:27:22 +0100 Subject: [PATCH 3/3] Tweak docs --- docs/converters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/converters.md b/docs/converters.md index 7492c4bc..05c04bf7 100644 --- a/docs/converters.md +++ b/docs/converters.md @@ -50,6 +50,7 @@ These behaviors can be customized by providing custom [hook factories](usage.md# >>> converter = Converter(unstructure_fallback_factory=lambda _: dumps) >>> instance = Unsupported() >>> converter.unstructure(instance) +b'\x80\x04\x95\x18\x00\x00\x00\x00\x00\x00\x00\x8c\x08__main__\x94\x8c\x04Test\x94\x93\x94)\x81\x94.' ``` This also enables converters to be chained.