From ba569eaf3ea8d0a12ac22ae8d81b1a0b3b2feec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sun, 14 Jan 2024 01:17:51 +0100 Subject: [PATCH 1/8] Initial work on decorators --- src/cattrs/_compat.py | 7 +++++ src/cattrs/converters.py | 56 ++++++++++++++++++++++++++++++++++++++-- tests/test_converter.py | 25 ++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index 0e010eda..8493f0c9 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -4,6 +4,8 @@ from collections.abc import Set as AbcSet from dataclasses import MISSING, Field, is_dataclass from dataclasses import fields as dataclass_fields +from functools import partial +from inspect import signature as _signature from typing import AbstractSet as TypingAbstractSet from typing import ( Any, @@ -211,6 +213,11 @@ def get_final_base(type) -> Optional[type]: OriginAbstractSet = AbcSet OriginMutableSet = AbcMutableSet +signature = _signature + +if sys.version_info >= (3, 10): + signature = partial(_signature, eval_str=True) + if sys.version_info >= (3, 9): from collections import Counter from collections.abc import Mapping as AbcMapping diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 5e7dcc61..9e1654cd 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -5,7 +5,7 @@ from dataclasses import Field from enum import Enum from pathlib import Path -from typing import Any, Callable, Iterable, Optional, Tuple, TypeVar +from typing import Any, Callable, Iterable, Optional, Tuple, TypeVar, overload from attrs import Attribute, resolve_types from attrs import has as attrs_has @@ -46,6 +46,7 @@ is_type_alias, is_typeddict, is_union_type, + signature, ) from .disambiguators import create_default_dis_func, is_supported_union from .dispatch import ( @@ -245,12 +246,38 @@ def unstruct_strat(self) -> UnstructureStrategy: else UnstructureStrategy.AS_TUPLE ) + @overload + def register_unstructure_hook(self) -> Callable[[UnstructureHook], None]: + ... + + @overload def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: + ... + + def register_unstructure_hook( + self, cls: Any = None, func: UnstructureHook | None = None + ) -> Callable[[UnstructureHook]] | None: """Register a class-to-primitive converter function for a class. The converter function should take an instance of the class and return its Python equivalent. + + May also be used as a decorator. When used as a decorator, the first + argument annotation from the decorated function will be used as the + type to register the hook for. + + .. versionchanged:: 24.1.0 + This method may now be used as a decorator. """ + if func is None: + # Autodetecting decorator. + func = cls + sig = signature(func) + cls = next(iter(sig.parameters.values())).annotation + self.register_unstructure_hook(cls, func) + + return None + if attrs_has(cls): resolve_types(cls) if is_union_type(cls): @@ -260,6 +287,7 @@ def register_unstructure_hook(self, cls: Any, func: UnstructureHook) -> None: self._unstructure_func.register_func_list([(lambda t: t is cls, func)]) else: self._unstructure_func.register_cls_list([(cls, func)]) + return None def register_unstructure_hook_func( self, check_func: Callable[[Any], bool], func: UnstructureHook @@ -303,7 +331,17 @@ def get_unstructure_hook( else self._unstructure_func.dispatch_without_caching(type) ) - def register_structure_hook(self, cl: Any, func: StructureHook) -> None: + @overload + def register_structure_hook(self) -> Callable[[StructureHook], None]: + ... + + @overload + def register_structure_hook(self, cl: Any, func: StructuredValue) -> None: + ... + + def register_structure_hook( + self, cl: Any, func: StructureHook | None = None + ) -> None: """Register a primitive-to-class converter function for a type. The converter function should take two arguments: @@ -312,7 +350,21 @@ def register_structure_hook(self, cl: Any, func: StructureHook) -> None: and return the instance of the class. The type may seem redundant, but is sometimes needed (for example, when dealing with generic classes). + + This method may be used as a decorator. In this case, the decorated + hook must have a return type annotation, and this annotation will be used + as the type for the hook. + + .. versionchanged:: 24.1.0 + This method may now be used as a decorator. """ + if func is None: + # The autodetecting decorator. + func = cl + sig = signature(func) + self.register_structure_hook(sig.return_annotation, func) + return + if attrs_has(cl): resolve_types(cl) if is_union_type(cl): diff --git a/tests/test_converter.py b/tests/test_converter.py index 65ee8496..85994a5a 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -783,3 +783,28 @@ class Test: structure = converter.get_structure_hook(Test) assert structure({"a": 1}, Test) == Test(1) + + +def test_decorators(converter: BaseConverter): + """The decorator versions work.""" + + @define + class Test: + a: int + + @converter.register_unstructure_hook + def my_hook(value: Test) -> dict: + res = {"a": value.a} + + res["a"] += 1 + + return res + + assert converter.unstructure(Test(1)) == {"a": 2} + + @converter.register_structure_hook + def my_structure_hook(value, _) -> Test: + value["a"] += 1 + return Test(**value) + + assert converter.structure({"a": 5}, Test) == Test(6) From 6059cb4870e84813f04027c2bb53289f10def271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Wed, 24 Jan 2024 00:08:29 +0100 Subject: [PATCH 2/8] Docs --- HISTORY.md | 3 +++ docs/basics.md | 4 ++-- docs/customizing.md | 31 +++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index f88ecc06..c43d16b5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -19,6 +19,9 @@ Our backwards-compatibility policy can be found [here](https://github.com/python ([#486](https://github.com/python-attrs/cattrs/pull/486)) - Introduce {meth}`BaseConverter.get_structure_hook` and {meth}`BaseConverter.get_unstructure_hook` methods. ([#432](https://github.com/python-attrs/cattrs/issues/432) [#472](https://github.com/python-attrs/cattrs/pull/472)) +- {meth}`BaseConverter.register_structure_hook` and {meth}`BaseConverter.register_unstructure_hook` can now be used as decorators. + See [here](https://catt.rs/en/latest/customizing.html#custom-un-structuring-hooks) for more details. + ([#487](https://github.com/python-attrs/cattrs/pull/487)) - Introduce the [_msgspec_](https://jcristharif.com/msgspec/) {mod}`preconf converter `. Only JSON is supported for now, with other formats supported by _msgspec_ to come later. ([#481](https://github.com/python-attrs/cattrs/pull/481)) diff --git a/docs/basics.md b/docs/basics.md index cf978de6..f40559ab 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -63,7 +63,7 @@ A base hook can be obtained from a converter and then be subjected to the very r ... return result ``` -(`cattrs.structure({}, Model)` is shorthand for `cattrs.get_structure_hook(Model)({}, Model)`.) +(`cattrs.structure({}, Model)` is equivalent to `cattrs.get_structure_hook(Model)({}, Model)`.) This new hook can be used directly or registered to a converter (the original instance, or a different one): @@ -72,7 +72,7 @@ This new hook can be used directly or registered to a converter (the original in ``` -Now if we use this hook to structure a `Model`, through the ✨magic of function composition✨ that hook will use our old `int_hook`. +Now if we use this hook to structure a `Model`, through ✨the magic of function composition✨ that hook will use our old `int_hook`. ```python >>> converter.structure({"a": "1"}, Model) diff --git a/docs/customizing.md b/docs/customizing.md index e10a9743..d3efb65b 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -17,9 +17,40 @@ Some examples of this are: * protocols, unless they are `runtime_checkable` * various modifiers, such as `Final` and `NotRequired` * newtypes and 3.12 type aliases +* `typing.Annotated` ... and many others. In these cases, predicate functions should be used instead. +### Use as Decorators + +{meth}`register_structure_hook() ` and {meth}`register_unstructure_hook() ` can also be used as _decorators_. +When used this way they behave a little differently. + +{meth}`register_structure_hook() ` will inspect the return type of the hook and register the hook for that type. + +```python +@converter.register_structure_hook +def my_int_hook(val: Any, _) -> int: + """This hook will be registered for `int`s.""" + return int(val) +``` + +{meth}`register_unstructure_hook() ` will inspect the type of the first argument and register the hook for that type. + +```python +from datetime import datetime + +@converter.register_unstructure_hook +def my_datetime_hook(val: datetime) -> str: + """This hook will be registered for `datetime`s.""" + return val.isoformat() +``` + +The non-decorator approach is still recommended when dealing with lambdas, hooks produced elsewhere, and unannotated hooks. + +```{versionadded} 24.1.0 +``` + ### Predicate Hooks A predicate is a function that takes a type and returns true or false, depending on whether the associated hook can handle the given type. From 7d11c097f8867cbc97deb88785b4d34bda1d6235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Thu, 25 Jan 2024 01:00:49 +0100 Subject: [PATCH 3/8] More decorators --- HISTORY.md | 2 +- docs/conf.py | 1 - docs/customizing.md | 31 +++++++++++++++++++-- src/cattrs/converters.py | 60 +++++++++++++++++++++++++++++++++++++--- tests/test_converter.py | 24 +++++++++++++++- 5 files changed, 108 insertions(+), 10 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index c43d16b5..7e8bf932 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -20,7 +20,7 @@ Our backwards-compatibility policy can be found [here](https://github.com/python - Introduce {meth}`BaseConverter.get_structure_hook` and {meth}`BaseConverter.get_unstructure_hook` methods. ([#432](https://github.com/python-attrs/cattrs/issues/432) [#472](https://github.com/python-attrs/cattrs/pull/472)) - {meth}`BaseConverter.register_structure_hook` and {meth}`BaseConverter.register_unstructure_hook` can now be used as decorators. - See [here](https://catt.rs/en/latest/customizing.html#custom-un-structuring-hooks) for more details. + See [here](https://catt.rs/en/latest/customizing.html#use-as-decorators) and [here](https://catt.rs/en/latest/customizing.html#id1) for more details. ([#487](https://github.com/python-attrs/cattrs/pull/487)) - Introduce the [_msgspec_](https://jcristharif.com/msgspec/) {mod}`preconf converter `. Only JSON is supported for now, with other formats supported by _msgspec_ to come later. diff --git a/docs/conf.py b/docs/conf.py index 5badbac3..9643037a 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,7 +40,6 @@ "sphinx.ext.autodoc", "sphinx.ext.viewcode", "sphinx.ext.doctest", - "sphinx.ext.autosectionlabel", "sphinx_copybutton", "myst_parser", ] diff --git a/docs/customizing.md b/docs/customizing.md index d3efb65b..f6e90bc7 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -46,7 +46,7 @@ def my_datetime_hook(val: datetime) -> str: return val.isoformat() ``` -The non-decorator approach is still recommended when dealing with lambdas, hooks produced elsewhere, and unannotated hooks. +The non-decorator approach is still recommended when dealing with lambdas, hooks produced elsewhere, unannotated hooks and situations where type introspection doesn't work. ```{versionadded} 24.1.0 ``` @@ -95,9 +95,10 @@ Here's an example showing how to use hook factories to apply the `forbid_extra_k ```python >>> from attrs import define, has +>>> from cattrs import Converter >>> from cattrs.gen import make_dict_structure_fn ->>> c = cattrs.Converter() +>>> c = Converter() >>> c.register_structure_hook_factory( ... has, ... lambda cl: make_dict_structure_fn(cl, c, _cattrs_forbid_extra_keys=True) @@ -113,8 +114,32 @@ Traceback (most recent call last): cattrs.errors.ForbiddenExtraKeysError: Extra fields in constructor for E: else ``` -A complex use case for hook factories is described over at {ref}`usage:Using factory hooks`. +A complex use case for hook factories is described over at [](usage.md#using-factory-hooks). + +#### Use as Decorators + +{meth}`Converter.register_unstructure_hook_factory() ` and {meth}`Converter.register_structure_hook_factory() ` can also be used as decorators. + +Here's an example of using an unstructure hook factory to unstructure all attrs classes using [field aliases](#use_alias). + +```{doctest} +>>> from attrs import define, has +>>> from cattrs import Converter +>>> from cattrs.gen import make_dict_unstructure_fn +>>> c = Converter() + +>>> @c.register_unstructure_hook_factory(has) +... def attrs_hook_factory(cl: Any) -> Callable: +... return make_dict_unstructure_fn(cl, c, _cattrs_use_alias=True) + +>>> @define +... class F: +... _a: int + +>>> c.unstructure(F(1)) +{'a': 1} +``` ## Using `cattrs.gen` Generators diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 9e1654cd..c7376864 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -4,6 +4,7 @@ from collections.abc import MutableSet as AbcMutableSet from dataclasses import Field from enum import Enum +from functools import partial from pathlib import Path from typing import Any, Callable, Iterable, Optional, Tuple, TypeVar, overload @@ -84,6 +85,10 @@ T = TypeVar("T") V = TypeVar("V") +UnstructureHookFactory = TypeVar( + "UnstructureHookFactory", bound=HookFactory[UnstructureHook] +) +StructureHookFactory = TypeVar("StructureHookFactory", bound=HookFactory[StructureHook]) class UnstructureStrategy(Enum): @@ -297,18 +302,43 @@ def register_unstructure_hook_func( """ self._unstructure_func.register_func_list([(check_func, func)]) + @overload def register_unstructure_hook_factory( - self, predicate: Callable[[Any], bool], factory: HookFactory[UnstructureHook] - ) -> None: + self, predicate: Callable[[Any], bool] + ) -> Callable[[UnstructureHookFactory], UnstructureHookFactory]: + ... + + @overload + def register_unstructure_hook_factory( + self, predicate: Callable[[Any], bool], factory: UnstructureHookFactory + ) -> UnstructureHookFactory: + ... + + def register_unstructure_hook_factory( + self, + predicate: Callable[[Any], bool], + factory: UnstructureHookFactory | None = None, + ) -> ( + Callable[[UnstructureHookFactory], UnstructureHookFactory] + | UnstructureHookFactory + ): """ Register a hook factory for a given predicate. + May also be used as a decorator. + :param predicate: A function that, given a type, returns whether the factory can produce a hook for that type. :param factory: A callable that, given a type, produces an unstructuring hook for that type. This unstructuring hook will be cached. + + .. versionchanged:: 24.1.0 + This method may now be used as a decorator. """ + if factory is None: + return partial(self.register_unstructure_hook_factory, predicate) self._unstructure_func.register_func_list([(predicate, factory, True)]) + return factory def get_unstructure_hook( self, type: Any, cache_result: bool = True @@ -384,18 +414,40 @@ def register_structure_hook_func( """ self._structure_func.register_func_list([(check_func, func)]) + @overload def register_structure_hook_factory( - self, predicate: Callable[[Any], bool], factory: HookFactory[StructureHook] - ) -> None: + self, predicate: Callable[[Any, bool]] + ) -> Callable[[StructureHookFactory, StructureHookFactory]]: + ... + + @overload + def register_structure_hook_factory( + self, predicate: Callable[[Any], bool], factory: StructureHookFactory + ) -> StructureHookFactory: + ... + + def register_structure_hook_factory( + self, + predicate: Callable[[Any], bool], + factory: HookFactory[StructureHook] | None = None, + ) -> Callable[[StructureHookFactory, StructureHookFactory]] | StructureHookFactory: """ Register a hook factory for a given predicate. + May also be used as a decorator. + :param predicate: A function that, given a type, returns whether the factory can produce a hook for that type. :param factory: A callable that, given a type, produces a structuring hook for that type. This structuring hook will be cached. + + .. versionchanged:: 24.1.0 + This method may now be used as a decorator. """ + if factory is None: + return partial(self.register_structure_hook_factory, predicate) self._structure_func.register_func_list([(predicate, factory, True)]) + return factory def structure(self, obj: UnstructuredValue, cl: type[T]) -> T: """Convert unstructured Python data structures to structured data.""" diff --git a/tests/test_converter.py b/tests/test_converter.py index 85994a5a..87dfc0cf 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -1,6 +1,7 @@ """Test both structuring and unstructuring.""" from collections import deque from typing import ( + Any, Deque, FrozenSet, List, @@ -14,11 +15,12 @@ ) import pytest -from attrs import Factory, define, fields, make_class +from attrs import Factory, define, fields, has, make_class from hypothesis import HealthCheck, assume, given, settings from hypothesis.strategies import booleans, just, lists, one_of, sampled_from from cattrs import BaseConverter, Converter, UnstructureStrategy +from cattrs.dispatch import StructureHook, UnstructureHook from cattrs.errors import ( ClassValidationError, ForbiddenExtraKeysError, @@ -808,3 +810,23 @@ def my_structure_hook(value, _) -> Test: return Test(**value) assert converter.structure({"a": 5}, Test) == Test(6) + + +def test_hook_factory_decorators(converter: BaseConverter): + """Hook factory decorators work.""" + + @define + class Test: + a: int + + @converter.register_unstructure_hook_factory(has) + def my_hook_factory(type: Any) -> UnstructureHook: + return lambda v: v.a + + assert converter.unstructure(Test(1)) == 1 + + @converter.register_structure_hook_factory(has) + def my_structure_hook_factory(type: Any) -> StructureHook: + return lambda v, _: Test(v) + + assert converter.structure(1, Test) == Test(1) From 436d651b9e555da2cb99727795462627a4c37680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Thu, 25 Jan 2024 01:20:36 +0100 Subject: [PATCH 4/8] exclude_also? --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9f3530ab..682d179f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,11 @@ source = [ ".tox/pypy*/site-packages", ] +[tool.coverage.report] +exclude_also = [ + "@overload", +] + [tool.ruff] src = ["src", "tests"] select = [ From e613d748645fe95a6c35ad564c010620b17e76b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 27 Jan 2024 19:01:29 +0100 Subject: [PATCH 5/8] Enable hook factories to take converters --- src/cattrs/converters.py | 92 ++++++++++++++++++++++---- src/cattrs/dispatch.py | 79 +++++++++++++--------- src/cattrs/gen/typeddicts.py | 4 +- src/cattrs/preconf/orjson.py | 2 +- tests/strategies/test_native_unions.py | 2 +- tests/test_converter.py | 26 ++++++++ tests/test_function_dispatch.py | 5 +- tests/test_multistrategy_dispatch.py | 10 ++- 8 files changed, 169 insertions(+), 51 deletions(-) diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index c7376864..4d438beb 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -5,6 +5,7 @@ from dataclasses import Field from enum import Enum from functools import partial +from inspect import Signature from pathlib import Path from typing import Any, Callable, Iterable, Optional, Tuple, TypeVar, overload @@ -55,6 +56,7 @@ MultiStrategyDispatch, StructuredValue, StructureHook, + TargetType, UnstructuredValue, UnstructureHook, ) @@ -85,11 +87,25 @@ T = TypeVar("T") V = TypeVar("V") + UnstructureHookFactory = TypeVar( "UnstructureHookFactory", bound=HookFactory[UnstructureHook] ) + +# The Extended factory also takes a converter. +ExtendedUnstructureHookFactory = TypeVar( + "ExtendedUnstructureHookFactory", + bound=Callable[[TargetType, "BaseConverter"], UnstructureHook], +) + StructureHookFactory = TypeVar("StructureHookFactory", bound=HookFactory[StructureHook]) +# The Extended factory also takes a converter. +ExtendedStructureHookFactory = TypeVar( + "ExtendedStructureHookFactory", + bound=Callable[[TargetType, "BaseConverter"], StructureHook], +) + class UnstructureStrategy(Enum): """`attrs` classes unstructuring strategies.""" @@ -151,7 +167,9 @@ def __init__( self._unstructure_attrs = self.unstructure_attrs_astuple self._structure_attrs = self.structure_attrs_fromtuple - self._unstructure_func = MultiStrategyDispatch(unstructure_fallback_factory) + self._unstructure_func = MultiStrategyDispatch( + unstructure_fallback_factory, self + ) self._unstructure_func.register_cls_list( [(bytes, identity), (str, identity), (Path, str)] ) @@ -163,12 +181,12 @@ def __init__( ), ( lambda t: get_final_base(t) is not None, - lambda t: self._unstructure_func.dispatch(get_final_base(t)), + lambda t: self.get_unstructure_hook(get_final_base(t)), True, ), ( is_type_alias, - lambda t: self._unstructure_func.dispatch(get_type_alias_base(t)), + lambda t: self.get_unstructure_hook(get_type_alias_base(t)), True, ), (is_mapping, self._unstructure_mapping), @@ -185,7 +203,7 @@ def __init__( # Per-instance register of to-attrs converters. # Singledispatch dispatches based on the first argument, so we # store the function and switch the arguments in self.loads. - self._structure_func = MultiStrategyDispatch(structure_fallback_factory) + self._structure_func = MultiStrategyDispatch(structure_fallback_factory, self) self._structure_func.register_func_list( [ ( @@ -308,6 +326,12 @@ def register_unstructure_hook_factory( ) -> Callable[[UnstructureHookFactory], UnstructureHookFactory]: ... + @overload + def register_unstructure_hook_factory( + self, predicate: Callable[[Any], bool] + ) -> Callable[[ExtendedUnstructureHookFactory], ExtendedUnstructureHookFactory]: + ... + @overload def register_unstructure_hook_factory( self, predicate: Callable[[Any], bool], factory: UnstructureHookFactory @@ -325,7 +349,10 @@ def register_unstructure_hook_factory( """ Register a hook factory for a given predicate. - May also be used as a decorator. + May also be used as a decorator. When used as a decorator, the hook + factory may expose an additional required parameter. In this case, + the current converter will be provided to the hook factory as that + parameter. :param predicate: A function that, given a type, returns whether the factory can produce a hook for that type. @@ -336,7 +363,23 @@ def register_unstructure_hook_factory( This method may now be used as a decorator. """ if factory is None: - return partial(self.register_unstructure_hook_factory, predicate) + + def decorator(factory): + # Is this an extended factory (takes a converter too)? + sig = signature(factory) + if ( + len(sig.parameters) >= 2 + and (list(sig.parameters.values())[1]).default is Signature.empty + ): + self._unstructure_func.register_func_list( + [(predicate, factory, "extended")] + ) + else: + self._unstructure_func.register_func_list( + [(predicate, factory, True)] + ) + + return decorator self._unstructure_func.register_func_list([(predicate, factory, True)]) return factory @@ -420,6 +463,12 @@ def register_structure_hook_factory( ) -> Callable[[StructureHookFactory, StructureHookFactory]]: ... + @overload + def register_structure_hook_factory( + self, predicate: Callable[[Any, bool]] + ) -> Callable[[ExtendedStructureHookFactory, ExtendedStructureHookFactory]]: + ... + @overload def register_structure_hook_factory( self, predicate: Callable[[Any], bool], factory: StructureHookFactory @@ -434,7 +483,10 @@ def register_structure_hook_factory( """ Register a hook factory for a given predicate. - May also be used as a decorator. + May also be used as a decorator. When used as a decorator, the hook + factory may expose an additional required parameter. In this case, + the current converter will be provided to the hook factory as that + parameter. :param predicate: A function that, given a type, returns whether the factory can produce a hook for that type. @@ -445,7 +497,23 @@ def register_structure_hook_factory( This method may now be used as a decorator. """ if factory is None: - return partial(self.register_structure_hook_factory, predicate) + # Decorator use. + def decorator(factory): + # Is this an extended factory (takes a converter too)? + sig = signature(factory) + if ( + len(sig.parameters) >= 2 + and (list(sig.parameters.values())[1]).default is Signature.empty + ): + self._structure_func.register_func_list( + [(predicate, factory, "extended")] + ) + else: + self._structure_func.register_func_list( + [(predicate, factory, True)] + ) + + return decorator self._structure_func.register_func_list([(predicate, factory, True)]) return factory @@ -684,7 +752,7 @@ def _structure_list(self, obj: Iterable[T], cl: Any) -> list[T]: def _structure_deque(self, obj: Iterable[T], cl: Any) -> deque[T]: """Convert an iterable to a potentially generic deque.""" if is_bare(cl) or cl.__args__[0] in ANIES: - res = deque(e for e in obj) + res = deque(obj) else: elem_type = cl.__args__[0] handler = self._structure_func.dispatch(elem_type) @@ -1048,7 +1116,7 @@ def __init__( ) self.register_unstructure_hook_factory( lambda t: get_newtype_base(t) is not None, - lambda t: self._unstructure_func.dispatch(get_newtype_base(t)), + lambda t: self.get_unstructure_hook(get_newtype_base(t)), ) self.register_structure_hook_factory(is_annotated, self.gen_structure_annotated) @@ -1070,7 +1138,7 @@ def get_structure_newtype(self, type: type[T]) -> Callable[[Any, Any], T]: def gen_unstructure_annotated(self, type): origin = type.__origin__ - return self._unstructure_func.dispatch(origin) + return self.get_unstructure_hook(origin) def gen_structure_annotated(self, type) -> Callable: """A hook factory for annotated types.""" @@ -1111,7 +1179,7 @@ def gen_unstructure_optional(self, cl: type[T]) -> Callable[[T], Any]: if isinstance(other, TypeVar): handler = self.unstructure else: - handler = self._unstructure_func.dispatch(other) + handler = self.get_unstructure_hook(other) def unstructure_optional(val, _handler=handler): return None if val is None else _handler(val) diff --git a/src/cattrs/dispatch.py b/src/cattrs/dispatch.py index fe3ceba8..e72f8bb9 100644 --- a/src/cattrs/dispatch.py +++ b/src/cattrs/dispatch.py @@ -1,10 +1,15 @@ -from functools import lru_cache, partial, singledispatch -from typing import Any, Callable, Dict, Generic, List, Optional, Tuple, TypeVar, Union +from __future__ import annotations -from attrs import Factory, define, field +from functools import lru_cache, singledispatch +from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, TypeVar + +from attrs import Factory, define from cattrs._compat import TypeAlias +if TYPE_CHECKING: + from .converters import BaseConverter + T = TypeVar("T") TargetType: TypeAlias = Any @@ -33,23 +38,25 @@ class FunctionDispatch: objects that help determine dispatch should be instantiated objects. """ - _handler_pairs: List[ - Tuple[Callable[[Any], bool], Callable[[Any, Any], Any], bool] + _converter: BaseConverter + _handler_pairs: list[ + tuple[Callable[[Any], bool], Callable[[Any, Any], Any], bool, bool] ] = Factory(list) def register( self, - can_handle: Callable[[Any], bool], + predicate: Callable[[Any], bool], func: Callable[..., Any], is_generator=False, + takes_converter=False, ) -> None: - self._handler_pairs.insert(0, (can_handle, func, is_generator)) + self._handler_pairs.insert(0, (predicate, func, is_generator, takes_converter)) - def dispatch(self, typ: Any) -> Optional[Callable[..., Any]]: + def dispatch(self, typ: Any) -> Callable[..., Any] | None: """ Return the appropriate handler for the object passed. """ - for can_handle, handler, is_generator in self._handler_pairs: + for can_handle, handler, is_generator, takes_converter in self._handler_pairs: # can handle could raise an exception here # such as issubclass being called on an instance. # it's easier to just ignore that case. @@ -59,6 +66,8 @@ def dispatch(self, typ: Any) -> Optional[Callable[..., Any]]: continue if ch: if is_generator: + if takes_converter: + return handler(typ, self._converter) return handler(typ) return handler @@ -67,11 +76,11 @@ def dispatch(self, typ: Any) -> Optional[Callable[..., Any]]: def get_num_fns(self) -> int: return len(self._handler_pairs) - def copy_to(self, other: "FunctionDispatch", skip: int = 0) -> None: + def copy_to(self, other: FunctionDispatch, skip: int = 0) -> None: other._handler_pairs = self._handler_pairs[:-skip] + other._handler_pairs -@define +@define(init=False) class MultiStrategyDispatch(Generic[Hook]): """ MultiStrategyDispatch uses a combination of exact-match dispatch, @@ -85,18 +94,20 @@ class MultiStrategyDispatch(Generic[Hook]): """ _fallback_factory: HookFactory[Hook] - _direct_dispatch: Dict[TargetType, Hook] = field(init=False, factory=dict) - _function_dispatch: FunctionDispatch = field(init=False, factory=FunctionDispatch) - _single_dispatch: Any = field( - init=False, factory=partial(singledispatch, _DispatchNotFound) - ) - dispatch: Callable[[TargetType], Hook] = field( - init=False, - default=Factory( - lambda self: lru_cache(maxsize=None)(self.dispatch_without_caching), - takes_self=True, - ), - ) + _converter: BaseConverter + _direct_dispatch: dict[TargetType, Hook] + _function_dispatch: FunctionDispatch + _single_dispatch: Any + dispatch: Callable[[TargetType, BaseConverter], Hook] + + def __init__( + self, fallback_factory: HookFactory[Hook], converter: BaseConverter + ) -> None: + self._fallback_factory = fallback_factory + self._direct_dispatch = {} + self._function_dispatch = FunctionDispatch(converter) + self._single_dispatch = singledispatch(_DispatchNotFound) + self.dispatch = lru_cache(maxsize=None)(self.dispatch_without_caching) def dispatch_without_caching(self, typ: TargetType) -> Hook: """Dispatch on the type but without caching the result.""" @@ -126,15 +137,18 @@ def register_cls_list(self, cls_and_handler, direct: bool = False) -> None: def register_func_list( self, - pred_and_handler: List[ - Union[ - Tuple[Callable[[Any], bool], Any], - Tuple[Callable[[Any], bool], Any, bool], + pred_and_handler: list[ + tuple[Callable[[Any], bool], Any] + | tuple[Callable[[Any], bool], Any, bool] + | tuple[ + Callable[[Any], bool], + Callable[[Any, BaseConverter], Any], + Literal["extended"], ] ], ): """ - Register a predicate function to determine if the handle + Register a predicate function to determine if the handler should be used for the type. """ for tup in pred_and_handler: @@ -143,7 +157,12 @@ def register_func_list( self._function_dispatch.register(func, handler) else: func, handler, is_gen = tup - self._function_dispatch.register(func, handler, is_generator=is_gen) + if is_gen == "extended": + self._function_dispatch.register( + func, handler, is_generator=is_gen, takes_converter=True + ) + else: + self._function_dispatch.register(func, handler, is_generator=is_gen) self.clear_direct() self.dispatch.cache_clear() @@ -159,7 +178,7 @@ def clear_cache(self) -> None: def get_num_fns(self) -> int: return self._function_dispatch.get_num_fns() - def copy_to(self, other: "MultiStrategyDispatch", skip: int = 0) -> None: + def copy_to(self, other: MultiStrategyDispatch, skip: int = 0) -> None: self._function_dispatch.copy_to(other._function_dispatch, skip=skip) for cls, fn in self._single_dispatch.registry.items(): other._single_dispatch.register(cls, fn) diff --git a/src/cattrs/gen/typeddicts.py b/src/cattrs/gen/typeddicts.py index f77c0a86..99bc786d 100644 --- a/src/cattrs/gen/typeddicts.py +++ b/src/cattrs/gen/typeddicts.py @@ -139,7 +139,7 @@ def make_dict_unstructure_fn( if nrb is not NOTHING: t = nrb try: - handler = converter._unstructure_func.dispatch(t) + handler = converter.get_unstructure_hook(t) except RecursionError: # There's a circular reference somewhere down the line handler = converter.unstructure @@ -185,7 +185,7 @@ def make_dict_unstructure_fn( if nrb is not NOTHING: t = nrb try: - handler = converter._unstructure_func.dispatch(t) + handler = converter.get_unstructure_hook(t) except RecursionError: # There's a circular reference somewhere down the line handler = converter.unstructure diff --git a/src/cattrs/preconf/orjson.py b/src/cattrs/preconf/orjson.py index 8df76a78..f913dd8f 100644 --- a/src/cattrs/preconf/orjson.py +++ b/src/cattrs/preconf/orjson.py @@ -56,7 +56,7 @@ def key_handler(v): # (For example base85 encoding for bytes.) # In that case, we want to use the override. - kh = converter._unstructure_func.dispatch(args[0]) + kh = converter.get_unstructure_hook(args[0]) if kh != identity: key_handler = kh diff --git a/tests/strategies/test_native_unions.py b/tests/strategies/test_native_unions.py index a7c5ca61..cbe0b7e8 100644 --- a/tests/strategies/test_native_unions.py +++ b/tests/strategies/test_native_unions.py @@ -58,7 +58,7 @@ def test_skip_optionals() -> None: configure_union_passthrough(Union[int, str, None], c) - h = c._structure_func.dispatch(Optional[int]) + h = c.get_structure_hook(Optional[int]) assert h.__name__ != "structure_native_union" diff --git a/tests/test_converter.py b/tests/test_converter.py index 87dfc0cf..526464bd 100644 --- a/tests/test_converter.py +++ b/tests/test_converter.py @@ -830,3 +830,29 @@ def my_structure_hook_factory(type: Any) -> StructureHook: return lambda v, _: Test(v) assert converter.structure(1, Test) == Test(1) + + +def test_hook_factory_decorators_with_converters(converter: BaseConverter): + """Hook factory decorators with converters work.""" + + @define + class Test: + a: int + + converter.register_unstructure_hook(int, lambda v: v + 1) + + @converter.register_unstructure_hook_factory(has) + def my_hook_factory(type: Any, converter: BaseConverter) -> UnstructureHook: + int_handler = converter.get_unstructure_hook(int) + return lambda v: (int_handler(v.a),) + + assert converter.unstructure(Test(1)) == (2,) + + converter.register_structure_hook(int, lambda v: v - 1) + + @converter.register_structure_hook_factory(has) + def my_structure_hook_factory(type: Any, converter: BaseConverter) -> StructureHook: + int_handler = converter.get_structure_hook(int) + return lambda v, _: Test(int_handler(v[0])) + + assert converter.structure((2,), Test) == Test(1) diff --git a/tests/test_function_dispatch.py b/tests/test_function_dispatch.py index 7485d726..4641443d 100644 --- a/tests/test_function_dispatch.py +++ b/tests/test_function_dispatch.py @@ -1,8 +1,9 @@ +from cattrs import BaseConverter from cattrs.dispatch import FunctionDispatch def test_function_dispatch(): - dispatch = FunctionDispatch() + dispatch = FunctionDispatch(BaseConverter()) assert dispatch.dispatch(float) is None @@ -14,7 +15,7 @@ def test_function_dispatch(): def test_function_clears_cache_after_function_added(): - dispatch = FunctionDispatch() + dispatch = FunctionDispatch(BaseConverter()) class Foo: pass diff --git a/tests/test_multistrategy_dispatch.py b/tests/test_multistrategy_dispatch.py index 6d9c3499..f36f4445 100644 --- a/tests/test_multistrategy_dispatch.py +++ b/tests/test_multistrategy_dispatch.py @@ -1,3 +1,4 @@ +from cattrs import BaseConverter from cattrs.dispatch import MultiStrategyDispatch @@ -17,18 +18,21 @@ def _foo_cls(): pass +c = BaseConverter() + + def test_multistrategy_dispatch_register_cls(): _fallback() _foo_func() _foo_cls() - dispatch = MultiStrategyDispatch(lambda _: _fallback) + dispatch = MultiStrategyDispatch(lambda _: _fallback, c) assert dispatch.dispatch(Foo) == _fallback dispatch.register_cls_list([(Foo, _foo_cls)]) assert dispatch.dispatch(Foo) == _foo_cls def test_multistrategy_dispatch_register_func(): - dispatch = MultiStrategyDispatch(lambda _: _fallback) + dispatch = MultiStrategyDispatch(lambda _: _fallback, c) assert dispatch.dispatch(Foo) == _fallback dispatch.register_func_list([(lambda cls: issubclass(cls, Foo), _foo_func)]) assert dispatch.dispatch(Foo) == _foo_func @@ -40,7 +44,7 @@ def test_multistrategy_dispatch_conflict_class_wins(): are registered which handle the same type, the class dispatch should return. """ - dispatch = MultiStrategyDispatch(lambda _: _fallback) + dispatch = MultiStrategyDispatch(lambda _: _fallback, c) dispatch.register_func_list([(lambda cls: issubclass(cls, Foo), _foo_func)]) dispatch.register_cls_list([(Foo, _foo_cls)]) assert dispatch.dispatch(Foo) == _foo_cls From a691684f016c8c9527a208d8d78da34f8c5604c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sat, 27 Jan 2024 19:13:14 +0100 Subject: [PATCH 6/8] Fix lint --- src/cattrs/converters.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cattrs/converters.py b/src/cattrs/converters.py index 4d438beb..7df20632 100644 --- a/src/cattrs/converters.py +++ b/src/cattrs/converters.py @@ -4,7 +4,6 @@ from collections.abc import MutableSet as AbcMutableSet from dataclasses import Field from enum import Enum -from functools import partial from inspect import Signature from pathlib import Path from typing import Any, Callable, Iterable, Optional, Tuple, TypeVar, overload From 5e68f267683e0132386c6832b4d1d932b1944e1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sun, 28 Jan 2024 02:11:44 +0100 Subject: [PATCH 7/8] Docs --- HISTORY.md | 4 +++- docs/customizing.md | 33 ++++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 7e8bf932..ed2c54ff 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -19,7 +19,9 @@ Our backwards-compatibility policy can be found [here](https://github.com/python ([#486](https://github.com/python-attrs/cattrs/pull/486)) - Introduce {meth}`BaseConverter.get_structure_hook` and {meth}`BaseConverter.get_unstructure_hook` methods. ([#432](https://github.com/python-attrs/cattrs/issues/432) [#472](https://github.com/python-attrs/cattrs/pull/472)) -- {meth}`BaseConverter.register_structure_hook` and {meth}`BaseConverter.register_unstructure_hook` can now be used as decorators. +- {meth}`BaseConverter.register_structure_hook`, {meth}`BaseConverter.register_unstructure_hook`, +{meth}`BaseConverter.register_unstructure_hook_factory` and {meth}`BaseConverter.register_structure_hook_factory` +can now be used as decorators and have gained new features when used this way. See [here](https://catt.rs/en/latest/customizing.html#use-as-decorators) and [here](https://catt.rs/en/latest/customizing.html#id1) for more details. ([#487](https://github.com/python-attrs/cattrs/pull/487)) - Introduce the [_msgspec_](https://jcristharif.com/msgspec/) {mod}`preconf converter `. diff --git a/docs/customizing.md b/docs/customizing.md index f6e90bc7..25a1ab68 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -120,25 +120,36 @@ A complex use case for hook factories is described over at [](usage.md#using-fac {meth}`Converter.register_unstructure_hook_factory() ` and {meth}`Converter.register_structure_hook_factory() ` can also be used as decorators. -Here's an example of using an unstructure hook factory to unstructure all attrs classes using [field aliases](#use_alias). +When registered via decorators, factory hooks can receive the current converter by exposing an additional required parameter. + +Here's an example of using an unstructure hook factory to handle unstructuring queues. ```{doctest} ->>> from attrs import define, has +>>> from queue import Queue +>>> from typing import get_origin >>> from cattrs import Converter ->>> from cattrs.gen import make_dict_unstructure_fn >>> c = Converter() ->>> @c.register_unstructure_hook_factory(has) -... def attrs_hook_factory(cl: Any) -> Callable: -... return make_dict_unstructure_fn(cl, c, _cattrs_use_alias=True) +>>> @c.register_unstructure_hook_factory(lambda t: get_origin(t) is Queue) +... def queue_hook_factory(cl: Any, converter: Converter) -> Callable: +... type_arg = get_args(cl)[0] +... elem_handler = converter.get_unstructure_hook(type_arg) +... +... def unstructure_hook(v: Queue) -> list: +... res = [] +... while not v.empty(): +... res.append(elem_handler(v.get_nowait())) +... return res +... +... return unstructure_hook ->>> @define -... class F: -... _a: int +>>> q = Queue() +>>> q.put(1) +>>> q.put(2) ->>> c.unstructure(F(1)) -{'a': 1} +>>> c.unstructure(q, unstructure_as=Queue[int]) +[1, 2] ``` ## Using `cattrs.gen` Generators From a0615563ab882bea9420b23d764ef24256d7a9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Sun, 28 Jan 2024 02:17:15 +0100 Subject: [PATCH 8/8] Tweak docs some more --- docs/customizing.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/customizing.md b/docs/customizing.md index 25a1ab68..475acaa8 100644 --- a/docs/customizing.md +++ b/docs/customizing.md @@ -53,7 +53,8 @@ The non-decorator approach is still recommended when dealing with lambdas, hooks ### Predicate Hooks -A predicate is a function that takes a type and returns true or false, depending on whether the associated hook can handle the given type. +A _predicate_ is a function that takes a type and returns true or false +depending on whether the associated hook can handle the given type. The {meth}`register_unstructure_hook_func() ` and {meth}`register_structure_hook_func() ` are used to link un/structuring hooks to arbitrary types. These hooks are then called _predicate hooks_, and are very powerful. @@ -99,6 +100,7 @@ Here's an example showing how to use hook factories to apply the `forbid_extra_k >>> from cattrs.gen import make_dict_structure_fn >>> c = Converter() + >>> c.register_structure_hook_factory( ... has, ... lambda cl: make_dict_structure_fn(cl, c, _cattrs_forbid_extra_keys=True) @@ -118,11 +120,12 @@ A complex use case for hook factories is described over at [](usage.md#using-fac #### Use as Decorators -{meth}`Converter.register_unstructure_hook_factory() ` and {meth}`Converter.register_structure_hook_factory() ` can also be used as decorators. +{meth}`register_unstructure_hook_factory() ` and +{meth}`register_structure_hook_factory() ` can also be used as decorators. -When registered via decorators, factory hooks can receive the current converter by exposing an additional required parameter. +When registered via decorators, hook factories can receive the current converter by exposing an additional required parameter. -Here's an example of using an unstructure hook factory to handle unstructuring queues. +Here's an example of using an unstructure hook factory to handle unstructuring [queues](https://docs.python.org/3/library/queue.html#queue.Queue). ```{doctest} >>> from queue import Queue