From 0fc329ee0b303a1fb9593d3d6c9e6f283b4c18fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tin=20Tvrtkovi=C4=87?= Date: Tue, 12 Nov 2024 23:12:25 +0100 Subject: [PATCH] Fix preconf mapping optimizations --- HISTORY.md | 2 + src/cattrs/_compat.py | 1 - src/cattrs/preconf/bson.py | 18 ++++----- src/cattrs/preconf/cbor2.py | 5 +-- src/cattrs/preconf/json.py | 5 ++- src/cattrs/preconf/msgpack.py | 5 +-- src/cattrs/preconf/orjson.py | 9 +++-- src/cattrs/preconf/tomlkit.py | 12 +++--- src/cattrs/preconf/ujson.py | 4 +- tests/test_cols.py | 5 ++- tests/test_preconf.py | 69 ++++++++++++++++++++++++----------- 11 files changed, 81 insertions(+), 54 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 02dfc8c0..f8917d17 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -27,6 +27,8 @@ Our backwards-compatibility policy can be found [here](https://github.com/python ([#598](https://github.com/python-attrs/cattrs/pull/598)) - Literals containing enums are now unstructured properly, and their unstructuring is greatly optimized in the _bson_, stdlib JSON, _cbor2_, _msgpack_, _msgspec_, _orjson_ and _ujson_ preconf converters. ([#598](https://github.com/python-attrs/cattrs/pull/598)) +- Preconf converters now handle dictionaries with literal keys properly. + ([#599](https://github.com/python-attrs/cattrs/pull/599)) - Replace `cattrs.gen.MappingStructureFn` with `cattrs.SimpleStructureHook[In, T]`. - Python 3.13 is now supported. ([#543](https://github.com/python-attrs/cattrs/pull/543) [#547](https://github.com/python-attrs/cattrs/issues/547)) diff --git a/src/cattrs/_compat.py b/src/cattrs/_compat.py index b691a7e7..85b41a95 100644 --- a/src/cattrs/_compat.py +++ b/src/cattrs/_compat.py @@ -252,7 +252,6 @@ def is_literal(_) -> bool: Set = AbcSet -AbstractSet = AbcSet MutableSet = AbcMutableSet Sequence = AbcSequence MutableSequence = AbcMutableSequence diff --git a/src/cattrs/preconf/bson.py b/src/cattrs/preconf/bson.py index ed6e361d..7d398b4a 100644 --- a/src/cattrs/preconf/bson.py +++ b/src/cattrs/preconf/bson.py @@ -1,14 +1,14 @@ """Preconfigured converters for bson.""" from base64 import b85decode, b85encode +from collections.abc import Set from datetime import date, datetime from typing import Any, TypeVar, Union from bson import DEFAULT_CODEC_OPTIONS, CodecOptions, Int64, ObjectId, decode, encode -from cattrs._compat import AbstractSet, is_mapping -from cattrs.gen import make_mapping_structure_fn - +from .._compat import is_mapping, is_subclass +from ..cols import mapping_structure_factory from ..converters import BaseConverter, Converter from ..dispatch import StructureHook from ..fns import identity @@ -69,9 +69,9 @@ def gen_unstructure_mapping(cl: Any, unstructure_to=None): key_handler = str args = getattr(cl, "__args__", None) if args: - if issubclass(args[0], str): + if is_subclass(args[0], str): key_handler = None - elif issubclass(args[0], bytes): + elif is_subclass(args[0], bytes): def key_handler(k): return b85encode(k).decode("utf8") @@ -82,10 +82,10 @@ def key_handler(k): def gen_structure_mapping(cl: Any) -> StructureHook: args = getattr(cl, "__args__", None) - if args and issubclass(args[0], bytes): - h = make_mapping_structure_fn(cl, converter, key_type=Base85Bytes) + if args and is_subclass(args[0], bytes): + h = mapping_structure_factory(cl, converter, key_type=Base85Bytes) else: - h = make_mapping_structure_fn(cl, converter) + h = mapping_structure_factory(cl, converter) return h converter.register_structure_hook(Base85Bytes, lambda v, _: b85decode(v)) @@ -112,7 +112,7 @@ def gen_structure_mapping(cl: Any) -> StructureHook: @wrap(BsonConverter) def make_converter(*args: Any, **kwargs: Any) -> BsonConverter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, **kwargs.get("unstruct_collection_overrides", {}), } res = BsonConverter(*args, **kwargs) diff --git a/src/cattrs/preconf/cbor2.py b/src/cattrs/preconf/cbor2.py index 13e224ef..6341d898 100644 --- a/src/cattrs/preconf/cbor2.py +++ b/src/cattrs/preconf/cbor2.py @@ -1,12 +1,11 @@ """Preconfigured converters for cbor2.""" +from collections.abc import Set from datetime import date, datetime, timezone from typing import Any, TypeVar, Union from cbor2 import dumps, loads -from cattrs._compat import AbstractSet - from ..converters import BaseConverter, Converter from ..fns import identity from ..literals import is_literal_containing_enums @@ -48,7 +47,7 @@ def configure_converter(converter: BaseConverter): @wrap(Cbor2Converter) def make_converter(*args: Any, **kwargs: Any) -> Cbor2Converter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, **kwargs.get("unstruct_collection_overrides", {}), } res = Cbor2Converter(*args, **kwargs) diff --git a/src/cattrs/preconf/json.py b/src/cattrs/preconf/json.py index 2865326f..b6c0ecc2 100644 --- a/src/cattrs/preconf/json.py +++ b/src/cattrs/preconf/json.py @@ -1,11 +1,12 @@ """Preconfigured converters for the stdlib json.""" from base64 import b85decode, b85encode +from collections.abc import Set from datetime import date, datetime from json import dumps, loads from typing import Any, TypeVar, Union -from .._compat import AbstractSet, Counter +from .._compat import Counter from ..converters import BaseConverter, Converter from ..fns import identity from ..literals import is_literal_containing_enums @@ -56,7 +57,7 @@ def configure_converter(converter: BaseConverter): @wrap(JsonConverter) def make_converter(*args: Any, **kwargs: Any) -> JsonConverter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, Counter: dict, **kwargs.get("unstruct_collection_overrides", {}), } diff --git a/src/cattrs/preconf/msgpack.py b/src/cattrs/preconf/msgpack.py index 9549dfcb..4e1bddd5 100644 --- a/src/cattrs/preconf/msgpack.py +++ b/src/cattrs/preconf/msgpack.py @@ -1,12 +1,11 @@ """Preconfigured converters for msgpack.""" +from collections.abc import Set from datetime import date, datetime, time, timezone from typing import Any, TypeVar, Union from msgpack import dumps, loads -from cattrs._compat import AbstractSet - from ..converters import BaseConverter, Converter from ..fns import identity from ..literals import is_literal_containing_enums @@ -55,7 +54,7 @@ def configure_converter(converter: BaseConverter): @wrap(MsgpackConverter) def make_converter(*args: Any, **kwargs: Any) -> MsgpackConverter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, **kwargs.get("unstruct_collection_overrides", {}), } res = MsgpackConverter(*args, **kwargs) diff --git a/src/cattrs/preconf/orjson.py b/src/cattrs/preconf/orjson.py index 6609febd..6e0b6b80 100644 --- a/src/cattrs/preconf/orjson.py +++ b/src/cattrs/preconf/orjson.py @@ -1,6 +1,7 @@ """Preconfigured converters for orjson.""" from base64 import b85decode, b85encode +from collections.abc import Set from datetime import date, datetime from enum import Enum from functools import partial @@ -8,8 +9,8 @@ from orjson import dumps, loads -from .._compat import AbstractSet, is_mapping -from ..cols import is_namedtuple, namedtuple_unstructure_factory +from .._compat import is_subclass +from ..cols import is_mapping, is_namedtuple, namedtuple_unstructure_factory from ..converters import BaseConverter, Converter from ..fns import identity from ..literals import is_literal_containing_enums @@ -56,7 +57,7 @@ def gen_unstructure_mapping(cl: Any, unstructure_to=None): key_handler = str args = getattr(cl, "__args__", None) if args: - if issubclass(args[0], str) and issubclass(args[0], Enum): + if is_subclass(args[0], str) and is_subclass(args[0], Enum): def key_handler(v): return v.value @@ -96,7 +97,7 @@ def key_handler(v): @wrap(OrjsonConverter) def make_converter(*args: Any, **kwargs: Any) -> OrjsonConverter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, **kwargs.get("unstruct_collection_overrides", {}), } res = OrjsonConverter(*args, **kwargs) diff --git a/src/cattrs/preconf/tomlkit.py b/src/cattrs/preconf/tomlkit.py index f940aeac..ace6c360 100644 --- a/src/cattrs/preconf/tomlkit.py +++ b/src/cattrs/preconf/tomlkit.py @@ -1,6 +1,7 @@ """Preconfigured converters for tomlkit.""" from base64 import b85decode, b85encode +from collections.abc import Set from datetime import date, datetime from enum import Enum from operator import attrgetter @@ -9,8 +10,7 @@ from tomlkit import dumps, loads from tomlkit.items import Float, Integer, String -from cattrs._compat import AbstractSet, is_mapping - +from .._compat import is_mapping, is_subclass from ..converters import BaseConverter, Converter from ..strategies import configure_union_passthrough from . import validate_datetime, wrap @@ -48,9 +48,9 @@ def gen_unstructure_mapping(cl: Any, unstructure_to=None): # Currently, tomlkit has inconsistent behavior on 3.11 # so we paper over it here. # https://github.com/sdispater/tomlkit/issues/237 - if issubclass(args[0], str): - key_handler = _enum_value_getter if issubclass(args[0], Enum) else None - elif issubclass(args[0], bytes): + if is_subclass(args[0], str): + key_handler = _enum_value_getter if is_subclass(args[0], Enum) else None + elif is_subclass(args[0], bytes): def key_handler(k: bytes): return b85encode(k).decode("utf8") @@ -77,7 +77,7 @@ def key_handler(k: bytes): @wrap(TomlkitConverter) def make_converter(*args: Any, **kwargs: Any) -> TomlkitConverter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, tuple: list, **kwargs.get("unstruct_collection_overrides", {}), } diff --git a/src/cattrs/preconf/ujson.py b/src/cattrs/preconf/ujson.py index 0c7fec4e..bc9b1084 100644 --- a/src/cattrs/preconf/ujson.py +++ b/src/cattrs/preconf/ujson.py @@ -1,12 +1,12 @@ """Preconfigured converters for ujson.""" from base64 import b85decode, b85encode +from collections.abc import Set from datetime import date, datetime from typing import Any, AnyStr, TypeVar, Union from ujson import dumps, loads -from .._compat import AbstractSet from ..converters import BaseConverter, Converter from ..fns import identity from ..literals import is_literal_containing_enums @@ -55,7 +55,7 @@ def configure_converter(converter: BaseConverter): @wrap(UjsonConverter) def make_converter(*args: Any, **kwargs: Any) -> UjsonConverter: kwargs["unstruct_collection_overrides"] = { - AbstractSet: list, + Set: list, **kwargs.get("unstruct_collection_overrides", {}), } res = UjsonConverter(*args, **kwargs) diff --git a/tests/test_cols.py b/tests/test_cols.py index 61353dd3..92bb6a2b 100644 --- a/tests/test_cols.py +++ b/tests/test_cols.py @@ -1,11 +1,12 @@ """Tests for the `cattrs.cols` module.""" +from collections.abc import Set from typing import Dict from immutables import Map from cattrs import BaseConverter, Converter -from cattrs._compat import AbstractSet, FrozenSet +from cattrs._compat import FrozenSet from cattrs.cols import ( is_any_set, iterable_unstructure_factory, @@ -23,7 +24,7 @@ def test_set_overriding(converter: BaseConverter): lambda t, c: iterable_unstructure_factory(t, c, unstructure_to=sorted), ) - assert converter.unstructure({"c", "b", "a"}, AbstractSet[str]) == ["a", "b", "c"] + assert converter.unstructure({"c", "b", "a"}, Set[str]) == ["a", "b", "c"] assert converter.unstructure(frozenset(["c", "b", "a"]), FrozenSet[str]) == [ "a", "b", diff --git a/tests/test_preconf.py b/tests/test_preconf.py index 2ab0b107..fec750ff 100644 --- a/tests/test_preconf.py +++ b/tests/test_preconf.py @@ -1,4 +1,5 @@ import sys +from collections.abc import Callable, Set from datetime import date, datetime, timezone from enum import Enum, IntEnum, unique from json import dumps as json_dumps @@ -31,8 +32,8 @@ text, ) +from cattrs import Converter from cattrs._compat import ( - AbstractSet, Counter, FrozenSet, Mapping, @@ -40,7 +41,6 @@ MutableSequence, MutableSet, Sequence, - Set, TupleSubscriptable, ) from cattrs.fns import identity @@ -48,6 +48,7 @@ from cattrs.preconf.cbor2 import make_converter as cbor2_make_converter from cattrs.preconf.json import make_converter as json_make_converter from cattrs.preconf.msgpack import make_converter as msgpack_make_converter +from cattrs.preconf.pyyaml import make_converter as pyyaml_make_converter from cattrs.preconf.tomlkit import make_converter as tomlkit_make_converter from cattrs.preconf.ujson import make_converter as ujson_make_converter @@ -301,7 +302,7 @@ def test_stdlib_json_converter(everything: Everything): @given(everythings()) def test_stdlib_json_converter_unstruct_collection_overrides(everything: Everything): - converter = json_make_converter(unstruct_collection_overrides={AbstractSet: sorted}) + converter = json_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -395,9 +396,7 @@ def test_ujson_converter(everything: Everything): ) ) def test_ujson_converter_unstruct_collection_overrides(everything: Everything): - converter = ujson_make_converter( - unstruct_collection_overrides={AbstractSet: sorted} - ) + converter = ujson_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -484,9 +483,7 @@ def test_orjson_converter(everything: Everything, detailed_validation: bool): def test_orjson_converter_unstruct_collection_overrides(everything: Everything): from cattrs.preconf.orjson import make_converter as orjson_make_converter - converter = orjson_make_converter( - unstruct_collection_overrides={AbstractSet: sorted} - ) + converter = orjson_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -568,9 +565,7 @@ def test_msgpack_converter(everything: Everything): @given(everythings(min_int=-9223372036854775808, max_int=18446744073709551615)) def test_msgpack_converter_unstruct_collection_overrides(everything: Everything): - converter = msgpack_make_converter( - unstruct_collection_overrides={AbstractSet: sorted} - ) + converter = msgpack_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -663,7 +658,7 @@ def test_bson_converter(everything: Everything, detailed_validation: bool): ) ) def test_bson_converter_unstruct_collection_overrides(everything: Everything): - converter = bson_make_converter(unstruct_collection_overrides={AbstractSet: sorted}) + converter = bson_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -755,9 +750,7 @@ def test_tomlkit_converter(everything: Everything, detailed_validation: bool): ) ) def test_tomlkit_converter_unstruct_collection_overrides(everything: Everything): - converter = tomlkit_make_converter( - unstruct_collection_overrides={AbstractSet: sorted} - ) + converter = tomlkit_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -797,9 +790,7 @@ def test_cbor2_converter(everything: Everything): @given(everythings(min_int=-9223372036854775808, max_int=18446744073709551615)) def test_cbor2_converter_unstruct_collection_overrides(everything: Everything): - converter = cbor2_make_converter( - unstruct_collection_overrides={AbstractSet: sorted} - ) + converter = cbor2_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -856,9 +847,7 @@ def test_msgspec_json_unstruct_collection_overrides(everything: Everything): """Ensure collection overrides work.""" from cattrs.preconf.msgspec import make_converter as msgspec_make_converter - converter = msgspec_make_converter( - unstruct_collection_overrides={AbstractSet: sorted} - ) + converter = msgspec_make_converter(unstruct_collection_overrides={Set: sorted}) raw = converter.unstructure(everything) assert raw["a_set"] == sorted(raw["a_set"]) assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) @@ -913,3 +902,39 @@ def test_msgspec_efficient_enum(): converter.get_unstructure_hook(fields(Everything).a_literal_with_bare.type) == identity ) + + +@pytest.mark.parametrize( + "converter_factory", + [ + bson_make_converter, + cbor2_make_converter, + json_make_converter, + msgpack_make_converter, + tomlkit_make_converter, + ujson_make_converter, + pyyaml_make_converter, + ], +) +def test_literal_dicts(converter_factory: Callable[[], Converter]): + """Dicts with keys that aren't subclasses of `type` work.""" + converter = converter_factory() + + assert converter.structure({"a": 1}, Dict[Literal["a"], int]) == {"a": 1} + assert converter.unstructure({"a": 1}, Dict[Literal["a"], int]) == {"a": 1} + + +@pytest.mark.skipif(NO_ORJSON, reason="orjson not available") +def test_literal_dicts_orjson(): + """Dicts with keys that aren't subclasses of `type` work.""" + from cattrs.preconf.orjson import make_converter as orjson_make_converter + + test_literal_dicts(orjson_make_converter) + + +@pytest.mark.skipif(NO_MSGSPEC, reason="msgspec not available") +def test_literal_dicts_msgspec(): + """Dicts with keys that aren't subclasses of `type` work.""" + from cattrs.preconf.msgspec import make_converter as msgspec_make_converter + + test_literal_dicts(msgspec_make_converter)