From 1ea99bc462d64bc5a37c67bc28e56a54a07951a1 Mon Sep 17 00:00:00 2001 From: Eric Nielsen <4120606+ericbn@users.noreply.github.com> Date: Wed, 12 Apr 2023 15:18:52 -0500 Subject: [PATCH 1/2] Allow overriding unstruct_collection_overrides Fixes #350 --- src/cattrs/preconf/bson.py | 2 +- src/cattrs/preconf/cbor2.py | 2 +- src/cattrs/preconf/json.py | 2 +- src/cattrs/preconf/msgpack.py | 2 +- src/cattrs/preconf/orjson.py | 2 +- src/cattrs/preconf/pyyaml.py | 2 +- src/cattrs/preconf/tomlkit.py | 2 +- src/cattrs/preconf/ujson.py | 2 +- tests/test_preconf.py | 109 ++++++++++++++++++++++++++++++++++ 9 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src/cattrs/preconf/bson.py b/src/cattrs/preconf/bson.py index f6d56582..b9333219 100644 --- a/src/cattrs/preconf/bson.py +++ b/src/cattrs/preconf/bson.py @@ -88,8 +88,8 @@ def gen_structure_mapping(cl: Any): def make_converter(*args, **kwargs) -> BsonConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = BsonConverter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/cbor2.py b/src/cattrs/preconf/cbor2.py index 5a0d4d6f..e198ba8c 100644 --- a/src/cattrs/preconf/cbor2.py +++ b/src/cattrs/preconf/cbor2.py @@ -34,8 +34,8 @@ def configure_converter(converter: BaseConverter): def make_converter(*args, **kwargs) -> Cbor2Converter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = Cbor2Converter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/json.py b/src/cattrs/preconf/json.py index 2de70774..a391f45c 100644 --- a/src/cattrs/preconf/json.py +++ b/src/cattrs/preconf/json.py @@ -38,9 +38,9 @@ def configure_converter(converter: BaseConverter): def make_converter(*args, **kwargs) -> JsonConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, Counter: dict, + **kwargs.get("unstruct_collection_overrides", {}), } res = JsonConverter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/msgpack.py b/src/cattrs/preconf/msgpack.py index 54b8bad2..21ed179b 100644 --- a/src/cattrs/preconf/msgpack.py +++ b/src/cattrs/preconf/msgpack.py @@ -34,8 +34,8 @@ def configure_converter(converter: BaseConverter): def make_converter(*args, **kwargs) -> MsgpackConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = MsgpackConverter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/orjson.py b/src/cattrs/preconf/orjson.py index fe0b143b..b856b974 100644 --- a/src/cattrs/preconf/orjson.py +++ b/src/cattrs/preconf/orjson.py @@ -68,8 +68,8 @@ def key_handler(v): def make_converter(*args, **kwargs) -> OrjsonConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = OrjsonConverter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/pyyaml.py b/src/cattrs/preconf/pyyaml.py index a2fd7c82..cf132991 100644 --- a/src/cattrs/preconf/pyyaml.py +++ b/src/cattrs/preconf/pyyaml.py @@ -35,8 +35,8 @@ def configure_converter(converter: BaseConverter): def make_converter(*args, **kwargs) -> PyyamlConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), FrozenSetSubscriptable: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = PyyamlConverter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/tomlkit.py b/src/cattrs/preconf/tomlkit.py index 3c103a3e..80b9f8af 100644 --- a/src/cattrs/preconf/tomlkit.py +++ b/src/cattrs/preconf/tomlkit.py @@ -67,9 +67,9 @@ def key_handler(k: bytes): def make_converter(*args, **kwargs) -> TomlkitConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, tuple: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = TomlkitConverter(*args, **kwargs) configure_converter(res) diff --git a/src/cattrs/preconf/ujson.py b/src/cattrs/preconf/ujson.py index c340ad67..22407e99 100644 --- a/src/cattrs/preconf/ujson.py +++ b/src/cattrs/preconf/ujson.py @@ -39,8 +39,8 @@ def configure_converter(converter: BaseConverter): def make_converter(*args, **kwargs) -> UjsonConverter: kwargs["unstruct_collection_overrides"] = { - **kwargs.get("unstruct_collection_overrides", {}), AbstractSet: list, + **kwargs.get("unstruct_collection_overrides", {}), } res = UjsonConverter(*args, **kwargs) configure_converter(res) diff --git a/tests/test_preconf.py b/tests/test_preconf.py index 976cf39e..5f57ece7 100644 --- a/tests/test_preconf.py +++ b/tests/test_preconf.py @@ -26,8 +26,10 @@ ) from cattrs._compat import ( + AbstractSet, Counter, FrozenSet, + FrozenSetSubscriptable, Mapping, MutableMapping, MutableSequence, @@ -173,6 +175,15 @@ def test_stdlib_json_converter(everything: Everything): assert converter.loads(converter.dumps(everything), Everything) == everything +@given(everythings()) +def test_stdlib_json_converter_unstruct_collection_overrides(everything: Everything): + converter = json_make_converter(unstruct_collection_overrides={AbstractSet: sorted}) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @given( everythings( min_int=-9223372036854775808, max_int=9223372036854775807, allow_inf=False @@ -198,6 +209,21 @@ def test_ujson_converter(everything: Everything): assert converter.loads(raw, Everything) == everything +@given( + everythings( + min_int=-9223372036854775808, max_int=9223372036854775807, allow_inf=False + ) +) +def test_ujson_converter_unstruct_collection_overrides(everything: Everything): + converter = ujson_make_converter( + unstruct_collection_overrides={AbstractSet: sorted} + ) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @pytest.mark.skipif(python_implementation() == "PyPy", reason="no orjson on PyPy") @given( everythings( @@ -231,6 +257,24 @@ def test_orjson_converter(everything: Everything, detailed_validation: bool): assert converter.loads(raw, Everything) == everything +@pytest.mark.skipif(python_implementation() == "PyPy", reason="no orjson on PyPy") +@given( + everythings( + min_int=-9223372036854775808, max_int=9223372036854775807, allow_inf=False + ) +) +def test_orjson_converter_unstruct_collection_overrides(everything: Everything): + from cattrs.preconf.orjson import make_converter as orjson_make_converter + + converter = orjson_make_converter( + unstruct_collection_overrides={AbstractSet: sorted} + ) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @given(everythings(min_int=-9223372036854775808, max_int=18446744073709551615)) def test_msgpack(everything: Everything): from msgpack import dumps as msgpack_dumps @@ -251,6 +295,17 @@ def test_msgpack_converter(everything: Everything): assert converter.loads(raw, Everything, strict_map_key=False) == everything +@given(everythings(min_int=-9223372036854775808, max_int=18446744073709551615)) +def test_msgpack_converter_unstruct_collection_overrides(everything: Everything): + converter = msgpack_make_converter( + unstruct_collection_overrides={AbstractSet: sorted} + ) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @given( everythings( min_int=-9223372036854775808, @@ -294,6 +349,22 @@ def test_bson_converter(everything: Everything, detailed_validation: bool): ) +@given( + everythings( + min_int=-9223372036854775808, + max_int=9223372036854775807, + allow_null_bytes_in_keys=False, + allow_datetime_microseconds=False, + ) +) +def test_bson_converter_unstruct_collection_overrides(everything: Everything): + converter = bson_make_converter(unstruct_collection_overrides={AbstractSet: sorted}) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @given(everythings()) def test_pyyaml(everything: Everything): from yaml import safe_dump, safe_load @@ -311,6 +382,15 @@ def test_pyyaml_converter(everything: Everything): assert converter.loads(raw, Everything) == everything +@given(everythings()) +def test_pyyaml_converter_unstruct_collection_overrides(everything: Everything): + converter = pyyaml_make_converter( + unstruct_collection_overrides={FrozenSetSubscriptable: sorted} + ) + raw = converter.unstructure(everything) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + @given( everythings( min_key_length=1, @@ -345,6 +425,24 @@ def test_tomlkit_converter(everything: Everything, detailed_validation: bool): assert converter.loads(raw, Everything) == everything +@given( + everythings( + min_key_length=1, + allow_null_bytes_in_keys=False, + key_blacklist_characters=['"', "\\"], + allow_control_characters_in_values=False, + ) +) +def test_tomlkit_converter_unstruct_collection_overrides(everything: Everything): + converter = tomlkit_make_converter( + unstruct_collection_overrides={AbstractSet: sorted} + ) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) + + def test_bson_objectid(): """BSON ObjectIds are supported by default.""" converter = bson_make_converter() @@ -368,3 +466,14 @@ def test_cbor2_converter(everything: Everything): converter = cbor2_make_converter() raw = converter.dumps(everything) assert converter.loads(raw, Everything) == everything + + +@given(everythings(min_int=-9223372036854775808, max_int=18446744073709551615)) +def test_cbor2_converter_unstruct_collection_overrides(everything: Everything): + converter = cbor2_make_converter( + unstruct_collection_overrides={AbstractSet: sorted} + ) + raw = converter.unstructure(everything) + assert raw["a_set"] == sorted(raw["a_set"]) + assert raw["a_mutable_set"] == sorted(raw["a_mutable_set"]) + assert raw["a_frozenset"] == sorted(raw["a_frozenset"]) From aeed3d653d5c5e08d2f10a0683a5114c83939300 Mon Sep 17 00:00:00 2001 From: Eric Nielsen <4120606+ericbn@users.noreply.github.com> Date: Fri, 14 Apr 2023 08:38:59 -0500 Subject: [PATCH 2/2] Update HISTORY.md --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index c3281ef3..cf2fd351 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -14,6 +14,8 @@ ([#81](https://github.com/python-attrs/cattrs/issues/81)) - Add `cbor2` serialization library to the `cattr.preconf` package. - Add optional dependencies for `cattrs.preconf` third-party libraries. ([#337](https://github.com/python-attrs/cattrs/pull/337)) +- All preconf converters now allow overriding the default `unstruct_collection_overrides` in `make_converter`. + ([#350](https://github.com/python-attrs/cattrs/issues/350) [#353](https://github.com/python-attrs/cattrs/pull/353)) ## 22.2.0 (2022-10-03)