From 30a7ee73632f0615d7cda45cea984bfaa2b6cf58 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Sun, 3 May 2020 18:37:21 +0300 Subject: [PATCH 1/7] Add types_styles parameter --- json_to_models/dynamic_typing/base.py | 25 ++++++++++--- json_to_models/dynamic_typing/complex.py | 35 +++++++++++-------- json_to_models/dynamic_typing/models_meta.py | 16 +++++---- .../dynamic_typing/string_serializable.py | 4 +-- json_to_models/dynamic_typing/typing.py | 11 +++--- .../test_models_code_generator.py | 16 ++++----- testing_tools/real_apis/pathofexile.py | 4 +-- 7 files changed, 70 insertions(+), 41 deletions(-) diff --git a/json_to_models/dynamic_typing/base.py b/json_to_models/dynamic_typing/base.py index 672cee8..151f973 100644 --- a/json_to_models/dynamic_typing/base.py +++ b/json_to_models/dynamic_typing/base.py @@ -1,5 +1,5 @@ from inspect import isclass -from typing import Any, Generator, Iterable, List, Tuple, Union +from typing import Any, Dict, Generator, Iterable, List, Tuple, Type, Union ImportPathList = List[Tuple[str, Union[Iterable[str], str, None]]] @@ -21,14 +21,29 @@ def replace(self, t: Union['MetaData', List['MetaData']], **kwargs) -> 'BaseType """ raise NotImplementedError() - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: """ Return typing code that represents this metadata and import path of classes that are used in this code + :param types_style: Hints for .to_typing_code() for different type wrappers :return: ((module_name, (class_name, ...)), code) """ raise NotImplementedError() + @classmethod + def get_kwargs_for_type( + cls, + t: Union['BaseType', Type['BaseType']], + types_style: Dict[Union['BaseType', Type['BaseType']], dict] + ) -> dict: + t_cls = t if isclass(t) else type(t) + mro = t_cls.__mro__ + for base in mro: + kwargs = types_style.get(base, ...) + if kwargs is not Ellipsis: + return kwargs + def to_hash_string(self) -> str: """ Return unique string that can be used to generate hash of type instance. @@ -71,7 +86,8 @@ def __iter__(self) -> Iterable['MetaData']: def replace(self, t: 'MetaData', **kwargs) -> 'UnknownType': return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: return ([('typing', 'Any')], 'Any') def to_hash_string(self) -> str: @@ -90,7 +106,8 @@ def __iter__(self) -> Iterable['MetaData']: def replace(self, t: 'MetaData', **kwargs) -> 'NoneType': return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: return ([], 'None') def to_hash_string(self) -> str: diff --git a/json_to_models/dynamic_typing/complex.py b/json_to_models/dynamic_typing/complex.py index 6bfd48f..fea1f12 100644 --- a/json_to_models/dynamic_typing/complex.py +++ b/json_to_models/dynamic_typing/complex.py @@ -1,5 +1,6 @@ +from functools import partial from itertools import chain -from typing import Iterable, List, Tuple, Union +from typing import Dict, Iterable, List, Tuple, Type, Union from .base import BaseType, ImportPathList, MetaData, get_hash_string from .typing import metadata_to_typing @@ -106,11 +107,12 @@ def replace(self, t: Union['MetaData', List['MetaData']], index=None, **kwargs) raise ValueError(f"Unsupported arguments: t={t} index={index} kwargs={kwargs}") return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = zip(*map(metadata_to_typing, self)) + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = zip(*map(partial(metadata_to_typing, types_style=types_style), self)) nested = ", ".join(nested) return ( - list(chain(*imports)), + list(chain.from_iterable(imports)), f"[{nested}]" ) @@ -123,8 +125,9 @@ class DOptional(SingleType): Field of this type may not be presented in JSON object """ - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = metadata_to_typing(self.type) + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = metadata_to_typing(self.type, types_style=types_style) return ( [*imports, ('typing', 'Optional')], f"Optional[{nested}]" @@ -165,8 +168,9 @@ def _extract_nested_types(self): else: yield t - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = super().to_typing_code() + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = super().to_typing_code(types_style) return ( [*imports, ('typing', 'Union')], "Union" + nested @@ -174,8 +178,9 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: class DTuple(ComplexType): - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = super().to_typing_code() + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = super().to_typing_code(types_style) return ( [*imports, ('typing', 'Tuple')], "Tuple" + nested @@ -183,8 +188,9 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: class DList(SingleType): - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = metadata_to_typing(self.type) + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = metadata_to_typing(self.type, types_style=types_style) return ( [*imports, ('typing', 'List')], f"List[{nested}]" @@ -193,8 +199,9 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: class DDict(SingleType): # Dict is single type because keys of JSON dict are always strings. - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = metadata_to_typing(self.type) + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = metadata_to_typing(self.type, types_style=types_style) return ( [*imports, ('typing', 'Dict')], f"Dict[str, {nested}]" diff --git a/json_to_models/dynamic_typing/models_meta.py b/json_to_models/dynamic_typing/models_meta.py index 895086d..119f9fd 100644 --- a/json_to_models/dynamic_typing/models_meta.py +++ b/json_to_models/dynamic_typing/models_meta.py @@ -1,8 +1,9 @@ import threading -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Tuple, Type, Union import inflection +from . import BaseType from .base import ImportPathList, MetaData from .complex import SingleType from ..utils import distinct_words @@ -94,7 +95,8 @@ def add_child_ref(self, ptr: 'ModelPtr'): def remove_child_ref(self, ptr: 'ModelPtr'): self.child_pointers.remove(ptr) - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: if self.name is None: raise ValueError('Model without name can not be typed') return [], self.name @@ -130,8 +132,9 @@ def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr': self.parent.add_child_ref(self) return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: - return AbsoluteModelRef(self.type).to_typing_code() + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + return AbsoluteModelRef(self.type).to_typing_code(types_style) def _to_hash_string(self) -> str: return f"{type(self).__name__}_#{self.type.index}" @@ -187,7 +190,8 @@ def inject(cls, patches: ContextInjectionType): def __init__(self, model: ModelMeta): self.model = model - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: context_data = self.Context.data.context if context_data: model_path = context_data.get(self.model, "") @@ -195,6 +199,6 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: model_path = model_path.name else: model_path = "" - imports, model = self.model.to_typing_code() + imports, model = self.model.to_typing_code(types_style) s = ".".join(filter(None, (model_path, model))) return imports, f"'{s}'" diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index c89c833..a9effe3 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -1,5 +1,5 @@ from itertools import permutations -from typing import ClassVar, Collection, Iterable, List, Set, Tuple, Type +from typing import ClassVar, Collection, Dict, Iterable, List, Set, Tuple, Type, Union from .base import BaseType, ImportPathList @@ -30,7 +30,7 @@ def to_representation(self) -> str: raise NotImplementedError() @classmethod - def to_typing_code(cls) -> Tuple[ImportPathList, str]: + def to_typing_code(cls, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) -> Tuple[ImportPathList, str]: """ Unlike other BaseType's subclasses it's a class method because StringSerializable instance is not parameterized as a metadata instance but contains actual data diff --git a/json_to_models/dynamic_typing/typing.py b/json_to_models/dynamic_typing/typing.py index 3cb51d3..6b6e5c6 100644 --- a/json_to_models/dynamic_typing/typing.py +++ b/json_to_models/dynamic_typing/typing.py @@ -1,13 +1,14 @@ import operator from datetime import date, datetime, time from inspect import isclass -from typing import Dict, Set, Tuple +from typing import Dict, Set, Tuple, Type, Union -from .base import ImportPathList, MetaData +from .base import BaseType, ImportPathList, MetaData from .string_serializable import StringSerializable -def metadata_to_typing(t: MetaData) -> Tuple[ImportPathList, str]: +def metadata_to_typing(t: MetaData, types_style: Dict[Union['BaseType', Type['BaseType']], dict] = None) \ + -> Tuple[ImportPathList, str]: """ Shortcut function to call ``to_typing_code`` method of BaseType instances or return name of type otherwise :param t: @@ -15,7 +16,7 @@ def metadata_to_typing(t: MetaData) -> Tuple[ImportPathList, str]: """ if isclass(t): if issubclass(t, StringSerializable): - return t.to_typing_code() + return t.to_typing_code(types_style) else: imports = [] if issubclass(t, (date, datetime, time)): @@ -24,7 +25,7 @@ def metadata_to_typing(t: MetaData) -> Tuple[ImportPathList, str]: elif isinstance(t, dict): raise ValueError("Can not convert dict instance to typing code. It should be wrapped into ModelMeta instance") else: - return t.to_typing_code() + return t.to_typing_code(types_style) def compile_imports(imports: ImportPathList) -> str: diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py index 3f3cff5..076658e 100644 --- a/test/test_code_generation/test_models_code_generator.py +++ b/test/test_code_generation/test_models_code_generator.py @@ -192,20 +192,20 @@ def test_absolute_model_ref(): test_model = ModelMeta({"field": int}, "A") test_model.name = "test_model" test_ptr = ModelPtr(test_model) - assert test_ptr.to_typing_code()[1] == "'TestModel'" + assert test_ptr.to_typing_code({})[1] == "'TestModel'" with AbsoluteModelRef.inject({test_model: "Parent"}): - assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" - assert test_ptr.to_typing_code()[1] == "'TestModel'" + assert test_ptr.to_typing_code({})[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'TestModel'" with AbsoluteModelRef.inject({test_model: "Parent"}): - assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'Parent.TestModel'" with AbsoluteModelRef.inject({test_model: "AnotherParent"}): - assert test_ptr.to_typing_code()[1] == "'AnotherParent.TestModel'" - assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'AnotherParent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'Parent.TestModel'" wrapper = DList(DList(test_ptr)) - assert wrapper.to_typing_code()[1] == "List[List['TestModel']]" + assert wrapper.to_typing_code({})[1] == "List[List['TestModel']]" with AbsoluteModelRef.inject({test_model: test_model}): - assert wrapper.to_typing_code()[1] == "List[List['TestModel.TestModel']]" + assert wrapper.to_typing_code({})[1] == "List[List['TestModel.TestModel']]" test_unicode_data = [ diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index 5ee78af..0443109 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -6,8 +6,8 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.pydantic import PydanticModelCodeGenerator from json_to_models.models.structure import compose_models_flat from json_to_models.registry import ModelRegistry from testing_tools.real_apis import dump_response @@ -40,7 +40,7 @@ def main(): # print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) # print("=" * 20) - print(generate_code(structure, AttrsModelCodeGenerator)) + print(generate_code(structure, PydanticModelCodeGenerator)) print(f"{(datetime.now() - start_t).total_seconds():.4f} seconds") From 75343630f1cbea60b4c1f763ee1d32a805cb06ca Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Sun, 3 May 2020 19:00:14 +0300 Subject: [PATCH 2/7] Pydantic: rewrite string_serializable replace with actual types using types_style --- json_to_models/dynamic_typing/base.py | 7 +++-- .../dynamic_typing/string_serializable.py | 9 ++++++ json_to_models/dynamic_typing/typing.py | 1 + json_to_models/models/base.py | 18 ++++++++---- json_to_models/models/pydantic.py | 29 ++++++++++--------- 5 files changed, 43 insertions(+), 21 deletions(-) diff --git a/json_to_models/dynamic_typing/base.py b/json_to_models/dynamic_typing/base.py index 151f973..d8533cb 100644 --- a/json_to_models/dynamic_typing/base.py +++ b/json_to_models/dynamic_typing/base.py @@ -40,9 +40,10 @@ def get_kwargs_for_type( t_cls = t if isclass(t) else type(t) mro = t_cls.__mro__ for base in mro: - kwargs = types_style.get(base, ...) - if kwargs is not Ellipsis: - return kwargs + options = types_style.get(base, ...) + if options is not Ellipsis: + return options + return {} def to_hash_string(self) -> str: """ diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index a9effe3..39c422e 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -8,6 +8,10 @@ class StringSerializable(BaseType): """ Mixin for classes which are used to (de-)serialize some values in a string form """ + + class TypeStyle: + use_actual_type = 'use_actual_type' + actual_type: ClassVar[Type] @classmethod @@ -36,6 +40,11 @@ def to_typing_code(cls, types_style: Dict[Union['BaseType', Type['BaseType']], d as a metadata instance but contains actual data """ cls_name = cls.__name__ + options = cls.get_kwargs_for_type(cls, types_style) + if options.get(cls.TypeStyle.use_actual_type): + if cls.actual_type.__module__ != 'builtins': + return [(cls.actual_type.__module__, cls.actual_type.__name__)], cls.actual_type.__name__ + return [], cls.actual_type.__name__ return [('json_to_models.dynamic_typing', cls_name)], cls_name def __iter__(self): diff --git a/json_to_models/dynamic_typing/typing.py b/json_to_models/dynamic_typing/typing.py index 6b6e5c6..68c0145 100644 --- a/json_to_models/dynamic_typing/typing.py +++ b/json_to_models/dynamic_typing/typing.py @@ -14,6 +14,7 @@ def metadata_to_typing(t: MetaData, types_style: Dict[Union['BaseType', Type['Ba :param t: :return: """ + types_style = types_style or {} if isclass(t): if issubclass(t, StringSerializable): return t.to_typing_code(types_style) diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index d9eaf15..4f41e17 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -1,6 +1,6 @@ import keyword import re -from typing import Iterable, List, Tuple, Type +from typing import Dict, Iterable, List, Tuple, Type, Union import inflection from jinja2 import Template @@ -10,7 +10,7 @@ from .string_converters import get_string_field_paths from .structure import sort_fields from .utils import indent -from ..dynamic_typing import (AbsoluteModelRef, ImportPathList, MetaData, +from ..dynamic_typing import (AbsoluteModelRef, BaseType, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing) from ..utils import cached_method @@ -73,8 +73,16 @@ class {{ name }}{% if bases %}({{ bases }}){% endif %}: STR_CONVERT_DECORATOR = template("convert_strings({{ str_fields }}{%% if kwargs %%}, %s{%% endif %%})" % KWAGRS_TEMPLATE) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") - - def __init__(self, model: ModelMeta, post_init_converters=False, convert_unicode=True): + default_types_style = {} + + def __init__( + self, + model: ModelMeta, + post_init_converters=False, + convert_unicode=True, + types_style: Dict[Union['BaseType', Type['BaseType']], dict] = None + ): + self.types_style = types_style if types_style is not None else self.default_types_style self.model = model self.post_init_converters = post_init_converters self.convert_unicode = convert_unicode @@ -133,7 +141,7 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP :param optional: Is field optional :return: imports, field data """ - imports, typing = metadata_to_typing(meta) + imports, typing = metadata_to_typing(meta, types_style=self.types_style) data = { "name": self.convert_field_name(name), diff --git a/json_to_models/models/pydantic.py b/json_to_models/models/pydantic.py index 2692ad4..6aca2d8 100644 --- a/json_to_models/models/pydantic.py +++ b/json_to_models/models/pydantic.py @@ -1,8 +1,17 @@ -from inspect import isclass from typing import List, Optional, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, sort_kwargs, template -from ..dynamic_typing import BaseType, DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, Null, StringSerializable, Unknown +from ..dynamic_typing import ( + DDict, + DList, + DOptional, + ImportPathList, + MetaData, + ModelMeta, + Null, + StringSerializable, + Unknown +) DEFAULT_ORDER = ( "*", @@ -12,6 +21,11 @@ class PydanticModelCodeGenerator(GenericModelCodeGenerator): PYDANTIC_FIELD = template("Field({{ default }}{% if kwargs %}, KWAGRS_TEMPLATE{% endif %})" .replace('KWAGRS_TEMPLATE', KWAGRS_TEMPLATE)) + default_types_style = { + StringSerializable: { + StringSerializable.TypeStyle.use_actual_type: True + } + } def __init__(self, model: ModelMeta, convert_unicode=True): """ @@ -51,7 +65,6 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP :param optional: Is field optional :return: imports, field data """ - _, meta = self.replace_string_serializable(meta) imports, data = super().field_data(name, meta, optional) default: Optional[str] = None body_kwargs = {} @@ -74,13 +87,3 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP elif default is not None: data["body"] = default return imports, data - - def replace_string_serializable(self, t: MetaData) -> Tuple[bool, MetaData]: - if isclass(t) and issubclass(t, StringSerializable): - return True, t.actual_type - elif isinstance(t, BaseType): - for i, sub_type in enumerate(t): - replaced, new_type = self.replace_string_serializable(sub_type) - if replaced: - t.replace(new_type, index=i) - return False, t From ba98defb3358fb4b99f63824bdc513f48c61481c Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Sun, 3 May 2020 21:00:00 +0300 Subject: [PATCH 3/7] Integrate StringLiteral into core --- json_to_models/dynamic_typing/__init__.py | 2 +- json_to_models/dynamic_typing/complex.py | 94 +++++++++++++++++++--- json_to_models/generator.py | 22 ++++- json_to_models/models/string_converters.py | 16 +++- test/test_generator/test_detect_type.py | 20 +++-- 5 files changed, 133 insertions(+), 21 deletions(-) diff --git a/json_to_models/dynamic_typing/__init__.py b/json_to_models/dynamic_typing/__init__.py index a6a7ebf..1790b6d 100644 --- a/json_to_models/dynamic_typing/__init__.py +++ b/json_to_models/dynamic_typing/__init__.py @@ -1,7 +1,7 @@ from .base import ( BaseType, ImportPathList, MetaData, Null, Unknown, get_hash_string ) -from .complex import ComplexType, DDict, DList, DOptional, DTuple, DUnion, SingleType +from .complex import ComplexType, DDict, DList, DOptional, DTuple, DUnion, SingleType, StringLiteral from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr from .string_datetime import IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes from .string_serializable import ( diff --git a/json_to_models/dynamic_typing/complex.py b/json_to_models/dynamic_typing/complex.py index fea1f12..5ae090e 100644 --- a/json_to_models/dynamic_typing/complex.py +++ b/json_to_models/dynamic_typing/complex.py @@ -1,6 +1,6 @@ from functools import partial from itertools import chain -from typing import Dict, Iterable, List, Tuple, Type, Union +from typing import AbstractSet, Dict, Iterable, List, Tuple, Type, Union from .base import BaseType, ImportPathList, MetaData, get_hash_string from .typing import metadata_to_typing @@ -142,20 +142,48 @@ class DUnion(ComplexType): def __init__(self, *types: Union[type, BaseType, dict]): hashes = set() unique_types = [] + use_literals = True + str_literals = set() + # Ensure that types in union are unique - for t in types: - if isinstance(t, DUnion): - # Merging nested DUnions - for t2 in list(t._extract_nested_types()): - h = get_hash_string(t2) - if h not in hashes: - unique_types.append(t2) - hashes.add(h) + + def handle_type(t, use_literals): + if t is str: + use_literals = False + + if isinstance(t, StringLiteral): + if not use_literals: + return + + if t.overflowed: + use_literals = False + else: + str_literals.update(t.literals) + else: h = get_hash_string(t) if h not in hashes: unique_types.append(t) hashes.add(h) + return use_literals + + for t in types: + if isinstance(t, DUnion): + # Merging nested DUnions + for t2 in list(t._extract_nested_types()): + use_literals = handle_type(t2, use_literals) and use_literals + else: + use_literals = handle_type(t, use_literals) and use_literals + + if str_literals and use_literals: + literal = StringLiteral(str_literals) + if literal.overflowed: + use_literals = False + else: + unique_types.append(literal) + + if not use_literals: + handle_type(str, use_literals=False) super().__init__(*unique_types) def _extract_nested_types(self): @@ -206,3 +234,51 @@ def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], [*imports, ('typing', 'Dict')], f"Dict[str, {nested}]" ) + + +class StringLiteral(BaseType): + MAX_LITERALS = 15 + MAX_STRING_LENGTH = 20 + __slots__ = ["_literals", "_hash", "_overflow"] + + def __init__(self, literals: AbstractSet[str]): + self._overflow = ( + len(literals) > self.MAX_LITERALS + or any(map(lambda s: len(s) >= self.MAX_STRING_LENGTH, literals)) + ) + self._literals = frozenset() if self._overflow else literals + + def __iter__(self) -> Iterable['MetaData']: + return iter(()) + + def __str__(self): + return f"{type(self).__name__}[{self._repr_literals()}]" + + def __repr__(self): + return f"<{type(self).__name__} [{self._repr_literals()}]>" + + def __eq__(self, other): + return type(other) is type(self) and self._literals == other._literals + + def replace(self, t: 'MetaData', **kwargs) -> 'StringLiteral': + return self + + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + return [], 'str' + + def _to_hash_string(self) -> str: + return f"{type(self).__name__}/{self._repr_literals()}" + + @property + def literals(self): + return self._literals + + @property + def overflowed(self): + return self._overflow + + def _repr_literals(self): + if self._overflow: + return '...' + return ','.join(self._literals) diff --git a/json_to_models/generator.py b/json_to_models/generator.py index 7e960be..6b7aac9 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -3,8 +3,22 @@ from unidecode import unidecode -from .dynamic_typing import (ComplexType, DDict, DList, DOptional, DUnion, MetaData, ModelPtr, Null, SingleType, - StringSerializable, StringSerializableRegistry, Unknown, registry) +from .dynamic_typing import ( + ComplexType, + DDict, + DList, + DOptional, + DUnion, + MetaData, + ModelPtr, + Null, + SingleType, + StringLiteral, + StringSerializable, + StringSerializableRegistry, + Unknown, + registry +) _static_types = {float, bool, int} @@ -108,7 +122,7 @@ def _detect_type(self, value, convert_dict=True) -> MetaData: except ValueError: continue return t - return str + return StringLiteral({value}) def merge_field_sets(self, field_sets: List[MetaData]) -> MetaData: """ @@ -199,7 +213,7 @@ def _optimize_union(self, t: DUnion): str_types: List[Union[type, StringSerializable]] = [] types_to_merge: List[dict] = [] list_types: List[DList] = [] - dict_types: List[DList] = [] + dict_types: List[DDict] = [] other_types: List[MetaData] = [] for item in t.types: if isinstance(item, DOptional): diff --git a/json_to_models/models/string_converters.py b/json_to_models/models/string_converters.py index 04326b7..988abdd 100644 --- a/json_to_models/models/string_converters.py +++ b/json_to_models/models/string_converters.py @@ -3,8 +3,18 @@ from typing import Any, Callable, List, Optional, Tuple from . import ClassType -from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, MetaData, ModelMeta, ModelPtr, - StringSerializable) +from ..dynamic_typing import ( + BaseType, + DDict, + DList, + DOptional, + DUnion, + MetaData, + ModelMeta, + ModelPtr, + StringLiteral, + StringSerializable +) from ..dynamic_typing.base import NoneType @@ -168,6 +178,8 @@ def get_string_field_paths(model: ModelMeta) -> List[Tuple[str, List[str]]]: break elif cls is NoneType: continue + elif cls in (StringLiteral,): + continue else: raise TypeError(f"Unsupported meta-type for converter path {cls}") diff --git a/test/test_generator/test_detect_type.py b/test/test_generator/test_detect_type.py index 3404757..b2398fe 100644 --- a/test/test_generator/test_detect_type.py +++ b/test/test_generator/test_detect_type.py @@ -1,6 +1,16 @@ import pytest -from json_to_models.dynamic_typing import BooleanString, DDict, DList, DUnion, FloatString, IntString, Null, Unknown +from json_to_models.dynamic_typing import ( + BooleanString, + DDict, + DList, + DUnion, + FloatString, + IntString, + Null, + StringLiteral, + Unknown +) from json_to_models.generator import MetadataGenerator # JSON data | MetaData @@ -8,16 +18,16 @@ pytest.param(1.0, float, id="float"), pytest.param(1, int, id="int"), pytest.param(True, bool, id="bool"), - pytest.param("abc", str, id="str"), + pytest.param("abc", StringLiteral({"abc"}), id="str"), pytest.param(None, Null, id="null"), pytest.param([], DList(Unknown), id="list_empty"), pytest.param([1], DList(int), id="list_single"), pytest.param([*range(100)], DList(int), id="list_single_type"), - pytest.param([1, "a", 2, "c"], DList(DUnion(int, str)), id="list_multi"), + pytest.param([1, "a", 2, "c"], DList(DUnion(int, StringLiteral({'a', 'c'}))), id="list_multi"), pytest.param("1", IntString, id="int_str"), pytest.param("1.0", FloatString, id="float_str"), pytest.param("true", BooleanString, id="bool_str"), - pytest.param({"test_dict_field_a": 1, "test_dict_field_b": "a"}, DDict(DUnion(int, str)), id="dict"), + pytest.param({"test_dict_field_a": 1, "test_dict_field_b": "a"}, DDict(DUnion(int, StringLiteral({"a"}))), id="simple_dict"), pytest.param({}, DDict(Unknown)) ] @@ -50,7 +60,7 @@ def test_convert(models_generator: MetadataGenerator): meta = models_generator._convert(data) assert meta == { "dict_field": DDict(Unknown), - "another_dict_field": DDict(DUnion(int, str)), + "another_dict_field": DDict(DUnion(int, StringLiteral({"a"}))), "another_dict_field_2": DDict(int), "another_dict_field_3": DDict(int), "int_field": int, From 1524c863032c63bfde23594e70c3cc24b1a34a9a Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Sun, 3 May 2020 22:01:44 +0300 Subject: [PATCH 4/7] Pydantic: Implement StringLiteral logic --- json_to_models/dynamic_typing/base.py | 2 +- json_to_models/dynamic_typing/complex.py | 21 +++++++++++++++++-- .../dynamic_typing/string_serializable.py | 2 +- json_to_models/generator.py | 3 +++ json_to_models/models/pydantic.py | 19 +++++++++++++++-- requirements.txt | 3 ++- test/test_cli/test_script.py | 12 +++++++++++ testing_tools/real_apis/openlibrary.py | 2 +- 8 files changed, 56 insertions(+), 8 deletions(-) diff --git a/json_to_models/dynamic_typing/base.py b/json_to_models/dynamic_typing/base.py index d8533cb..556ae8e 100644 --- a/json_to_models/dynamic_typing/base.py +++ b/json_to_models/dynamic_typing/base.py @@ -32,7 +32,7 @@ def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], raise NotImplementedError() @classmethod - def get_kwargs_for_type( + def get_options_for_type( cls, t: Union['BaseType', Type['BaseType']], types_style: Dict[Union['BaseType', Type['BaseType']], dict] diff --git a/json_to_models/dynamic_typing/complex.py b/json_to_models/dynamic_typing/complex.py index 5ae090e..b6d7663 100644 --- a/json_to_models/dynamic_typing/complex.py +++ b/json_to_models/dynamic_typing/complex.py @@ -2,6 +2,8 @@ from itertools import chain from typing import AbstractSet, Dict, Iterable, List, Tuple, Type, Union +from typing_extensions import Literal + from .base import BaseType, ImportPathList, MetaData, get_hash_string from .typing import metadata_to_typing @@ -237,14 +239,22 @@ def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], class StringLiteral(BaseType): - MAX_LITERALS = 15 + class TypeStyle: + use_literals = 'use_literals' + max_literals = 'max_literals' + + MAX_LITERALS = 15 # Hard limit for performance optimization MAX_STRING_LENGTH = 20 __slots__ = ["_literals", "_hash", "_overflow"] def __init__(self, literals: AbstractSet[str]): self._overflow = ( len(literals) > self.MAX_LITERALS - or any(map(lambda s: len(s) >= self.MAX_STRING_LENGTH, literals)) + or + any(map( + lambda s: len(s) >= self.MAX_STRING_LENGTH, + literals + )) ) self._literals = frozenset() if self._overflow else literals @@ -265,6 +275,13 @@ def replace(self, t: 'MetaData', **kwargs) -> 'StringLiteral': def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ -> Tuple[ImportPathList, str]: + options = self.get_options_for_type(self, types_style) + if options.get(self.TypeStyle.use_literals): + limit = options.get(self.TypeStyle.max_literals) + if limit is None or len(self.literals) < limit: + parts = ', '.join(f'"{s}"' for s in self.literals) + return [(Literal.__module__, 'Literal')], f"Literal[{parts}]" + return [], 'str' def _to_hash_string(self) -> str: diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index 39c422e..e86a589 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -40,7 +40,7 @@ def to_typing_code(cls, types_style: Dict[Union['BaseType', Type['BaseType']], d as a metadata instance but contains actual data """ cls_name = cls.__name__ - options = cls.get_kwargs_for_type(cls, types_style) + options = cls.get_options_for_type(cls, types_style) if options.get(cls.TypeStyle.use_actual_type): if cls.actual_type.__module__ != 'builtins': return [(cls.actual_type.__module__, cls.actual_type.__name__)], cls.actual_type.__name__ diff --git a/json_to_models/generator.py b/json_to_models/generator.py index 6b7aac9..99fd3e4 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -202,6 +202,9 @@ def optimize_type(self, meta: MetaData, process_model_ptr=False) -> MetaData: elif isinstance(meta, ComplexType): # Optimize all nested types return meta.replace([self.optimize_type(nested) for nested in meta]) + elif isinstance(meta, StringLiteral): + if meta.overflowed or not meta.literals: + return str return meta def _optimize_union(self, t: DUnion): diff --git a/json_to_models/models/pydantic.py b/json_to_models/models/pydantic.py index 6aca2d8..e65593d 100644 --- a/json_to_models/models/pydantic.py +++ b/json_to_models/models/pydantic.py @@ -9,6 +9,7 @@ MetaData, ModelMeta, Null, + StringLiteral, StringSerializable, Unknown ) @@ -24,17 +25,31 @@ class PydanticModelCodeGenerator(GenericModelCodeGenerator): default_types_style = { StringSerializable: { StringSerializable.TypeStyle.use_actual_type: True + }, + StringLiteral: { + StringLiteral.TypeStyle.use_literals: True } } - def __init__(self, model: ModelMeta, convert_unicode=True): + def __init__(self, model: ModelMeta, max_literals=10, convert_unicode=True): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument :param post_init_converters: Enable generation of type converters in __post_init__ methods :param kwargs: """ - super().__init__(model, post_init_converters=False, convert_unicode=convert_unicode) + super().__init__( + model, + post_init_converters=False, + convert_unicode=convert_unicode, + types_style={ + **self.default_types_style, + StringLiteral: { + **self.default_types_style[StringLiteral], + StringLiteral.TypeStyle.max_literals: int(max_literals) + } + } + ) def generate(self, nested_classes: List[str] = None, extra: str = "", **kwargs) \ -> Tuple[ImportPathList, str]: diff --git a/requirements.txt b/requirements.txt index 12ff7da..48afdcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ python-dateutil>=2.7.* inflection>=0.3.* unidecode>=1.0.* Jinja2>=2.10.* -ordered-set==4.* \ No newline at end of file +ordered-set==4.* +typing-extensions>=3.1.* diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 7413911..aa866d5 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -140,6 +140,18 @@ def test_script_pydantic(command): print(stdout) +@pytest.mark.parametrize("command", test_commands) +def test_script_pydantic_disable_literals(command): + command += " -f pydantic --code-generator-kwargs max_literals=0" + # Pydantic has native (str) -> (builtin_type) converters + command = command.replace('--strings-converters', '') + proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = _validate_result(proc) + assert "(BaseModel):" in stdout + assert "Literal" not in stdout + print(stdout) + + @pytest.mark.parametrize("command", test_commands) def test_script_dataclasses(command): command += " -f dataclasses" diff --git a/testing_tools/real_apis/openlibrary.py b/testing_tools/real_apis/openlibrary.py index 37e6625..f91fe90 100644 --- a/testing_tools/real_apis/openlibrary.py +++ b/testing_tools/real_apis/openlibrary.py @@ -4,9 +4,9 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.structure import compose_models from json_to_models.registry import ModelRegistry from testing_tools.real_apis import dump_response From 0792aa11a4d2a0b627a0d4079c56b32ab7e9869b Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Mon, 4 May 2020 17:53:03 +0300 Subject: [PATCH 5/7] Use StringLiteral for base generator and dataclass generator --- README.md | 77 ++++++++++++++-------------- json_to_models/models/attr.py | 13 +++-- json_to_models/models/base.py | 20 ++++++-- json_to_models/models/dataclasses.py | 6 +-- json_to_models/models/pydantic.py | 18 ++----- testing_tools/real_apis/f1.py | 3 +- 6 files changed, 70 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 5056f50..32e77de 100644 --- a/README.md +++ b/README.md @@ -83,47 +83,46 @@ driver_standings.json ``` ``` -json2models -f attrs -l DriverStandings driver_standings.json +json2models -f pydantic -s flat -l DriverStandings - driver_standings.json ``` ```python -import attr -from json_to_models.dynamic_typing import IntString, IsoDateString +r""" +generated by json2python-models v0.1.2 at Mon May 4 17:46:30 2020 +command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -l DriverStandings - driver_standings.json +""" +from pydantic import BaseModel, Field from typing import List - - -@attr.s -class DriverStandings: - @attr.s - class DriverStanding: - @attr.s - class Driver: - driver_id: str = attr.ib() - permanent_number: IntString = attr.ib(converter=IntString) - code: str = attr.ib() - url: str = attr.ib() - given_name: str = attr.ib() - family_name: str = attr.ib() - date_of_birth: IsoDateString = attr.ib(converter=IsoDateString) - nationality: str = attr.ib() - - @attr.s - class Constructor: - constructor_id: str = attr.ib() - url: str = attr.ib() - name: str = attr.ib() - nationality: str = attr.ib() - - position: IntString = attr.ib(converter=IntString) - position_text: IntString = attr.ib(converter=IntString) - points: IntString = attr.ib(converter=IntString) - wins: IntString = attr.ib(converter=IntString) - driver: 'Driver' = attr.ib() - constructors: List['Constructor'] = attr.ib() - - season: IntString = attr.ib(converter=IntString) - round: IntString = attr.ib(converter=IntString) - driver_standings: List['DriverStanding'] = attr.ib() +from typing_extensions import Literal + +class DriverStandings(BaseModel): + season: int + round_: int = Field(..., alias="round") + DriverStandings: List['DriverStanding'] + +class DriverStanding(BaseModel): + position: int + position_text: int = Field(..., alias="positionText") + points: int + wins: int + driver: 'Driver' = Field(..., alias="Driver") + constructors: List['Constructor'] = Field(..., alias="Constructors") + +class Driver(BaseModel): + driver_id: str = Field(..., alias="driverId") + permanent_number: int = Field(..., alias="permanentNumber") + code: str + url: str + given_name: str = Field(..., alias="givenName") + family_name: str = Field(..., alias="familyName") + date_of_birth: str = Field(..., alias="dateOfBirth") + nationality: str + +class Constructor(BaseModel): + constructor_id: str = Field(..., alias="constructorId") + url: str + name: str + nationality: Literal["Austrian", "German", "American", "British", "Italian", "French"] ```

@@ -141,8 +140,8 @@ It requires a lit bit of tweaking: * There is a lot of optinal fields so we reduce merging threshold ``` -json_to_models -s flat -f dataclasses -m Swagger testing_tools/swagger.json - --dict-keys-fields securityDefinitions paths responses definitions properties +json2models -s flat -f dataclasses -m Swagger testing_tools/swagger.json \ + --dict-keys-fields securityDefinitions paths responses definitions properties \ --merge percent_50 number ``` diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index 7ff31d4..ebc3dbb 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -2,7 +2,7 @@ from typing import List, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template -from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable +from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringLiteral, StringSerializable DEFAULT_ORDER = ( ("default", "converter", "factory"), @@ -14,17 +14,20 @@ class AttrsModelCodeGenerator(GenericModelCodeGenerator): ATTRS = template(f"attr.s{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") ATTRIB = template(f"attr.ib({KWAGRS_TEMPLATE})") + default_types_style = { + StringLiteral: { + StringLiteral.TypeStyle.use_literals: False + } + } - def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, attrs_kwargs: dict = None, - convert_unicode=True): + def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kwargs): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument - :param post_init_converters: Enable generation of type converters in __post_init__ methods :param attrs_kwargs: kwargs for @attr.s() decorators :param kwargs: """ - super().__init__(model, post_init_converters, convert_unicode) + super().__init__(model, **kwargs) self.no_meta = not meta self.attrs_kwargs = attrs_kwargs or {} diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index 4f41e17..be85a21 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -1,3 +1,4 @@ +import copy import keyword import re from typing import Dict, Iterable, List, Tuple, Type, Union @@ -11,7 +12,7 @@ from .structure import sort_fields from .utils import indent from ..dynamic_typing import (AbsoluteModelRef, BaseType, ImportPathList, MetaData, - ModelMeta, compile_imports, metadata_to_typing) + ModelMeta, StringLiteral, compile_imports, metadata_to_typing) from ..utils import cached_method METADATA_FIELD_NAME = "J2M_ORIGINAL_FIELD" @@ -73,19 +74,32 @@ class {{ name }}{% if bases %}({{ bases }}){% endif %}: STR_CONVERT_DECORATOR = template("convert_strings({{ str_fields }}{%% if kwargs %%}, %s{%% endif %%})" % KWAGRS_TEMPLATE) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") - default_types_style = {} + default_types_style = { + StringLiteral: { + StringLiteral.TypeStyle.use_literals: True + } + } def __init__( self, model: ModelMeta, + max_literals=10, post_init_converters=False, convert_unicode=True, types_style: Dict[Union['BaseType', Type['BaseType']], dict] = None ): - self.types_style = types_style if types_style is not None else self.default_types_style self.model = model self.post_init_converters = post_init_converters self.convert_unicode = convert_unicode + + resolved_types_style = copy.deepcopy(self.default_types_style) + types_style = types_style or {} + for t, style in types_style.items(): + resolved_types_style.setdefault(t, {}) + resolved_types_style[t].update(style) + resolved_types_style[StringLiteral][StringLiteral.TypeStyle.max_literals] = int(max_literals) + self.types_style = resolved_types_style + self.model.set_raw_name(self.convert_class_name(self.model.name), generated=self.model.is_name_generated) @cached_method diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index e9123cb..09b91b2 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -15,16 +15,14 @@ class DataclassModelCodeGenerator(GenericModelCodeGenerator): DC_DECORATOR = template(f"dataclass{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") DC_FIELD = template(f"field({KWAGRS_TEMPLATE})") - def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None, - convert_unicode=True): + def __init__(self, model: ModelMeta, meta=False, dataclass_kwargs: dict = None, **kwargs): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument - :param post_init_converters: Enable generation of type converters in __post_init__ methods :param dataclass_kwargs: kwargs for @dataclass() decorators :param kwargs: """ - super().__init__(model, post_init_converters, convert_unicode) + super().__init__(model, **kwargs) self.no_meta = not meta self.dataclass_kwargs = dataclass_kwargs or {} diff --git a/json_to_models/models/pydantic.py b/json_to_models/models/pydantic.py index e65593d..6520fd6 100644 --- a/json_to_models/models/pydantic.py +++ b/json_to_models/models/pydantic.py @@ -31,25 +31,13 @@ class PydanticModelCodeGenerator(GenericModelCodeGenerator): } } - def __init__(self, model: ModelMeta, max_literals=10, convert_unicode=True): + def __init__(self, model: ModelMeta, **kwargs): """ :param model: ModelMeta instance - :param meta: Enable generation of metadata as attrib argument - :param post_init_converters: Enable generation of type converters in __post_init__ methods :param kwargs: """ - super().__init__( - model, - post_init_converters=False, - convert_unicode=convert_unicode, - types_style={ - **self.default_types_style, - StringLiteral: { - **self.default_types_style[StringLiteral], - StringLiteral.TypeStyle.max_literals: int(max_literals) - } - } - ) + kwargs['post_init_converters'] = False + super().__init__(model, **kwargs) def generate(self, nested_classes: List[str] = None, extra: str = "", **kwargs) \ -> Tuple[ImportPathList, str]: diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index fa25d83..1b6eb84 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -46,7 +46,8 @@ def main(): register_datetime_classes() gen = MetadataGenerator() reg = ModelRegistry() - for name, data in (results_data, drivers_data, driver_standings_data): + # for name, data in (results_data, drivers_data, driver_standings_data): + for name, data in (driver_standings_data,): fields = gen.generate(*data) reg.process_meta_data(fields, model_name=inflection.camelize(name)) reg.merge_models(generator=gen) From 5b2c34e3c5eeb463b375a56ff201620d7e22bdfd Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Mon, 4 May 2020 18:11:15 +0300 Subject: [PATCH 6/7] Add --max-strings-literals CLI arg --- README.md | 31 ++++++++++++++++++------------- json_to_models/cli.py | 18 ++++++++++++++++-- json_to_models/models/base.py | 3 ++- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 32e77de..538e2f2 100644 --- a/README.md +++ b/README.md @@ -138,14 +138,19 @@ class Constructor(BaseModel): It requires a lit bit of tweaking: * Some fields store routes/models specs as dicts * There is a lot of optinal fields so we reduce merging threshold +* Disable string literals ``` json2models -s flat -f dataclasses -m Swagger testing_tools/swagger.json \ --dict-keys-fields securityDefinitions paths responses definitions properties \ - --merge percent_50 number + --merge percent_50 number --max-strings-literals 0 ``` ```python +r""" +generated by json2python-models v0.1.2 at Mon May 4 18:08:09 2020 +command: /opt/projects/json2python-models/json_to_models/__main__.py -s flat -f dataclasses -m Swagger testing_tools/swagger.json --max-strings-literals 0 --dict-keys-fields securityDefinitions paths responses definitions properties --merge percent_50 number +""" from dataclasses import dataclass, field from json_to_models.dynamic_typing import FloatString from typing import Any, Dict, List, Optional, Union @@ -191,15 +196,15 @@ class Path: @dataclass class Property: - type: str - format: Optional[str] = None + type_: str + format_: Optional[str] = None xnullable: Optional[bool] = None items: Optional['Item_Schema'] = None @dataclass class Property_2E: - type: str + type_: str title: Optional[str] = None read_only: Optional[bool] = None max_length: Optional[int] = None @@ -208,26 +213,26 @@ class Property_2E: enum: Optional[List[str]] = field(default_factory=list) maximum: Optional[int] = None minimum: Optional[int] = None - format: Optional[str] = None + format_: Optional[str] = None @dataclass class Item: - ref: Optional[str] = None title: Optional[str] = None - type: Optional[str] = None + type_: Optional[str] = None + ref: Optional[str] = None max_length: Optional[int] = None min_length: Optional[int] = None @dataclass class Parameter_SecurityDefinition: - name: str - in_: str + name: Optional[str] = None + in_: Optional[str] = None required: Optional[bool] = None schema: Optional['Item_Schema'] = None - type: Optional[str] = None description: Optional[str] = None + type_: Optional[str] = None @dataclass @@ -252,10 +257,10 @@ class Response: @dataclass class Definition_Schema: - ref: Optional[str] = None + type_: str required: Optional[List[str]] = field(default_factory=list) - type: Optional[str] = None - properties: Optional[Dict[str, Union['Property_2E', 'Property']]] = field(default_factory=dict) + properties: Optional[Dict[str, Union['Property', 'Property_2E']]] = field(default_factory=dict) + ref: Optional[str] = None ```

diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 304b07a..22c4b33 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -53,6 +53,7 @@ def __init__(self): self.models_data: Dict[str, Iterable[dict]] = {} # -m/-l self.enable_datetime: bool = False # --datetime self.strings_converters: bool = False # --strings-converters + self.max_literals: int = -1 # --max-strings-literals self.merge_policy: List[ModelCmp] = [] # --merge self.structure_fn: STRUCTURE_FN_TYPE = None # -s self.model_generator: Type[GenericModelCodeGenerator] = None # -f & --code-generator @@ -83,6 +84,7 @@ def parse_args(self, args: List[str] = None): self.enable_datetime = namespace.datetime disable_unicode_conversion = namespace.disable_unicode_conversion self.strings_converters = namespace.strings_converters + self.max_literals = namespace.max_strings_literals merge_policy = [m.split("_") if "_" in m else m for m in namespace.merge] structure = namespace.structure framework = namespace.framework @@ -204,8 +206,11 @@ def set_args( m = importlib.import_module(module) self.model_generator = getattr(m, cls) - self.model_generator_kwargs = {} if not self.strings_converters else {'post_init_converters': True} - self.model_generator_kwargs['convert_unicode'] = not disable_unicode_conversion + self.model_generator_kwargs = dict( + post_init_converters=self.strings_converters, + convert_unicode=not disable_unicode_conversion, + max_literals=self.max_literals + ) if code_generator_kwargs_raw: for item in code_generator_kwargs_raw: if item[0] == '"': @@ -279,6 +284,15 @@ def _create_argparser(cls) -> argparse.ArgumentParser: action="store_true", help="Enable generation of string types converters (i.e. IsoDatetimeString or BooleanString).\n\n" ) + parser.add_argument( + "--max-strings-literals", + type=int, + default=GenericModelCodeGenerator.DEFAULT_MAX_LITERALS, + metavar='NUMBER', + help="Generate Literal['foo', 'bar'] when field have less than NUMBER string constants as values.\n" + f"Pass 0 to disable. By default NUMBER={GenericModelCodeGenerator.DEFAULT_MAX_LITERALS}" + f" (some generator classes could override it)\n\n" + ) parser.add_argument( "--disable-unicode-conversion", "--no-unidecode", action="store_true", diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index be85a21..b7fa215 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -74,6 +74,7 @@ class {{ name }}{% if bases %}({{ bases }}){% endif %}: STR_CONVERT_DECORATOR = template("convert_strings({{ str_fields }}{%% if kwargs %%}, %s{%% endif %%})" % KWAGRS_TEMPLATE) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") + DEFAULT_MAX_LITERALS = 10 default_types_style = { StringLiteral: { StringLiteral.TypeStyle.use_literals: True @@ -83,7 +84,7 @@ class {{ name }}{% if bases %}({{ bases }}){% endif %}: def __init__( self, model: ModelMeta, - max_literals=10, + max_literals=DEFAULT_MAX_LITERALS, post_init_converters=False, convert_unicode=True, types_style: Dict[Union['BaseType', Type['BaseType']], dict] = None From 925c7bd9f7c79cff1691f5874c4e3bb9b7dd3666 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Mon, 4 May 2020 18:33:01 +0300 Subject: [PATCH 7/7] Add test for type styles and string literals --- json_to_models/dynamic_typing/complex.py | 2 +- .../test_models_code_generator.py | 96 ++++++++++++++++++- .../test_dynamic_typing.py | 19 +++- 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/json_to_models/dynamic_typing/complex.py b/json_to_models/dynamic_typing/complex.py index b6d7663..cdf097a 100644 --- a/json_to_models/dynamic_typing/complex.py +++ b/json_to_models/dynamic_typing/complex.py @@ -279,7 +279,7 @@ def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], if options.get(self.TypeStyle.use_literals): limit = options.get(self.TypeStyle.max_literals) if limit is None or len(self.literals) < limit: - parts = ', '.join(f'"{s}"' for s in self.literals) + parts = ', '.join(f'"{s}"' for s in sorted(self.literals)) return [(Literal.__module__, 'Literal')], f"Literal[{parts}]" return [], 'str' diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py index 076658e..8e5cee7 100644 --- a/test/test_code_generation/test_models_code_generator.py +++ b/test/test_code_generation/test_models_code_generator.py @@ -1,9 +1,8 @@ -from typing import Dict, List +from typing import Dict, List, Type, Union import pytest -from json_to_models.dynamic_typing import (AbsoluteModelRef, DDict, DList, DOptional, IntString, ModelMeta, ModelPtr, - Unknown, compile_imports) +from json_to_models.dynamic_typing import (AbsoluteModelRef, BaseType, DDict, DList, DOptional, IntString, IsoDateString, ModelMeta, ModelPtr, StringLiteral, StringSerializable, Unknown, compile_imports) from json_to_models.models.base import GenericModelCodeGenerator, generate_code from json_to_models.models.structure import sort_fields from json_to_models.models.utils import indent @@ -288,7 +287,96 @@ class Тест: @pytest.mark.parametrize("value,kwargs,expected", test_unicode_data) -def test_generated(value: ModelMeta, kwargs: dict, expected: str): +def test_unicode(value: ModelMeta, kwargs: dict, expected: str): generated = generate_code(([{"model": value, "nested": []}], {}), GenericModelCodeGenerator, class_generator_kwargs=kwargs) assert generated.rstrip() == expected, generated + + +# Data format: +# ( +# model metadata, +# style override, +# expected +# ) +test_override_style_data = [ + pytest.param( + model_factory("M", { + "bar": StringLiteral({'bar', 'foo'}) + }), + {}, + trim(""" + from typing_extensions import Literal + + + class M: + bar: Literal["bar", "foo"] + """), + id='default_behaviour' + ), + pytest.param( + model_factory("M", { + "bar": StringLiteral({'bar', 'foo'}) + }), + {StringLiteral: { + StringLiteral.TypeStyle.use_literals: False + }}, + trim(""" + class M: + bar: str + """), + id='disable_literal' + ), + pytest.param( + model_factory("M", { + "bar": IntString + }), + {IntString: { + IntString.TypeStyle.use_actual_type: True + }}, + trim(""" + class M: + bar: int + """), + id='string_serializable_use_actual_type' + ), + pytest.param( + model_factory("M", { + "bar": IntString + }), + {StringSerializable: { + StringSerializable.TypeStyle.use_actual_type: True + }}, + trim(""" + class M: + bar: int + """), + id='string_serializable_use_actual_type_wildcard' + ), + pytest.param( + model_factory("M", { + "bar": IsoDateString + }), + {IsoDateString: { + IsoDateString.TypeStyle.use_actual_type: True + }}, + trim(""" + from datetime import date + + + class M: + bar: date + """), + id='string_serializable_use_actual_type_date' + ), +] + + +@pytest.mark.parametrize("value,types_style,expected", test_override_style_data) +def test_override_style(value: ModelMeta, types_style: Dict[Union['BaseType', Type['BaseType']], dict], expected: str): + generated = generate_code( + ([{"model": value, "nested": []}], {}), + GenericModelCodeGenerator, + class_generator_kwargs=dict(types_style=types_style) + ) + assert generated.rstrip() == expected, generated diff --git a/test/test_dynamic_typing/test_dynamic_typing.py b/test/test_dynamic_typing/test_dynamic_typing.py index 1826c99..d8eb834 100644 --- a/test/test_dynamic_typing/test_dynamic_typing.py +++ b/test/test_dynamic_typing/test_dynamic_typing.py @@ -2,7 +2,7 @@ import pytest -from json_to_models.dynamic_typing import DUnion, get_hash_string +from json_to_models.dynamic_typing import DUnion, StringLiteral, get_hash_string # *args | MetaData test_dunion = [ @@ -20,7 +20,22 @@ [str, DUnion(int, DUnion(float, complex))], DUnion(int, float, complex, str), id="complex_merge" - ) + ), + pytest.param( + [str, StringLiteral({'a'})], + DUnion(str), + id="str_literal_to_string" + ), + pytest.param( + [StringLiteral({'b'}), StringLiteral({'a'})], + DUnion(StringLiteral({'a', 'b'})), + id="str_literal_merge" + ), + pytest.param( + [StringLiteral({str(i)}) for i in range(100)], + DUnion(str), + id="str_literal_too_much" + ), ]