diff --git a/README.md b/README.md index 5056f50..4f4d800 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ ![Example](/etc/convert.png) json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes -(dataclasses, [attrs](https://github.com/python-attrs/attrs)) from JSON dataset. +([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs)) +from JSON dataset. ## Features @@ -17,11 +18,11 @@ json2python-models is a [Python](https://www.python.org/) tool that can generate * Fields and models **names** generation (unicode support included) * Similar **models generalization** * Handling **recursive data** structures (i.e family tree) -* Detecting **string literals** (i.e. datetime or just stringify numbers) - and providing decorators to easily convert into Python representation +* Detecting **string serializable types** (i.e. datetime or just stringify numbers) +* Detecting fields containing string constants (`Literal['foo', 'bar']`) * Generation models as **tree** (nested models) or **list** * Specifying when dictionaries should be processed as **`dict` type** (by default every dict is considered as some model) -* **CLI** tool +* **CLI** API with a lot of options ## Table of Contents @@ -38,7 +39,26 @@ json2python-models is a [Python](https://www.python.org/) tool that can generate * [Contributing](#contributing) * [License](#license) -## Example +## Examples + +### Part of Path of Exile public items API + +```python +from pydantic import BaseModel, Field +from typing import List, Optional +from typing_extensions import Literal + + +class Tab(BaseModel): + id_: str = Field(..., alias="id") + public: bool + stash_type: Literal["CurrencyStash", "NormalStash", "PremiumStash"] = Field(..., alias="stashType") + items: List['Item'] + account_name: Optional[str] = Field(None, alias="accountName") + last_character_name: Optional[str] = Field(None, alias="lastCharacterName") + stash: Optional[str] = None + league: Optional[Literal["Hardcore", "Standard"]] = None +``` ### F1 Season Results @@ -83,47 +103,46 @@ driver_standings.json ``` ``` -json2models -f attrs -l DriverStandings driver_standings.json +json2models -f pydantic -s flat -l DriverStandings - driver_standings.json ``` ```python -import attr -from json_to_models.dynamic_typing import IntString, IsoDateString +r""" +generated by json2python-models v0.1.2 at Mon May 4 17:46:30 2020 +command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -l DriverStandings - driver_standings.json +""" +from pydantic import BaseModel, Field from typing import List - - -@attr.s -class DriverStandings: - @attr.s - class DriverStanding: - @attr.s - class Driver: - driver_id: str = attr.ib() - permanent_number: IntString = attr.ib(converter=IntString) - code: str = attr.ib() - url: str = attr.ib() - given_name: str = attr.ib() - family_name: str = attr.ib() - date_of_birth: IsoDateString = attr.ib(converter=IsoDateString) - nationality: str = attr.ib() - - @attr.s - class Constructor: - constructor_id: str = attr.ib() - url: str = attr.ib() - name: str = attr.ib() - nationality: str = attr.ib() - - position: IntString = attr.ib(converter=IntString) - position_text: IntString = attr.ib(converter=IntString) - points: IntString = attr.ib(converter=IntString) - wins: IntString = attr.ib(converter=IntString) - driver: 'Driver' = attr.ib() - constructors: List['Constructor'] = attr.ib() - - season: IntString = attr.ib(converter=IntString) - round: IntString = attr.ib(converter=IntString) - driver_standings: List['DriverStanding'] = attr.ib() +from typing_extensions import Literal + +class DriverStandings(BaseModel): + season: int + round_: int = Field(..., alias="round") + DriverStandings: List['DriverStanding'] + +class DriverStanding(BaseModel): + position: int + position_text: int = Field(..., alias="positionText") + points: int + wins: int + driver: 'Driver' = Field(..., alias="Driver") + constructors: List['Constructor'] = Field(..., alias="Constructors") + +class Driver(BaseModel): + driver_id: str = Field(..., alias="driverId") + permanent_number: int = Field(..., alias="permanentNumber") + code: str + url: str + given_name: str = Field(..., alias="givenName") + family_name: str = Field(..., alias="familyName") + date_of_birth: str = Field(..., alias="dateOfBirth") + nationality: str + +class Constructor(BaseModel): + constructor_id: str = Field(..., alias="constructorId") + url: str + name: str + nationality: Literal["Austrian", "German", "American", "British", "Italian", "French"] ```

@@ -139,14 +158,19 @@ class DriverStandings: It requires a lit bit of tweaking: * Some fields store routes/models specs as dicts * There is a lot of optinal fields so we reduce merging threshold +* Disable string literals ``` -json_to_models -s flat -f dataclasses -m Swagger testing_tools/swagger.json - --dict-keys-fields securityDefinitions paths responses definitions properties - --merge percent_50 number +json2models -s flat -f dataclasses -m Swagger testing_tools/swagger.json \ + --dict-keys-fields securityDefinitions paths responses definitions properties \ + --merge percent_50 number --max-strings-literals 0 ``` ```python +r""" +generated by json2python-models v0.1.2 at Mon May 4 18:08:09 2020 +command: /opt/projects/json2python-models/json_to_models/__main__.py -s flat -f dataclasses -m Swagger testing_tools/swagger.json --max-strings-literals 0 --dict-keys-fields securityDefinitions paths responses definitions properties --merge percent_50 number +""" from dataclasses import dataclass, field from json_to_models.dynamic_typing import FloatString from typing import Any, Dict, List, Optional, Union @@ -192,15 +216,15 @@ class Path: @dataclass class Property: - type: str - format: Optional[str] = None + type_: str + format_: Optional[str] = None xnullable: Optional[bool] = None items: Optional['Item_Schema'] = None @dataclass class Property_2E: - type: str + type_: str title: Optional[str] = None read_only: Optional[bool] = None max_length: Optional[int] = None @@ -209,26 +233,26 @@ class Property_2E: enum: Optional[List[str]] = field(default_factory=list) maximum: Optional[int] = None minimum: Optional[int] = None - format: Optional[str] = None + format_: Optional[str] = None @dataclass class Item: - ref: Optional[str] = None title: Optional[str] = None - type: Optional[str] = None + type_: Optional[str] = None + ref: Optional[str] = None max_length: Optional[int] = None min_length: Optional[int] = None @dataclass class Parameter_SecurityDefinition: - name: str - in_: str + name: Optional[str] = None + in_: Optional[str] = None required: Optional[bool] = None schema: Optional['Item_Schema'] = None - type: Optional[str] = None description: Optional[str] = None + type_: Optional[str] = None @dataclass @@ -253,10 +277,10 @@ class Response: @dataclass class Definition_Schema: - ref: Optional[str] = None + type_: str required: Optional[List[str]] = field(default_factory=list) - type: Optional[str] = None - properties: Optional[Dict[str, Union['Property_2E', 'Property']]] = field(default_factory=dict) + properties: Optional[Dict[str, Union['Property', 'Property_2E']]] = field(default_factory=dict) + ref: Optional[str] = None ```

@@ -309,8 +333,8 @@ Arguments: * `-f`, `--framework` - Model framework for which python code is generated. `base` (default) mean no framework so code will be generated without any decorators and additional meta-data. - * **Format**: `-f {base,attrs,dataclasses,custom}` - * **Example**: `-f attrs` + * **Format**: `-f {base, pydantic, attrs, dataclasses, custom}` + * **Example**: `-f pydantic` * **Default**: `-f base` * `-s`, `--structure` - Models composition style. @@ -327,6 +351,13 @@ Arguments: * `--strings-converters` - Enable generation of string types converters (i.e. `IsoDatetimeString` or `BooleanString`). * **Default**: disabled + +* `--max-strings-literals` - Generate `Literal['foo', 'bar']` when field have less than NUMBER string constants as values. + * **Format**: `--max-strings-literals ` + * **Default**: 10 (generator classes could override it) + * **Example**: `--max-strings-literals 5` - only 5 literals will be saved and used to code generation + * **Note**: There could not be more than **15** literals per field (for performance reasons) + * **Note**: `attrs` code generator do not use Literals and just generate `str` fields instead * `--merge` - Merge policy settings. Possible values are: * **Format**: `--merge MERGE_POLICY [MERGE_POLICY ...]` @@ -369,7 +400,7 @@ One of model arguments (`-m` or `-l`) is required. ### Low level API -> Coming soon (Wiki) +\- ## Tests diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 2488eb7..22c4b33 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -17,6 +17,7 @@ from .models.attr import AttrsModelCodeGenerator from .models.base import GenericModelCodeGenerator, generate_code from .models.dataclasses import DataclassModelCodeGenerator +from .models.pydantic import PydanticModelCodeGenerator from .models.structure import compose_models, compose_models_flat from .registry import ( ModelCmp, ModelFieldsEquals, ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry @@ -42,7 +43,9 @@ class Cli: MODEL_GENERATOR_MAPPING: Dict[str, Type[GenericModelCodeGenerator]] = { "base": convert_args(GenericModelCodeGenerator), "attrs": convert_args(AttrsModelCodeGenerator, meta=bool_js_style), - "dataclasses": convert_args(DataclassModelCodeGenerator, meta=bool_js_style, post_init_converters=bool_js_style) + "dataclasses": convert_args(DataclassModelCodeGenerator, meta=bool_js_style, + post_init_converters=bool_js_style), + "pydantic": convert_args(PydanticModelCodeGenerator), } def __init__(self): @@ -50,6 +53,7 @@ def __init__(self): self.models_data: Dict[str, Iterable[dict]] = {} # -m/-l self.enable_datetime: bool = False # --datetime self.strings_converters: bool = False # --strings-converters + self.max_literals: int = -1 # --max-strings-literals self.merge_policy: List[ModelCmp] = [] # --merge self.structure_fn: STRUCTURE_FN_TYPE = None # -s self.model_generator: Type[GenericModelCodeGenerator] = None # -f & --code-generator @@ -80,6 +84,7 @@ def parse_args(self, args: List[str] = None): self.enable_datetime = namespace.datetime disable_unicode_conversion = namespace.disable_unicode_conversion self.strings_converters = namespace.strings_converters + self.max_literals = namespace.max_strings_literals merge_policy = [m.split("_") if "_" in m else m for m in namespace.merge] structure = namespace.structure framework = namespace.framework @@ -201,8 +206,11 @@ def set_args( m = importlib.import_module(module) self.model_generator = getattr(m, cls) - self.model_generator_kwargs = {} if not self.strings_converters else {'post_init_converters': True} - self.model_generator_kwargs['convert_unicode'] = not disable_unicode_conversion + self.model_generator_kwargs = dict( + post_init_converters=self.strings_converters, + convert_unicode=not disable_unicode_conversion, + max_literals=self.max_literals + ) if code_generator_kwargs_raw: for item in code_generator_kwargs_raw: if item[0] == '"': @@ -276,6 +284,15 @@ def _create_argparser(cls) -> argparse.ArgumentParser: action="store_true", help="Enable generation of string types converters (i.e. IsoDatetimeString or BooleanString).\n\n" ) + parser.add_argument( + "--max-strings-literals", + type=int, + default=GenericModelCodeGenerator.DEFAULT_MAX_LITERALS, + metavar='NUMBER', + help="Generate Literal['foo', 'bar'] when field have less than NUMBER string constants as values.\n" + f"Pass 0 to disable. By default NUMBER={GenericModelCodeGenerator.DEFAULT_MAX_LITERALS}" + f" (some generator classes could override it)\n\n" + ) parser.add_argument( "--disable-unicode-conversion", "--no-unidecode", action="store_true", diff --git a/json_to_models/dynamic_typing/__init__.py b/json_to_models/dynamic_typing/__init__.py index a6a7ebf..1790b6d 100644 --- a/json_to_models/dynamic_typing/__init__.py +++ b/json_to_models/dynamic_typing/__init__.py @@ -1,7 +1,7 @@ from .base import ( BaseType, ImportPathList, MetaData, Null, Unknown, get_hash_string ) -from .complex import ComplexType, DDict, DList, DOptional, DTuple, DUnion, SingleType +from .complex import ComplexType, DDict, DList, DOptional, DTuple, DUnion, SingleType, StringLiteral from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr from .string_datetime import IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes from .string_serializable import ( diff --git a/json_to_models/dynamic_typing/base.py b/json_to_models/dynamic_typing/base.py index 672cee8..556ae8e 100644 --- a/json_to_models/dynamic_typing/base.py +++ b/json_to_models/dynamic_typing/base.py @@ -1,5 +1,5 @@ from inspect import isclass -from typing import Any, Generator, Iterable, List, Tuple, Union +from typing import Any, Dict, Generator, Iterable, List, Tuple, Type, Union ImportPathList = List[Tuple[str, Union[Iterable[str], str, None]]] @@ -21,14 +21,30 @@ def replace(self, t: Union['MetaData', List['MetaData']], **kwargs) -> 'BaseType """ raise NotImplementedError() - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: """ Return typing code that represents this metadata and import path of classes that are used in this code + :param types_style: Hints for .to_typing_code() for different type wrappers :return: ((module_name, (class_name, ...)), code) """ raise NotImplementedError() + @classmethod + def get_options_for_type( + cls, + t: Union['BaseType', Type['BaseType']], + types_style: Dict[Union['BaseType', Type['BaseType']], dict] + ) -> dict: + t_cls = t if isclass(t) else type(t) + mro = t_cls.__mro__ + for base in mro: + options = types_style.get(base, ...) + if options is not Ellipsis: + return options + return {} + def to_hash_string(self) -> str: """ Return unique string that can be used to generate hash of type instance. @@ -71,7 +87,8 @@ def __iter__(self) -> Iterable['MetaData']: def replace(self, t: 'MetaData', **kwargs) -> 'UnknownType': return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: return ([('typing', 'Any')], 'Any') def to_hash_string(self) -> str: @@ -90,7 +107,8 @@ def __iter__(self) -> Iterable['MetaData']: def replace(self, t: 'MetaData', **kwargs) -> 'NoneType': return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: return ([], 'None') def to_hash_string(self) -> str: diff --git a/json_to_models/dynamic_typing/complex.py b/json_to_models/dynamic_typing/complex.py index 6bfd48f..6fba7a5 100644 --- a/json_to_models/dynamic_typing/complex.py +++ b/json_to_models/dynamic_typing/complex.py @@ -1,11 +1,15 @@ +from functools import partial from itertools import chain -from typing import Iterable, List, Tuple, Union +from typing import AbstractSet, Dict, Iterable, List, Optional, Tuple, Type, Union + +from typing_extensions import Literal from .base import BaseType, ImportPathList, MetaData, get_hash_string from .typing import metadata_to_typing class SingleType(BaseType): + _typing_cls = None __slots__ = ["_type", "_hash"] def __init__(self, t: MetaData): @@ -37,11 +41,20 @@ def replace(self, t: 'MetaData', **kwargs) -> 'SingleType': self.type = t return self + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = metadata_to_typing(self.type, types_style=types_style) + return ( + [*imports, (self._typing_cls.__module__, self._typing_cls._name)], + f"{self._typing_cls._name}[{nested}]" + ) + def _to_hash_string(self) -> str: return f"{type(self).__name__}/{get_hash_string(self.type)}" class ComplexType(BaseType): + _typing_cls = None __slots__ = ["_types", "_sorted", "_hash"] def __init__(self, *types: MetaData): @@ -106,12 +119,13 @@ def replace(self, t: Union['MetaData', List['MetaData']], index=None, **kwargs) raise ValueError(f"Unsupported arguments: t={t} index={index} kwargs={kwargs}") return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = zip(*map(metadata_to_typing, self)) + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = zip(*map(partial(metadata_to_typing, types_style=types_style), self)) nested = ", ".join(nested) return ( - list(chain(*imports)), - f"[{nested}]" + [*chain.from_iterable(imports), (self._typing_cls.__module__, self._typing_cls._name)], + f"{self._typing_cls._name}[{nested}]" ) def _to_hash_string(self) -> str: @@ -122,37 +136,60 @@ class DOptional(SingleType): """ Field of this type may not be presented in JSON object """ - - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = metadata_to_typing(self.type) - return ( - [*imports, ('typing', 'Optional')], - f"Optional[{nested}]" - ) + _typing_cls = Optional class DUnion(ComplexType): """ Same as typing.Union. Nested types are unique. """ + _typing_cls = Union def __init__(self, *types: Union[type, BaseType, dict]): hashes = set() unique_types = [] + use_literals = True + str_literals = set() + # Ensure that types in union are unique - for t in types: - if isinstance(t, DUnion): - # Merging nested DUnions - for t2 in list(t._extract_nested_types()): - h = get_hash_string(t2) - if h not in hashes: - unique_types.append(t2) - hashes.add(h) + + def handle_type(t, use_literals): + if t is str: + use_literals = False + + if isinstance(t, StringLiteral): + if not use_literals: + return + + if t.overflowed: + use_literals = False + else: + str_literals.update(t.literals) + else: h = get_hash_string(t) if h not in hashes: unique_types.append(t) hashes.add(h) + return use_literals + + for t in types: + if isinstance(t, DUnion): + # Merging nested DUnions + for t2 in list(t._extract_nested_types()): + use_literals = handle_type(t2, use_literals) and use_literals + else: + use_literals = handle_type(t, use_literals) and use_literals + + if str_literals and use_literals: + literal = StringLiteral(str_literals) + if literal.overflowed: + use_literals = False + else: + unique_types.append(literal) + + if not use_literals: + handle_type(str, use_literals=False) super().__init__(*unique_types) def _extract_nested_types(self): @@ -165,37 +202,86 @@ def _extract_nested_types(self): else: yield t - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = super().to_typing_code() - return ( - [*imports, ('typing', 'Union')], - "Union" + nested - ) - class DTuple(ComplexType): - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = super().to_typing_code() - return ( - [*imports, ('typing', 'Tuple')], - "Tuple" + nested - ) + _typing_cls = Tuple class DList(SingleType): - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = metadata_to_typing(self.type) - return ( - [*imports, ('typing', 'List')], - f"List[{nested}]" - ) + _typing_cls = List class DDict(SingleType): + _typing_cls = Dict + # Dict is single type because keys of JSON dict are always strings. - def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, nested = metadata_to_typing(self.type) + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + imports, nested = metadata_to_typing(self.type, types_style=types_style) return ( [*imports, ('typing', 'Dict')], f"Dict[str, {nested}]" ) + + +class StringLiteral(BaseType): + class TypeStyle: + use_literals = 'use_literals' + max_literals = 'max_literals' + + MAX_LITERALS = 15 # Hard limit for performance optimization + MAX_STRING_LENGTH = 20 + __slots__ = ["_literals", "_hash", "_overflow"] + + def __init__(self, literals: AbstractSet[str]): + self._overflow = ( + len(literals) > self.MAX_LITERALS + or + any(map( + lambda s: len(s) >= self.MAX_STRING_LENGTH, + literals + )) + ) + self._literals = frozenset() if self._overflow else literals + + def __iter__(self) -> Iterable['MetaData']: + return iter(()) + + def __str__(self): + return f"{type(self).__name__}[{self._repr_literals()}]" + + def __repr__(self): + return f"<{type(self).__name__} [{self._repr_literals()}]>" + + def __eq__(self, other): + return type(other) is type(self) and self._literals == other._literals + + def replace(self, t: 'MetaData', **kwargs) -> 'StringLiteral': + return self + + def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \ + -> Tuple[ImportPathList, str]: + options = self.get_options_for_type(self, types_style) + if options.get(self.TypeStyle.use_literals): + limit = options.get(self.TypeStyle.max_literals) + if limit is None or len(self.literals) < limit: + parts = ', '.join(f'"{s}"' for s in sorted(self.literals)) + return [(Literal.__module__, 'Literal')], f"Literal[{parts}]" + + return [], 'str' + + def _to_hash_string(self) -> str: + return f"{type(self).__name__}/{self._repr_literals()}" + + @property + def literals(self): + return self._literals + + @property + def overflowed(self): + return self._overflow + + def _repr_literals(self): + if self._overflow: + return '...' + return ','.join(self._literals) diff --git a/json_to_models/dynamic_typing/models_meta.py b/json_to_models/dynamic_typing/models_meta.py index 895086d..4e78964 100644 --- a/json_to_models/dynamic_typing/models_meta.py +++ b/json_to_models/dynamic_typing/models_meta.py @@ -1,8 +1,9 @@ import threading -from typing import Dict, List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Tuple, Type, Union import inflection +from . import BaseType from .base import ImportPathList, MetaData from .complex import SingleType from ..utils import distinct_words @@ -94,7 +95,8 @@ def add_child_ref(self, ptr: 'ModelPtr'): def remove_child_ref(self, ptr: 'ModelPtr'): self.child_pointers.remove(ptr) - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union[BaseType, Type[BaseType]], dict]) \ + -> Tuple[ImportPathList, str]: if self.name is None: raise ValueError('Model without name can not be typed') return [], self.name @@ -130,8 +132,9 @@ def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr': self.parent.add_child_ref(self) return self - def to_typing_code(self) -> Tuple[ImportPathList, str]: - return AbsoluteModelRef(self.type).to_typing_code() + def to_typing_code(self, types_style: Dict[Union[BaseType, Type[BaseType]], dict]) \ + -> Tuple[ImportPathList, str]: + return AbsoluteModelRef(self.type).to_typing_code(types_style) def _to_hash_string(self) -> str: return f"{type(self).__name__}_#{self.type.index}" @@ -187,7 +190,8 @@ def inject(cls, patches: ContextInjectionType): def __init__(self, model: ModelMeta): self.model = model - def to_typing_code(self) -> Tuple[ImportPathList, str]: + def to_typing_code(self, types_style: Dict[Union[BaseType, Type[BaseType]], dict]) \ + -> Tuple[ImportPathList, str]: context_data = self.Context.data.context if context_data: model_path = context_data.get(self.model, "") @@ -195,6 +199,6 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: model_path = model_path.name else: model_path = "" - imports, model = self.model.to_typing_code() + imports, model = self.model.to_typing_code(types_style) s = ".".join(filter(None, (model_path, model))) return imports, f"'{s}'" diff --git a/json_to_models/dynamic_typing/string_datetime.py b/json_to_models/dynamic_typing/string_datetime.py index 19db008..cf1a810 100644 --- a/json_to_models/dynamic_typing/string_datetime.py +++ b/json_to_models/dynamic_typing/string_datetime.py @@ -71,6 +71,7 @@ class IsoDateString(StringSerializable, date): Parse date using dateutil.parser.isoparse. Representation format always is ``YYYY-MM-DD``. You can override to_representation method to customize it. Just don't forget to call registry.remove(IsoDateString) """ + actual_type = date @classmethod def to_internal_value(cls, value: str) -> 'IsoDateString': @@ -92,6 +93,7 @@ class IsoTimeString(StringSerializable, time): Parse time using dateutil.parser.parse. Representation format always is ``hh:mm:ss.ms``. You can override to_representation method to customize it. """ + actual_type = time @classmethod def to_internal_value(cls, value: str) -> 'IsoTimeString': @@ -113,6 +115,7 @@ class IsoDatetimeString(StringSerializable, datetime): Parse datetime using dateutil.parser.isoparse. Representation format always is ``YYYY-MM-DDThh:mm:ss.ms`` (datetime.isoformat method). """ + actual_type = datetime @classmethod def to_internal_value(cls, value: str) -> 'IsoDatetimeString': diff --git a/json_to_models/dynamic_typing/string_serializable.py b/json_to_models/dynamic_typing/string_serializable.py index 559b87f..e86a589 100644 --- a/json_to_models/dynamic_typing/string_serializable.py +++ b/json_to_models/dynamic_typing/string_serializable.py @@ -1,5 +1,5 @@ from itertools import permutations -from typing import Collection, Iterable, List, Set, Tuple, Type +from typing import ClassVar, Collection, Dict, Iterable, List, Set, Tuple, Type, Union from .base import BaseType, ImportPathList @@ -9,6 +9,11 @@ class StringSerializable(BaseType): Mixin for classes which are used to (de-)serialize some values in a string form """ + class TypeStyle: + use_actual_type = 'use_actual_type' + + actual_type: ClassVar[Type] + @classmethod def to_internal_value(cls, value: str) -> 'StringSerializable': """ @@ -29,12 +34,17 @@ def to_representation(self) -> str: raise NotImplementedError() @classmethod - def to_typing_code(cls) -> Tuple[ImportPathList, str]: + def to_typing_code(cls, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) -> Tuple[ImportPathList, str]: """ Unlike other BaseType's subclasses it's a class method because StringSerializable instance is not parameterized as a metadata instance but contains actual data """ cls_name = cls.__name__ + options = cls.get_options_for_type(cls, types_style) + if options.get(cls.TypeStyle.use_actual_type): + if cls.actual_type.__module__ != 'builtins': + return [(cls.actual_type.__module__, cls.actual_type.__name__)], cls.actual_type.__name__ + return [], cls.actual_type.__name__ return [('json_to_models.dynamic_typing', cls_name)], cls_name def __iter__(self): @@ -117,6 +127,8 @@ def resolve(self, *types: T_StringSerializable) -> Collection[T_StringSerializab @registry.add() class IntString(StringSerializable, int): + actual_type = int + @classmethod def to_internal_value(cls, value: str) -> 'IntString': return cls(value) @@ -127,6 +139,8 @@ def to_representation(self) -> str: @registry.add(replace_types=(IntString,)) class FloatString(StringSerializable, float): + actual_type = float + @classmethod def to_internal_value(cls, value: str) -> 'FloatString': return cls(value) @@ -138,6 +152,7 @@ def to_representation(self) -> str: @registry.add() class BooleanString(StringSerializable, int): # We can't extend bool class, but we can extend int with same result excepting isinstance and issubclass check + actual_type = bool @classmethod def to_internal_value(cls, value: str) -> 'BooleanString': diff --git a/json_to_models/dynamic_typing/typing.py b/json_to_models/dynamic_typing/typing.py index e6e0ba6..2f76892 100644 --- a/json_to_models/dynamic_typing/typing.py +++ b/json_to_models/dynamic_typing/typing.py @@ -1,26 +1,32 @@ import operator +from datetime import date, datetime, time from inspect import isclass -from typing import Dict, Set, Tuple +from typing import Dict, Set, Tuple, Type, Union -from .base import ImportPathList, MetaData +from .base import BaseType, ImportPathList, MetaData from .string_serializable import StringSerializable -def metadata_to_typing(t: MetaData) -> Tuple[ImportPathList, str]: +def metadata_to_typing(t: MetaData, types_style: Dict[Union[BaseType, Type[BaseType]], dict] = None) \ + -> Tuple[ImportPathList, str]: """ Shortcut function to call ``to_typing_code`` method of BaseType instances or return name of type otherwise :param t: :return: """ + types_style = types_style or {} if isclass(t): if issubclass(t, StringSerializable): - return t.to_typing_code() + return t.to_typing_code(types_style) else: - return ([], t.__name__) + imports = [] + if issubclass(t, (date, datetime, time)): + imports.append((t.__module__, [t.__name__])) + return (imports, t.__name__) elif isinstance(t, dict): raise ValueError("Can not convert dict instance to typing code. It should be wrapped into ModelMeta instance") else: - return t.to_typing_code() + return t.to_typing_code(types_style) def compile_imports(imports: ImportPathList) -> str: diff --git a/json_to_models/generator.py b/json_to_models/generator.py index 7e960be..603fc5c 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -1,10 +1,22 @@ import re from typing import Any, Callable, List, Optional, Pattern, Union -from unidecode import unidecode - -from .dynamic_typing import (ComplexType, DDict, DList, DOptional, DUnion, MetaData, ModelPtr, Null, SingleType, - StringSerializable, StringSerializableRegistry, Unknown, registry) +from .dynamic_typing import ( + ComplexType, + DDict, + DList, + DOptional, + DUnion, + MetaData, + ModelPtr, + Null, + SingleType, + StringLiteral, + StringSerializable, + StringSerializableRegistry, + Unknown, + registry +) _static_types = {float, bool, int} @@ -44,12 +56,8 @@ def _convert(self, data: dict): """ fields = dict() for key, value in data.items(): - # TODO: Check if is 0xC0000005 crash has a place at linux systems - # ! _detect_type function can crash at some complex data sets if value is unicode with some characters (maybe German) - # Crash does not produce any useful logs and can occur any time after bad string was processed - # It can be reproduced on real_apis tests (openlibrary API) convert_dict = key not in self.dict_keys_fields - fields[key] = self._detect_type(value if not isinstance(value, str) else unidecode(value), convert_dict) + fields[key] = self._detect_type(value, convert_dict) return fields def _detect_type(self, value, convert_dict=True) -> MetaData: @@ -108,7 +116,7 @@ def _detect_type(self, value, convert_dict=True) -> MetaData: except ValueError: continue return t - return str + return StringLiteral({value}) def merge_field_sets(self, field_sets: List[MetaData]) -> MetaData: """ @@ -188,6 +196,9 @@ def optimize_type(self, meta: MetaData, process_model_ptr=False) -> MetaData: elif isinstance(meta, ComplexType): # Optimize all nested types return meta.replace([self.optimize_type(nested) for nested in meta]) + elif isinstance(meta, StringLiteral): + if meta.overflowed or not meta.literals: + return str return meta def _optimize_union(self, t: DUnion): @@ -199,7 +210,7 @@ def _optimize_union(self, t: DUnion): str_types: List[Union[type, StringSerializable]] = [] types_to_merge: List[dict] = [] list_types: List[DList] = [] - dict_types: List[DList] = [] + dict_types: List[DDict] = [] other_types: List[MetaData] = [] for item in t.types: if isinstance(item, DOptional): diff --git a/json_to_models/models/__init__.py b/json_to_models/models/__init__.py index 748542a..c601110 100644 --- a/json_to_models/models/__init__.py +++ b/json_to_models/models/__init__.py @@ -13,3 +13,4 @@ class ClassType(Enum): Dataclass = "dataclass" Attrs = "attrs" + Pydantic = "pydantic" diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index 7ff31d4..ebc3dbb 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -2,7 +2,7 @@ from typing import List, Tuple from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template -from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable +from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringLiteral, StringSerializable DEFAULT_ORDER = ( ("default", "converter", "factory"), @@ -14,17 +14,20 @@ class AttrsModelCodeGenerator(GenericModelCodeGenerator): ATTRS = template(f"attr.s{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") ATTRIB = template(f"attr.ib({KWAGRS_TEMPLATE})") + default_types_style = { + StringLiteral: { + StringLiteral.TypeStyle.use_literals: False + } + } - def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, attrs_kwargs: dict = None, - convert_unicode=True): + def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kwargs): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument - :param post_init_converters: Enable generation of type converters in __post_init__ methods :param attrs_kwargs: kwargs for @attr.s() decorators :param kwargs: """ - super().__init__(model, post_init_converters, convert_unicode) + super().__init__(model, **kwargs) self.no_meta = not meta self.attrs_kwargs = attrs_kwargs or {} diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index c9bd596..a352e63 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -1,6 +1,7 @@ +import copy import keyword import re -from typing import Iterable, List, Tuple, Type +from typing import Dict, Iterable, List, Tuple, Type, Union import inflection from jinja2 import Template @@ -10,8 +11,8 @@ from .string_converters import get_string_field_paths from .structure import sort_fields from .utils import indent -from ..dynamic_typing import (AbsoluteModelRef, ImportPathList, MetaData, - ModelMeta, compile_imports, metadata_to_typing) +from ..dynamic_typing import (AbsoluteModelRef, BaseType, ImportPathList, MetaData, + ModelMeta, StringLiteral, compile_imports, metadata_to_typing) from ..utils import cached_method METADATA_FIELD_NAME = "J2M_ORIGINAL_FIELD" @@ -52,7 +53,7 @@ class GenericModelCodeGenerator: {%- for decorator in decorators -%} @{{ decorator }} {% endfor -%} - class {{ name }}: + class {{ name }}{% if bases %}({{ bases }}){% endif %}: {%- for code in nested %} {{ code }} @@ -73,11 +74,33 @@ class {{ name }}: STR_CONVERT_DECORATOR = template("convert_strings({{ str_fields }}{%% if kwargs %%}, %s{%% endif %%})" % KWAGRS_TEMPLATE) FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") - - def __init__(self, model: ModelMeta, post_init_converters=False, convert_unicode=True): + DEFAULT_MAX_LITERALS = 10 + default_types_style = { + StringLiteral: { + StringLiteral.TypeStyle.use_literals: True + } + } + + def __init__( + self, + model: ModelMeta, + max_literals=DEFAULT_MAX_LITERALS, + post_init_converters=False, + convert_unicode=True, + types_style: Dict[Union['BaseType', Type['BaseType']], dict] = None + ): self.model = model self.post_init_converters = post_init_converters self.convert_unicode = convert_unicode + + resolved_types_style = copy.deepcopy(self.default_types_style) + types_style = types_style or {} + for t, style in types_style.items(): + resolved_types_style.setdefault(t, {}) + resolved_types_style[t].update(style) + resolved_types_style[StringLiteral][StringLiteral.TypeStyle.max_literals] = int(max_literals) + self.types_style = resolved_types_style + self.model.set_raw_name(self.convert_class_name(self.model.name), generated=self.model.is_name_generated) @cached_method @@ -88,7 +111,8 @@ def convert_class_name(self, name): def convert_field_name(self, name): return inflection.underscore(prepare_label(name, convert_unicode=self.convert_unicode)) - def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[ImportPathList, str]: + def generate(self, nested_classes: List[str] = None, bases: str = None, extra: str = "") \ + -> Tuple[ImportPathList, str]: """ :param nested_classes: list of strings that contains classes code :return: list of import data, class code @@ -98,8 +122,9 @@ def generate(self, nested_classes: List[str] = None, extra: str = "") -> Tuple[I data = { "decorators": decorators, "name": self.model.name, + "bases": bases or [], "fields": fields, - "extra": extra + "extra": extra, } if nested_classes: data["nested"] = [indent(s) for s in nested_classes] @@ -131,7 +156,7 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP :param optional: Is field optional :return: imports, field data """ - imports, typing = metadata_to_typing(meta) + imports, typing = metadata_to_typing(meta, types_style=self.types_style) data = { "name": self.convert_field_name(name), @@ -150,12 +175,16 @@ def fields(self) -> Tuple[ImportPathList, List[str]]: imports: ImportPathList = [] strings: List[str] = [] for is_optional, fields in enumerate((required, optional)): + fields = self._filter_fields(fields) for field in fields: field_imports, data = self.field_data(field, self.model.type[field], bool(is_optional)) imports.extend(field_imports) strings.append(self.FIELD.render(**data)) return imports, strings + def _filter_fields(self, fields): + return fields + @property def string_field_paths(self) -> List[str]: """ @@ -191,6 +220,7 @@ def _generate_code( """ imports = [] classes = [] + generators = [] for data in structure: nested_imports, nested_classes = _generate_code( data["nested"], @@ -199,7 +229,11 @@ def _generate_code( lvl=lvl + 1 ) imports.extend(nested_imports) - gen = class_generator(data["model"], **class_generator_kwargs) + generators.append(( + class_generator(data["model"], **class_generator_kwargs), + nested_classes + )) + for gen, nested_classes in generators: cls_imports, cls_string = gen.generate(nested_classes) imports.extend(cls_imports) classes.append(cls_string) diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py index e9123cb..09b91b2 100644 --- a/json_to_models/models/dataclasses.py +++ b/json_to_models/models/dataclasses.py @@ -15,16 +15,14 @@ class DataclassModelCodeGenerator(GenericModelCodeGenerator): DC_DECORATOR = template(f"dataclass{{% if kwargs %}}({KWAGRS_TEMPLATE}){{% endif %}}") DC_FIELD = template(f"field({KWAGRS_TEMPLATE})") - def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None, - convert_unicode=True): + def __init__(self, model: ModelMeta, meta=False, dataclass_kwargs: dict = None, **kwargs): """ :param model: ModelMeta instance :param meta: Enable generation of metadata as attrib argument - :param post_init_converters: Enable generation of type converters in __post_init__ methods :param dataclass_kwargs: kwargs for @dataclass() decorators :param kwargs: """ - super().__init__(model, post_init_converters, convert_unicode) + super().__init__(model, **kwargs) self.no_meta = not meta self.dataclass_kwargs = dataclass_kwargs or {} diff --git a/json_to_models/models/pydantic.py b/json_to_models/models/pydantic.py new file mode 100644 index 0000000..6520fd6 --- /dev/null +++ b/json_to_models/models/pydantic.py @@ -0,0 +1,92 @@ +from typing import List, Optional, Tuple + +from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, sort_kwargs, template +from ..dynamic_typing import ( + DDict, + DList, + DOptional, + ImportPathList, + MetaData, + ModelMeta, + Null, + StringLiteral, + StringSerializable, + Unknown +) + +DEFAULT_ORDER = ( + "*", +) + + +class PydanticModelCodeGenerator(GenericModelCodeGenerator): + PYDANTIC_FIELD = template("Field({{ default }}{% if kwargs %}, KWAGRS_TEMPLATE{% endif %})" + .replace('KWAGRS_TEMPLATE', KWAGRS_TEMPLATE)) + default_types_style = { + StringSerializable: { + StringSerializable.TypeStyle.use_actual_type: True + }, + StringLiteral: { + StringLiteral.TypeStyle.use_literals: True + } + } + + def __init__(self, model: ModelMeta, **kwargs): + """ + :param model: ModelMeta instance + :param kwargs: + """ + kwargs['post_init_converters'] = False + super().__init__(model, **kwargs) + + def generate(self, nested_classes: List[str] = None, extra: str = "", **kwargs) \ + -> Tuple[ImportPathList, str]: + imports, body = super(PydanticModelCodeGenerator, self).generate( + bases='BaseModel', + nested_classes=nested_classes, + extra=extra + ) + imports.append(('pydantic', ['BaseModel', 'Field'])) + return imports, body + + def _filter_fields(self, fields): + fields = super()._filter_fields(fields) + filtered = [] + for field in fields: + field_type = self.model.type[field] + if field_type in (Unknown, Null): + continue + filtered.append(field) + return filtered + + def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: + """ + Form field data for template + + :param name: Original field name + :param meta: Field metadata + :param optional: Is field optional + :return: imports, field data + """ + imports, data = super().field_data(name, meta, optional) + default: Optional[str] = None + body_kwargs = {} + if optional: + meta: DOptional + if isinstance(meta.type, DList): + default = "[]" + elif isinstance(meta.type, DDict): + default = "{}" + else: + default = "None" + + if name != data["name"]: + body_kwargs["alias"] = f'"{name}"' + if body_kwargs: + data["body"] = self.PYDANTIC_FIELD.render( + default=default or '...', + kwargs=sort_kwargs(body_kwargs, DEFAULT_ORDER) + ) + elif default is not None: + data["body"] = default + return imports, data diff --git a/json_to_models/models/string_converters.py b/json_to_models/models/string_converters.py index 04326b7..988abdd 100644 --- a/json_to_models/models/string_converters.py +++ b/json_to_models/models/string_converters.py @@ -3,8 +3,18 @@ from typing import Any, Callable, List, Optional, Tuple from . import ClassType -from ..dynamic_typing import (BaseType, DDict, DList, DOptional, DUnion, MetaData, ModelMeta, ModelPtr, - StringSerializable) +from ..dynamic_typing import ( + BaseType, + DDict, + DList, + DOptional, + DUnion, + MetaData, + ModelMeta, + ModelPtr, + StringLiteral, + StringSerializable +) from ..dynamic_typing.base import NoneType @@ -168,6 +178,8 @@ def get_string_field_paths(model: ModelMeta) -> List[Tuple[str, List[str]]]: break elif cls is NoneType: continue + elif cls in (StringLiteral,): + continue else: raise TypeError(f"Unsupported meta-type for converter path {cls}") diff --git a/json_to_models/utils.py b/json_to_models/utils.py index 609569d..280dcab 100644 --- a/json_to_models/utils.py +++ b/json_to_models/utils.py @@ -9,7 +9,7 @@ def __init__(self): self.i = 1 def __call__(self, *args, **kwargs): - value = '%i%s' % (self.i, self.ch) + value = f'{self.i}{self.ch}' ch = chr(ord(self.ch) + 1) if ch <= 'Z': self.ch = ch diff --git a/requirements.txt b/requirements.txt index 12ff7da..48afdcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ python-dateutil>=2.7.* inflection>=0.3.* unidecode>=1.0.* Jinja2>=2.10.* -ordered-set==4.* \ No newline at end of file +ordered-set==4.* +typing-extensions>=3.1.* diff --git a/setup.py b/setup.py index 21983a4..1fa6ec2 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,6 @@ def run_tests(self): }, install_requires=required, cmdclass={"test": PyTest}, - tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs"], + tests_require=["pytest>=4.4.0", "pytest-xdist", "requests", "attrs", "pydantic>=1.3"], data_files=[('', ['requirements.txt', 'pytest.ini', '.coveragerc', 'LICENSE', 'README.md', 'CHANGELOG.md'])] ) diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 1988850..aa866d5 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -98,7 +98,10 @@ def _validate_result(proc: subprocess.Popen, output=None, output_file: Path = No # Note: imp package is deprecated but I can't find a way to create dummy module using importlib module = imp.new_module("test_model") sys.modules["test_model"] = module - exec(compile(stdout, "test_model.py", "exec"), module.__dict__) + try: + exec(compile(stdout, "test_model.py", "exec"), module.__dict__) + except Exception as e: + assert not e, stdout return stdout, stderr @@ -126,6 +129,29 @@ def test_script_attrs(command): print(stdout) +@pytest.mark.parametrize("command", test_commands) +def test_script_pydantic(command): + command += " -f pydantic" + # Pydantic has native (str) -> (builtin_type) converters + command = command.replace('--strings-converters', '') + proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = _validate_result(proc) + assert "(BaseModel):" in stdout + print(stdout) + + +@pytest.mark.parametrize("command", test_commands) +def test_script_pydantic_disable_literals(command): + command += " -f pydantic --code-generator-kwargs max_literals=0" + # Pydantic has native (str) -> (builtin_type) converters + command = command.replace('--strings-converters', '') + proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = _validate_result(proc) + assert "(BaseModel):" in stdout + assert "Literal" not in stdout + print(stdout) + + @pytest.mark.parametrize("command", test_commands) def test_script_dataclasses(command): command += " -f dataclasses" diff --git a/test/test_cli/test_self_validate_pydantic.py b/test/test_cli/test_self_validate_pydantic.py new file mode 100644 index 0000000..b1a5f35 --- /dev/null +++ b/test/test_cli/test_self_validate_pydantic.py @@ -0,0 +1,56 @@ +import imp +import json +import sys +from inspect import isclass + +import pydantic +import pytest + +from json_to_models.generator import MetadataGenerator +from json_to_models.models.base import generate_code +from json_to_models.models.pydantic import PydanticModelCodeGenerator +from json_to_models.models.structure import compose_models_flat +from json_to_models.registry import ModelRegistry +from .test_script import test_data_path + +test_self_validate_pydantic_data = [ + pytest.param(test_data_path / "gists.json", list, id="gists.json"), + pytest.param(test_data_path / "users.json", list, id="users.json"), + pytest.param(test_data_path / "unicode.json", dict, id="unicode.json"), + pytest.param(test_data_path / "photos.json", dict, id="photos.json"), +] + + +@pytest.mark.parametrize("data,data_type", test_self_validate_pydantic_data) +def test_self_validate_pydantic(data, data_type): + with data.open() as f: + data = json.load(f) + + gen = MetadataGenerator( + dict_keys_fields=['files'] + ) + reg = ModelRegistry() + if data_type is not list: + data = [data] + fields = gen.generate(*data) + reg.process_meta_data(fields, model_name="TestModel") + reg.merge_models(generator=gen) + reg.generate_names() + + structure = compose_models_flat(reg.models_map) + code = generate_code(structure, PydanticModelCodeGenerator) + module = imp.new_module("test_models") + sys.modules["test_models"] = module + try: + exec(compile(code, "test_models.py", "exec"), module.__dict__) + except Exception as e: + assert not e, code + + import test_models + for name in dir(test_models): + cls = getattr(test_models, name) + if isclass(cls) and issubclass(cls, pydantic.BaseModel): + cls.update_forward_refs() + for item in data: + obj = test_models.TestModel.parse_obj(item) + assert obj diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py index 3f3cff5..d3200db 100644 --- a/test/test_code_generation/test_models_code_generator.py +++ b/test/test_code_generation/test_models_code_generator.py @@ -1,13 +1,15 @@ -from typing import Dict, List +from typing import Dict, List, Type, Union import pytest +from typing_extensions import Literal -from json_to_models.dynamic_typing import (AbsoluteModelRef, DDict, DList, DOptional, IntString, ModelMeta, ModelPtr, - Unknown, compile_imports) +from json_to_models.dynamic_typing import (AbsoluteModelRef, BaseType, DDict, DList, DOptional, IntString, IsoDateString, ModelMeta, ModelPtr, StringLiteral, StringSerializable, Unknown, compile_imports) from json_to_models.models.base import GenericModelCodeGenerator, generate_code from json_to_models.models.structure import sort_fields from json_to_models.models.utils import indent +LITERAL_SOURCE = f"from {Literal.__module__}" + # Data structure: # (string, indent lvl, indent string) # result @@ -192,20 +194,20 @@ def test_absolute_model_ref(): test_model = ModelMeta({"field": int}, "A") test_model.name = "test_model" test_ptr = ModelPtr(test_model) - assert test_ptr.to_typing_code()[1] == "'TestModel'" + assert test_ptr.to_typing_code({})[1] == "'TestModel'" with AbsoluteModelRef.inject({test_model: "Parent"}): - assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" - assert test_ptr.to_typing_code()[1] == "'TestModel'" + assert test_ptr.to_typing_code({})[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'TestModel'" with AbsoluteModelRef.inject({test_model: "Parent"}): - assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'Parent.TestModel'" with AbsoluteModelRef.inject({test_model: "AnotherParent"}): - assert test_ptr.to_typing_code()[1] == "'AnotherParent.TestModel'" - assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'AnotherParent.TestModel'" + assert test_ptr.to_typing_code({})[1] == "'Parent.TestModel'" wrapper = DList(DList(test_ptr)) - assert wrapper.to_typing_code()[1] == "List[List['TestModel']]" + assert wrapper.to_typing_code({})[1] == "List[List['TestModel']]" with AbsoluteModelRef.inject({test_model: test_model}): - assert wrapper.to_typing_code()[1] == "List[List['TestModel.TestModel']]" + assert wrapper.to_typing_code({})[1] == "List[List['TestModel.TestModel']]" test_unicode_data = [ @@ -288,7 +290,96 @@ class Тест: @pytest.mark.parametrize("value,kwargs,expected", test_unicode_data) -def test_generated(value: ModelMeta, kwargs: dict, expected: str): +def test_unicode(value: ModelMeta, kwargs: dict, expected: str): generated = generate_code(([{"model": value, "nested": []}], {}), GenericModelCodeGenerator, class_generator_kwargs=kwargs) assert generated.rstrip() == expected, generated + + +# Data format: +# ( +# model metadata, +# style override, +# expected +# ) +test_override_style_data = [ + pytest.param( + model_factory("M", { + "bar": StringLiteral({'bar', 'foo'}) + }), + {}, + trim(f""" + {LITERAL_SOURCE} import Literal + + + class M: + bar: Literal["bar", "foo"] + """), + id='default_behaviour' + ), + pytest.param( + model_factory("M", { + "bar": StringLiteral({'bar', 'foo'}) + }), + {StringLiteral: { + StringLiteral.TypeStyle.use_literals: False + }}, + trim(""" + class M: + bar: str + """), + id='disable_literal' + ), + pytest.param( + model_factory("M", { + "bar": IntString + }), + {IntString: { + IntString.TypeStyle.use_actual_type: True + }}, + trim(""" + class M: + bar: int + """), + id='string_serializable_use_actual_type' + ), + pytest.param( + model_factory("M", { + "bar": IntString + }), + {StringSerializable: { + StringSerializable.TypeStyle.use_actual_type: True + }}, + trim(""" + class M: + bar: int + """), + id='string_serializable_use_actual_type_wildcard' + ), + pytest.param( + model_factory("M", { + "bar": IsoDateString + }), + {IsoDateString: { + IsoDateString.TypeStyle.use_actual_type: True + }}, + trim(""" + from datetime import date + + + class M: + bar: date + """), + id='string_serializable_use_actual_type_date' + ), +] + + +@pytest.mark.parametrize("value,types_style,expected", test_override_style_data) +def test_override_style(value: ModelMeta, types_style: Dict[Union['BaseType', Type['BaseType']], dict], expected: str): + generated = generate_code( + ([{"model": value, "nested": []}], {}), + GenericModelCodeGenerator, + class_generator_kwargs=dict(types_style=types_style) + ) + assert generated.rstrip() == expected, generated diff --git a/test/test_code_generation/test_pydantic_generation.py b/test/test_code_generation/test_pydantic_generation.py new file mode 100644 index 0000000..95c058f --- /dev/null +++ b/test/test_code_generation/test_pydantic_generation.py @@ -0,0 +1,209 @@ +from typing import Dict, List + +import pytest + +from json_to_models.dynamic_typing import ( + DDict, + DList, + DOptional, + DUnion, + FloatString, + IntString, + ModelMeta, + compile_imports, +) +from json_to_models.models.base import generate_code +from json_to_models.models.pydantic import PydanticModelCodeGenerator +from json_to_models.models.structure import sort_fields +from test.test_code_generation.test_models_code_generator import model_factory, trim + +# Data structure: +# pytest.param id -> { +# "model" -> (model_name, model_metadata), +# test_name -> expected, ... +# } +test_data = { + "base": { + "model": ("Test", { + "foo": int, + "Bar": int, + "baz": float + }), + "fields_data": { + "foo": { + "name": "foo", + "type": "int" + }, + "Bar": { + "name": "bar", + "type": "int", + "body": 'Field(..., alias="Bar")' + }, + "baz": { + "name": "baz", + "type": "float" + } + }, + "fields": { + "imports": "", + "fields": [ + f"foo: int", + f'bar: int = Field(..., alias="Bar")', + f"baz: float", + ] + }, + "generated": trim(f""" + from pydantic import BaseModel, Field + + + class Test(BaseModel): + foo: int + bar: int = Field(..., alias="Bar") + baz: float + """) + }, + "complex": { + "model": ("Test", { + "foo": int, + "baz": DOptional(DList(DList(str))), + "bar": DOptional(IntString), + "qwerty": FloatString, + "asdfg": DOptional(int), + "dict": DDict(int), + "not": bool, + "1day": int, + "день_недели": str, + }), + "fields_data": { + "foo": { + "name": "foo", + "type": "int" + }, + "baz": { + "name": "baz", + "type": "Optional[List[List[str]]]", + "body": "[]" + }, + "bar": { + "name": "bar", + "type": "Optional[int]", + "body": "None" + }, + "qwerty": { + "name": "qwerty", + "type": "float" + }, + "asdfg": { + "name": "asdfg", + "type": "Optional[int]", + "body": "None" + }, + "dict": { + "name": "dict_", + "type": "Dict[str, int]", + "body": 'Field(..., alias="dict")' + }, + "not": { + "name": "not_", + "type": "bool", + "body": 'Field(..., alias="not")' + }, + "1day": { + "name": "one_day", + "type": "int", + "body": 'Field(..., alias="1day")' + }, + "день_недели": { + "name": "den_nedeli", + "type": "str", + "body": 'Field(..., alias="день_недели")' + } + }, + "generated": trim(f""" + from pydantic import BaseModel, Field + from typing import Dict, List, Optional + + + class Test(BaseModel): + foo: int + qwerty: float + dict_: Dict[str, int] = Field(..., alias="dict") + not_: bool = Field(..., alias="not") + one_day: int = Field(..., alias="1day") + den_nedeli: str = Field(..., alias="день_недели") + baz: Optional[List[List[str]]] = [] + bar: Optional[int] = None + asdfg: Optional[int] = None + """) + }, + "converters": { + "model": ("Test", { + "a": int, + "b": IntString, + "c": DOptional(FloatString), + "d": DList(DList(DList(IntString))), + "e": DDict(IntString), + "u": DUnion(DDict(IntString), DList(DList(IntString))), + }), + "generated": trim(""" + from pydantic import BaseModel, Field + from typing import Dict, List, Optional, Union + + + class Test(BaseModel): + a: int + b: int + d: List[List[List[int]]] + e: Dict[str, int] + u: Union[Dict[str, int], List[List[int]]] + c: Optional[float] = None + """) + } +} + +test_data_unzip = { + test: [ + pytest.param( + model_factory(*data["model"]), + data[test], + id=id + ) + for id, data in test_data.items() + if test in data + ] + for test in ("fields_data", "fields", "generated") +} + + +@pytest.mark.parametrize("value,expected", test_data_unzip["fields_data"]) +def test_fields_data_attr(value: ModelMeta, expected: Dict[str, dict]): + gen = PydanticModelCodeGenerator(value) + required, optional = sort_fields(value) + for is_optional, fields in enumerate((required, optional)): + for field in fields: + field_imports, data = gen.field_data(field, value.type[field], bool(is_optional)) + assert data == expected[field] + + +@pytest.mark.parametrize("value,expected", test_data_unzip["fields"]) +def test_fields_attr(value: ModelMeta, expected: dict): + expected_imports: str = expected["imports"] + expected_fields: List[str] = expected["fields"] + gen = PydanticModelCodeGenerator(value) + imports, fields = gen.fields + imports = compile_imports(imports) + assert imports == expected_imports + assert fields == expected_fields + + +@pytest.mark.parametrize("value,expected", test_data_unzip["generated"]) +def test_generated_attr(value: ModelMeta, expected: str): + generated = generate_code( + ( + [{"model": value, "nested": []}], + {} + ), + PydanticModelCodeGenerator, + class_generator_kwargs={} + ) + assert generated.rstrip() == expected, generated diff --git a/test/test_dynamic_typing/test_dynamic_typing.py b/test/test_dynamic_typing/test_dynamic_typing.py index 1826c99..d8eb834 100644 --- a/test/test_dynamic_typing/test_dynamic_typing.py +++ b/test/test_dynamic_typing/test_dynamic_typing.py @@ -2,7 +2,7 @@ import pytest -from json_to_models.dynamic_typing import DUnion, get_hash_string +from json_to_models.dynamic_typing import DUnion, StringLiteral, get_hash_string # *args | MetaData test_dunion = [ @@ -20,7 +20,22 @@ [str, DUnion(int, DUnion(float, complex))], DUnion(int, float, complex, str), id="complex_merge" - ) + ), + pytest.param( + [str, StringLiteral({'a'})], + DUnion(str), + id="str_literal_to_string" + ), + pytest.param( + [StringLiteral({'b'}), StringLiteral({'a'})], + DUnion(StringLiteral({'a', 'b'})), + id="str_literal_merge" + ), + pytest.param( + [StringLiteral({str(i)}) for i in range(100)], + DUnion(str), + id="str_literal_too_much" + ), ] diff --git a/test/test_generator/test_detect_type.py b/test/test_generator/test_detect_type.py index 3404757..b2398fe 100644 --- a/test/test_generator/test_detect_type.py +++ b/test/test_generator/test_detect_type.py @@ -1,6 +1,16 @@ import pytest -from json_to_models.dynamic_typing import BooleanString, DDict, DList, DUnion, FloatString, IntString, Null, Unknown +from json_to_models.dynamic_typing import ( + BooleanString, + DDict, + DList, + DUnion, + FloatString, + IntString, + Null, + StringLiteral, + Unknown +) from json_to_models.generator import MetadataGenerator # JSON data | MetaData @@ -8,16 +18,16 @@ pytest.param(1.0, float, id="float"), pytest.param(1, int, id="int"), pytest.param(True, bool, id="bool"), - pytest.param("abc", str, id="str"), + pytest.param("abc", StringLiteral({"abc"}), id="str"), pytest.param(None, Null, id="null"), pytest.param([], DList(Unknown), id="list_empty"), pytest.param([1], DList(int), id="list_single"), pytest.param([*range(100)], DList(int), id="list_single_type"), - pytest.param([1, "a", 2, "c"], DList(DUnion(int, str)), id="list_multi"), + pytest.param([1, "a", 2, "c"], DList(DUnion(int, StringLiteral({'a', 'c'}))), id="list_multi"), pytest.param("1", IntString, id="int_str"), pytest.param("1.0", FloatString, id="float_str"), pytest.param("true", BooleanString, id="bool_str"), - pytest.param({"test_dict_field_a": 1, "test_dict_field_b": "a"}, DDict(DUnion(int, str)), id="dict"), + pytest.param({"test_dict_field_a": 1, "test_dict_field_b": "a"}, DDict(DUnion(int, StringLiteral({"a"}))), id="simple_dict"), pytest.param({}, DDict(Unknown)) ] @@ -50,7 +60,7 @@ def test_convert(models_generator: MetadataGenerator): meta = models_generator._convert(data) assert meta == { "dict_field": DDict(Unknown), - "another_dict_field": DDict(DUnion(int, str)), + "another_dict_field": DDict(DUnion(int, StringLiteral({"a"}))), "another_dict_field_2": DDict(int), "another_dict_field_3": DDict(int), "int_field": int, diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index 71d550c..1b6eb84 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -8,8 +8,8 @@ from json_to_models.dynamic_typing import register_datetime_classes from json_to_models.generator import MetadataGenerator from json_to_models.models.base import generate_code -from json_to_models.models.dataclasses import DataclassModelCodeGenerator -from json_to_models.models.structure import compose_models +from json_to_models.models.pydantic import PydanticModelCodeGenerator +from json_to_models.models.structure import compose_models_flat from json_to_models.registry import ModelRegistry from testing_tools.pprint_meta_data import pretty_format_meta from testing_tools.real_apis import dump_response @@ -46,7 +46,8 @@ def main(): register_datetime_classes() gen = MetadataGenerator() reg = ModelRegistry() - for name, data in (results_data, drivers_data, driver_standings_data): + # for name, data in (results_data, drivers_data, driver_standings_data): + for name, data in (driver_standings_data,): fields = gen.generate(*data) reg.process_meta_data(fields, model_name=inflection.camelize(name)) reg.merge_models(generator=gen) @@ -56,11 +57,11 @@ def main(): print(pretty_format_meta(model)) print("=" * 20, end='') - structure = compose_models(reg.models_map) + structure = compose_models_flat(reg.models_map) # print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) # print("=" * 20) - print(generate_code(structure, DataclassModelCodeGenerator, class_generator_kwargs={"post_init_converters": True})) + print(generate_code(structure, PydanticModelCodeGenerator, class_generator_kwargs={})) if __name__ == '__main__': diff --git a/testing_tools/real_apis/large_data_set.py b/testing_tools/real_apis/large_data_set.py index 0336ad0..6c13f8b 100644 --- a/testing_tools/real_apis/large_data_set.py +++ b/testing_tools/real_apis/large_data_set.py @@ -11,7 +11,7 @@ def load_data() -> dict: - with (Path(__file__) / ".." / ".." / "large_data_set.json").open() as f: + with (Path(__file__) / ".." / ".." / "large_data_set.json").resolve().open() as f: data = json.load(f) return data diff --git a/testing_tools/real_apis/openlibrary.py b/testing_tools/real_apis/openlibrary.py index 37e6625..f91fe90 100644 --- a/testing_tools/real_apis/openlibrary.py +++ b/testing_tools/real_apis/openlibrary.py @@ -4,9 +4,9 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.structure import compose_models from json_to_models.registry import ModelRegistry from testing_tools.real_apis import dump_response diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index 8ded980..0443109 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -6,11 +6,10 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.pydantic import PydanticModelCodeGenerator from json_to_models.models.structure import compose_models_flat from json_to_models.registry import ModelRegistry -from testing_tools.pprint_meta_data import pretty_format_meta from testing_tools.real_apis import dump_response @@ -33,15 +32,15 @@ def main(): reg.merge_models(generator=gen) reg.generate_names() - print("Meta tree:") - print(pretty_format_meta(next(iter(reg.models)))) - print("\n" + "=" * 20, end='') + # print("Meta tree:") + # print(pretty_format_meta(next(iter(reg.models)))) + # print("\n" + "=" * 20, end='') structure = compose_models_flat(reg.models_map) # print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) # print("=" * 20) - print(generate_code(structure, AttrsModelCodeGenerator)) + print(generate_code(structure, PydanticModelCodeGenerator)) print(f"{(datetime.now() - start_t).total_seconds():.4f} seconds") diff --git a/testing_tools/real_apis/spotify-swagger.py b/testing_tools/real_apis/spotify-swagger.py index 67dd84a..96d3c47 100644 --- a/testing_tools/real_apis/spotify-swagger.py +++ b/testing_tools/real_apis/spotify-swagger.py @@ -23,7 +23,7 @@ def to_representation(self) -> str: def load_data() -> dict: - with (Path(__file__) / ".." / ".." / "spotify-swagger.yaml").open() as f: + with (Path(__file__) / ".." / ".." / "spotify-swagger.yaml").resolve().open() as f: data = yaml.load(f, Loader=yaml.SafeLoader) return data diff --git a/testing_tools/real_apis/swagger.py b/testing_tools/real_apis/swagger.py index 7e83443..76d43ab 100644 --- a/testing_tools/real_apis/swagger.py +++ b/testing_tools/real_apis/swagger.py @@ -22,7 +22,7 @@ def to_representation(self) -> str: def load_data() -> dict: - with (Path(__file__) / ".." / ".." / "swagger.json").open() as f: + with (Path(__file__) / ".." / ".." / "swagger.json").resolve().open() as f: data = json.load(f) return data