diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 5790f5c..201df53 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -12,7 +12,7 @@ import json_to_models from json_to_models.dynamic_typing import ModelMeta, register_datetime_classes from json_to_models.generator import MetadataGenerator -from json_to_models.models import ModelsStructureType, compose_models +from json_to_models.models import ModelsStructureType, compose_models, compose_models_flat from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import GenericModelCodeGenerator, generate_code from json_to_models.models.dataclasses import DataclassModelCodeGenerator @@ -34,8 +34,7 @@ class Cli: STRUCTURE_FN_MAPPING: Dict[str, STRUCTURE_FN_TYPE] = { "nested": compose_models, - # TODO: vvvvvvvvvvvv - "flat": lambda *args, **kwargs: None + "flat": compose_models_flat } MODEL_GENERATOR_MAPPING: Dict[str, Type[GenericModelCodeGenerator]] = { @@ -187,8 +186,8 @@ def set_args(self, merge_policy: List[Union[List[str], str]], self.initialize = True - @staticmethod - def _create_argparser() -> argparse.ArgumentParser: + @classmethod + def _create_argparser(cls) -> argparse.ArgumentParser: """ ArgParser factory """ @@ -255,13 +254,13 @@ def _create_argparser() -> argparse.ArgumentParser: parser.add_argument( "-s", "--structure", default="nested", - choices=["nested", "flat"], + choices=list(cls.STRUCTURE_FN_MAPPING.keys()), help="Models composition style. By default nested models become nested Python classes.\n\n" ) parser.add_argument( "-f", "--framework", default="base", - choices=["base", "attrs", "dataclasses", "custom"], + choices=list(cls.MODEL_GENERATOR_MAPPING.keys()) + ["custom"], help="Model framework for which python code is generated.\n" "'base' (default) mean no framework so code will be generated without any decorators\n" "and additional meta-data.\n" diff --git a/json_to_models/dynamic_typing/models_meta.py b/json_to_models/dynamic_typing/models_meta.py index 5fd74fa..895086d 100644 --- a/json_to_models/dynamic_typing/models_meta.py +++ b/json_to_models/dynamic_typing/models_meta.py @@ -23,6 +23,9 @@ def __init__(self, t: MetaData, index, _original_fields=None): def __str__(self): return f"Model#{self.index}" + ("-" + self._name if self._name else "") + def __repr__(self): + return f"<{self}>" + def __eq__(self, other): if isinstance(other, dict): return self.type == other diff --git a/json_to_models/generator.py b/json_to_models/generator.py index b0a0acb..6921f95 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -9,7 +9,7 @@ StringSerializable, StringSerializableRegistry, Unknown, registry) keywords_set = set(keyword.kwlist) - +_static_types = {float, bool, int} class MetadataGenerator: CONVERTER_TYPE = Optional[Callable[[str], Any]] @@ -61,15 +61,12 @@ def _detect_type(self, value, convert_dict=True) -> MetaData: Converts json value to metadata """ # Simple types - if isinstance(value, float): - return float - elif isinstance(value, bool): - return bool - elif isinstance(value, int): - return int + t = type(value) + if t in _static_types: + return t # List trying to yield nested type - elif isinstance(value, list): + elif t is list: if value: types = [self._detect_type(item) for item in value] if len(types) > 1: diff --git a/json_to_models/models/__init__.py b/json_to_models/models/__init__.py index 1743741..be56da4 100644 --- a/json_to_models/models/__init__.py +++ b/json_to_models/models/__init__.py @@ -1,8 +1,11 @@ -from typing import Dict, Generic, Iterable, List, Set, Tuple, TypeVar +from collections import defaultdict +from typing import Dict, Generic, Iterable, List, Set, Tuple, TypeVar, Union from ..dynamic_typing import DOptional, ModelMeta, ModelPtr Index = str +ModelsStructureType = Tuple[List[dict], Dict[ModelMeta, ModelMeta]] + T = TypeVar('T') @@ -26,40 +29,44 @@ def insert_before(self, value: T, *before: T): raise ValueError pos = min(ix) self.insert(pos, value) + return pos def insert_after(self, value: T, *after: T): ix = self._safe_indexes(*after) if not ix: raise ValueError - pos = max(ix) - self.insert(pos + 1, value) - + pos = max(ix) + 1 + self.insert(pos, value) + return pos -def filter_pointers(model: ModelMeta) -> Iterable[ModelPtr]: - """ - Return iterator over pointers with not None parent - """ - return (ptr for ptr in model.pointers if ptr.parent) +class PositionsDict(defaultdict): + # Dict contains mapping Index -> position, where position is list index to insert nested element of Index + INC = object() -def extract_root(model: ModelMeta) -> Set[Index]: - """ - Return set of indexes of root models that are use given ``model`` directly or through another nested model. - """ - seen: Set[Index] = set() - nodes: List[ModelPtr] = list(filter_pointers(model)) - roots: Set[Index] = set() - while nodes: - node = nodes.pop() - seen.add(node.type.index) - filtered = list(filter_pointers(node.parent)) - nodes.extend(ptr for ptr in filtered if ptr.type.index not in seen) - if not filtered: - roots.add(node.parent.index) - return roots + def __init__(self, default_factory=int, **kwargs): + super().__init__(default_factory, **kwargs) + def update_position(self, key: str, value: Union[object, int]): + """ + Shift all elements which are placed after updated one -ModelsStructureType = Tuple[List[dict], Dict[ModelMeta, ModelMeta]] + :param key: Index or "root" + :param value: Could be position or PositionsDict.INC to perform quick increment (x+=1) + :return: + """ + if value is self.INC: + value = self[key] + 1 + if key in self: + old_value = self[key] + delta = value - old_value + else: + old_value = value + delta = 1 + for k, v in self.items(): + if k != key and v >= old_value: + self[k] += delta + self[key] = value def compose_models(models_map: Dict[str, ModelMeta]) -> ModelsStructureType: @@ -116,6 +123,84 @@ def compose_models(models_map: Dict[str, ModelMeta]) -> ModelsStructureType: return root_models, path_injections +def compose_models_flat(models_map: Dict[Index, ModelMeta]) -> ModelsStructureType: + """ + Generate flat sorted (by nesting level, ASC) models structure for internal usage. + + :param models_map: Mapping (model index -> model meta instance). + :return: List of root models data, Map(child model -> root model) for absolute ref generation + """ + root_models = ListEx() + positions: PositionsDict[Index, int] = PositionsDict() + top_level_models: Set[Index] = set() + structure_hash_table: Dict[Index, dict] = { + key: { + "model": model, + "nested": ListEx(), + "roots": list(extract_root(model)), # Indexes of root level models + } for key, model in models_map.items() + } + + for key, model in models_map.items(): + pointers = list(filter_pointers(model)) + has_root_pointers = len(pointers) != len(model.pointers) + if not pointers: + # Root level model + if not has_root_pointers: + raise Exception(f'Model {model.name} has no pointers') + root_models.insert(positions["root"], structure_hash_table[key]) + top_level_models.add(key) + positions.update_position("root", PositionsDict.INC) + else: + parents = {ptr.parent.index for ptr in pointers} + struct = structure_hash_table[key] + # Model is using by other models + if has_root_pointers or len(parents) > 1 and len(struct["roots"]) >= 1: + # Model is using by different root models + if parents & top_level_models: + parents.add("root") + parents_positions = {positions[parent_key] for parent_key in parents + if parent_key in positions} + parents_joined = "#".join(sorted(parents)) + if parents_joined in positions: + parents_positions.add(positions[parents_joined]) + pos = max(parents_positions) if parents_positions else len(root_models) + positions.update_position(parents_joined, pos + 1) + else: + # Model is using by only one model + parent = next(iter(parents)) + pos = positions.get(parent, len(root_models)) + positions.update_position(parent, pos + 1) + positions.update_position(key, pos + 1) + root_models.insert(pos, struct) + + return root_models, {} + + +def filter_pointers(model: ModelMeta) -> Iterable[ModelPtr]: + """ + Return iterator over pointers with not None parent + """ + return (ptr for ptr in model.pointers if ptr.parent) + + +def extract_root(model: ModelMeta) -> Set[Index]: + """ + Return set of indexes of root models that are use given ``model`` directly or through another nested model. + """ + seen: Set[Index] = set() + nodes: List[ModelPtr] = list(filter_pointers(model)) + roots: Set[Index] = set() + while nodes: + node = nodes.pop() + seen.add(node.type.index) + filtered = list(filter_pointers(node.parent)) + nodes.extend(ptr for ptr in filtered if ptr.type.index not in seen) + if not filtered: + roots.add(node.parent.index) + return roots + + def sort_fields(model_meta: ModelMeta) -> Tuple[List[str], List[str]]: """ Split fields into required and optional groups diff --git a/json_to_models/registry.py b/json_to_models/registry.py index 5e7152d..2e7c800 100644 --- a/json_to_models/registry.py +++ b/json_to_models/registry.py @@ -2,6 +2,8 @@ from itertools import chain, combinations from typing import Dict, List, Set, Tuple +from ordered_set import OrderedSet + from .dynamic_typing import BaseType, MetaData, ModelMeta, ModelPtr from .utils import Index, distinct_words @@ -151,7 +153,7 @@ def merge_models(self, generator, strict=False) -> List[Tuple[ModelMeta, Set[Mod flag = True while flag: flag = False - new_groups: Set[Set[ModelMeta]] = set() + new_groups: OrderedSet[Set[ModelMeta]] = OrderedSet() for gr1, gr2 in combinations(groups, 2): if gr1 & gr2: old_len = len(new_groups) diff --git a/requirements.txt b/requirements.txt index 902cc7d..d73fc8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ python-dateutil>=2.7.* inflection>=0.3.* unidecode>=1.0.* -Jinja2>=2.10.* \ No newline at end of file +Jinja2>=2.10.* +ordered-set==3.* \ No newline at end of file diff --git a/setup.py b/setup.py index d2a9070..0929a37 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,4 @@ +import multiprocessing import sys from setuptools import find_packages, setup @@ -9,6 +10,8 @@ required = f.read().splitlines() URL = "https://github.com/bogdandm/json2python-models" +CPU_N = multiprocessing.cpu_count() + class PyTest(TestCommand): user_options = [("pytest-args=", "a", "Arguments to pass to pytest")] @@ -20,8 +23,10 @@ def initialize_options(self): def run_tests(self): import shlex import pytest - - errno = pytest.main(shlex.split(self.pytest_args + ' -m "not slow_http"')) + args = self.pytest_args + if CPU_N > 1 and "-n " not in args: + args += f" -n {CPU_N}" + errno = pytest.main(shlex.split(args)) sys.exit(errno) @@ -31,6 +36,7 @@ def run_tests(self): python_requires=">=3.7", url=URL, author="bogdandm (Bogdan Kalashnikov)", + author_email="bogdan.dm1995@yandex.ru", description="Python models (attrs, dataclasses or custom) generator from JSON data with typing module support", license="MIT", packages=find_packages(exclude=['test', 'testing_tools']), @@ -39,9 +45,6 @@ def run_tests(self): }, install_requires=required, cmdclass={"test": PyTest}, - tests_require=["pytest", "requests", "attrs"], - project_urls={ - 'Source': URL - }, + tests_require=["pytest", "pytest-xdist", "requests", "attrs"], data_files=[('', ['pytest.ini', '.coveragerc', 'LICENSE'])] ) diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index cefae48..f674b13 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -92,6 +92,14 @@ def test_script(command): print(stdout) +@pytest.mark.parametrize("command", test_commands) +def test_script_flat(command): + command += " -s flat" + proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = _validate_result(proc) + print(stdout) + + @pytest.mark.parametrize("command", test_commands) def test_script_attrs(command): command += " -f attrs" diff --git a/test/test_code_generation/test_models_composition.py b/test/test_code_generation/test_models_composition.py index 891b97e..76ddc12 100644 --- a/test/test_code_generation/test_models_composition.py +++ b/test/test_code_generation/test_models_composition.py @@ -4,7 +4,7 @@ from json_to_models.dynamic_typing import ModelMeta from json_to_models.generator import MetadataGenerator -from json_to_models.models import ListEx, compose_models, extract_root +from json_to_models.models import ListEx, compose_models, compose_models_flat, extract_root from json_to_models.registry import ModelRegistry @@ -20,16 +20,6 @@ def test_list_ex(): assert l == [0, 'a', *range(1, 6), 'b', *range(6, 10)] -def indexes_to_names(reg: ModelRegistry, *ix): - for i in ix: - yield reg.models_map[i].name - - -def names_to_indexes(reg: ModelRegistry, *ix): - for i in ix: - yield reg.models_map[i].name - - # This test relies on model names as a some sort of models ids # and may fail if some logic of their generation will be changed # List of Tuple[root_model_name, JSON data] | Dict[model_name, Set[root_model_names]] @@ -110,6 +100,33 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode base_dict = {"field_" + str(i): int for i in range(20)} + +def _test_compose_models( + function, request, + models_generator: MetadataGenerator, models_registry: ModelRegistry, + value: List[Tuple[str, dict]], expected: List[Tuple[str, list]], expected_mapping: Dict[str, str] +): + for model_name, metadata in value: + models_registry.process_meta_data(metadata, model_name=model_name) + models_registry.merge_models(models_generator) + models_registry.generate_names() + names_map = {model.index: model.name for model in models_registry.models} + names_map.update({model.name: model.index for model in models_registry.models}) + root, mapping = function(models_registry.models_map) + + def check(nested_value: List[dict], nested_expected: List[Tuple[str, list]]): + for model_dict, (model_name, nested) in zip(nested_value, nested_expected): + assert model_dict["model"].name == model_name, str(nested_value) + assert len(model_dict["nested"]) == len(nested), f"(Parent model is {model_name})" + check(model_dict["nested"], nested) + + check(root, expected) + + name = lambda model: model.name if isinstance(model, ModelMeta) else model + mapping = {name(model): name(parent) for model, parent in mapping.items()} + assert mapping == expected_mapping + + # This test relies on model names as a some sort of models ids # List of Tuple[root_model_name, model-meta] | List[Tuple[model_name, nested_models]]] # where nested_models is a recursive definition @@ -281,25 +298,316 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode @pytest.mark.parametrize("value,expected,expected_mapping", test_compose_models_data) def test_compose_models( + request, models_generator: MetadataGenerator, models_registry: ModelRegistry, value: List[Tuple[str, dict]], expected: List[Tuple[str, list]], expected_mapping: Dict[str, str] ): - for model_name, metadata in value: - models_registry.process_meta_data(metadata, model_name=model_name) - models_registry.merge_models(models_generator) - models_registry.generate_names() - names_map = {model.index: model.name for model in models_registry.models} - names_map.update({model.name: model.index for model in models_registry.models}) - root, mapping = compose_models(models_registry.models_map) + _test_compose_models(compose_models, request, models_generator, models_registry, value, expected, expected_mapping) - def check(nested_value: List[dict], nested_expected: List[Tuple[str, list]]): - for model_dict, (model_name, nested) in zip(nested_value, nested_expected): - assert model_dict["model"].name == model_name - assert len(model_dict["nested"]) == len(nested), f"(Parent model is {model_name})" - check(model_dict["nested"], nested) - check(root, expected) +test_compose_models_flat_data = [ + pytest.param( + [ + ("A", { + "field1": int, + "item": { + "another_field": float + } + }) + ], + [ + ("A", []), + ("Item", []) + ], + {}, + id="basic_test" + ), + pytest.param( + [ + ("RootA", { + "item": { + "field": float + } + }), + ("RootB", { + "item": { + "field": float + }, + "idontwantrootmodelstomerge": bool + }) + ], + [ + ("RootA", []), + ("RootB", []), + ("Item", []), + ], + {}, + id="global_nested_model" + ), + pytest.param( + [ + ("RootA", { + "item": { + "field": float + } + }), + ("RootB", { + "item": { + "field": float + } + }) + ], + [ + ("RootA_RootB", []), + ("Item", []) + ], + {}, + id="roots_merge" + ), + pytest.param( + [ + ("RootFirst", { + "root_field": float + }), + ("RootA", { + "item": { + "field": float + } + }), + ("RootB", { + "item": { + "field": float + }, + "idontwantrootmodelstomerge": bool + }) + ], + [ + ("RootFirst", []), + ("RootA", []), + ("RootB", []), + ("Item", []), + ], + {}, + id="root_order" + ), + pytest.param( + [ + ("Root", { + "model_a": { + "field_a": { + "field": float + } + }, + "model_b": { + "field_b": { + "field": float + } + } + }), + ], + [ + ("Root", []), + ("ModelA", []), + ("ModelB", []), + ("FieldA_FieldB", []), - name = lambda model: model.name if isinstance(model, ModelMeta) else model - mapping = {name(model): name(parent) for model, parent in mapping.items()} - assert mapping == expected_mapping + ], + {}, + id="generic_in_nested_models" + ), + pytest.param( + [ + ("RootItem", { + "field": float + }), + ("RootA", { + "item": { + "field": float + }, + **base_dict + }), + ("RootB", base_dict) + ], + [ + ("RootA_RootB", []), + ("RootItem", []), + ], + {}, + id="merge_with_root_model" + ), + pytest.param( + [ + ("Root", { + "model_a": { + "field_a": { + "field": { + "nested_field": float + } + } + }, + "model_b": { + "field_b": { + "field": { + "nested_field": float + } + } + } + }), + ], + [ + ("Root", []), + ("ModelA", []), + ("ModelB", []), + ("FieldA_FieldB", []), + ("Field", []) + ], + {}, + id="generic_in_nested_models_with_nested_model" + ), + pytest.param( + [ + ("RootA", { + "field_a1": {"nested": {"field": int}}, + "field_a2": {"nested_2": {"field_2": int}}, + "nested": {"field": int} + }), + ("RootB", { + "field_b1": {"nested": {"field": int}}, + "field_b2": {"nested_2": {"field_2": int}}, + "nested": {"field": int} + }), + ("RootC", { + "field_c1": float + }), + ], + [ + ("RootA", []), + ("RootB", []), + ("RootC", []), + ("FieldA1_FieldB1", []), + ("Nested", []), + ("FieldA2_FieldB2", []), + ("Nested2", []), + ], + {}, + id="sort_1" + ), + pytest.param( + [ + ("RootA", { + "field_a1": {"field_a1_n": {"field": int}}, + "field_a2": {"field_a2_n": {"field": int}}, + "field_a3": {"field_a3_n": {"field": int}}, + "field_a4": {"field_a4_n": {"field": int}}, + }), + ("RootB", { + "field_b1": {"field_b1_n": {"field": int}}, + "field_b2": {"field_b2_n": {"field": int}}, + "field_b3": {"field_b3_n": {"field": int}}, + "field_b4": {"field_b4_n": {"field": int}}, + }), + ("RootC", { + "field_c1": {"field_c1_n": {"field": int}}, + "field_c2": {"field_c2_n": {"field": int}}, + "field_c3": {"field_c3_n": {"field": int}}, + "field_c4": {"field_c4_n": {"field": int}}, + }), + ], + [ + ("RootA", []), + ("RootB", []), + ("RootC", []), + ("FieldA1", []), + ("FieldA2", []), + ("FieldA3", []), + ("FieldA4", []), + ("FieldB1", []), + ("FieldB2", []), + ("FieldB3", []), + ("FieldB4", []), + ("FieldC1", []), + ("FieldC2", []), + ("FieldC3", []), + ("FieldC4", []), + ("FieldA1N_FieldA2N_FieldA3N_FieldA4N_" + "FieldB1N_FieldB2N_FieldB3N_FieldB4N_" + "FieldC1N_FieldC2N_FieldC3N_FieldC4N", []), + ], + {}, + id="sort_2" + ), + pytest.param( + [ + ("RootA", { + "field_a1": { + "field_a1_n": {"field_1": int}, + "field_a1_n2": {"field_11": int}, + }, + "field_a2": {"field_a2_n": {"field_2": int}}, + "field_a3": {"field_a3_n": {"field_3": int}}, + "field_a4": {"field_a4_n": {"field_4": int}}, + }), + ("RootB", { + "field_b1": { + "field_b1_n": {"field_01": int}, + "field_b1_n2": {"field_b11": int}, + "field_b1_n3": {"field_b21": int}, + }, + "field_b2": {"field_b2_n": {"field_02": int}}, + "field_b3": { + "field_b3_n": {"field_03": int}, + "field_b3_n2": {"field_b13": int}, + "field_b3_n3": {"field_b23": int}, + "field_b3_n4": {"field_b33": int}, + }, + "field_b4": {"field_b4_n": {"field_04": int}}, + }), + ], + [ + ("RootA", []), + ("RootB", []), + + ("FieldA1", []), + ("FieldA1N", []), + ("FieldA1N2", []), + + ("FieldA2", []), + ("FieldA2N", []), + + ("FieldA3", []), + ("FieldA3N", []), + + ("FieldA4", []), + ("FieldA4N", []), + + ("FieldB1", []), + ("FieldB1N", []), + ("FieldB1N2", []), + ("FieldB1N3", []), + + ("FieldB2", []), + ("FieldB2N", []), + + ("FieldB3", []), + ("FieldB3N", []), + ("FieldB3N2", []), + ("FieldB3N3", []), + ("FieldB3N4", []), + + ("FieldB4", []), + ("FieldB4N", []), + ], + {}, + id="sort_3" + ), +] + + +@pytest.mark.parametrize("value,expected,expected_mapping", test_compose_models_flat_data) +def test_compose_models_flat( + request, + models_generator: MetadataGenerator, models_registry: ModelRegistry, + value: List[Tuple[str, dict]], expected: List[Tuple[str, list]], expected_mapping: Dict[str, str] +): + _test_compose_models(compose_models_flat, request, models_generator, models_registry, value, expected, + expected_mapping) diff --git a/testing_tools/real_apis/large_data_set.py b/testing_tools/real_apis/large_data_set.py index 5179053..4d5b789 100644 --- a/testing_tools/real_apis/large_data_set.py +++ b/testing_tools/real_apis/large_data_set.py @@ -3,9 +3,10 @@ from pathlib import Path from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models +from json_to_models.models import compose_models, compose_models_flat from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code +from json_to_models.models.dataclasses import DataclassModelCodeGenerator from json_to_models.registry import ModelRegistry @@ -30,11 +31,15 @@ def main(): reg.generate_names() structure = compose_models(reg.models_map) - code = generate_code(structure, AttrsModelCodeGenerator, class_generator_kwargs={"no_meta": True}) + code = generate_code(structure, AttrsModelCodeGenerator) + print(code) + + print("=" * 10, f"{(datetime.now() - start_t).total_seconds():.4f} seconds", "=" * 10, + "\nPress enter to continue...\n") + input() + structure_flat = compose_models_flat(reg.models_map) + code = generate_code(structure_flat, DataclassModelCodeGenerator) print(code) - # with open("tmp.py", "w") as f: - # f.write(code) - print(f"{(datetime.now() - start_t).total_seconds():.4f} seconds") if __name__ == '__main__': diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index 20112ad..ea6335c 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -6,7 +6,7 @@ import requests from json_to_models.generator import MetadataGenerator -from json_to_models.models import compose_models +from json_to_models.models import compose_models_flat from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import generate_code from json_to_models.registry import ModelRegistry @@ -37,7 +37,7 @@ def main(): print(pretty_format_meta(next(iter(reg.models)))) print("\n" + "=" * 20, end='') - structure = compose_models(reg.models_map) + structure = compose_models_flat(reg.models_map) print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) print("=" * 20)