diff --git a/TODO.md b/TODO.md index a0c7726..09cc808 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,5 @@ - (!) README.md +- Remove OrderedDict (dictionaries in Python 3.7 are now ordered) - Features - Models layer - [X] Data variant converting diff --git a/requirements.txt b/requirements.txt index a668a1f..868ace4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ python-dateutil==2.7.* inflection==0.3.* unidecode==1.0.* -ordered-set==3.0.* \ No newline at end of file +ordered-set==3.0.* +Jinja2==2.10.* \ No newline at end of file diff --git a/rest_client_gen/models/__init__.py b/rest_client_gen/models/__init__.py index de021ee..95c6bc9 100644 --- a/rest_client_gen/models/__init__.py +++ b/rest_client_gen/models/__init__.py @@ -1,5 +1,6 @@ -from typing import Dict, Generic, Iterable, List, Set, TypeVar +from typing import Dict, Generic, Iterable, List, Set, Tuple, TypeVar +from rest_client_gen.dynamic_typing import DOptional from ..dynamic_typing import ModelMeta, ModelPtr Index = str @@ -59,7 +60,10 @@ def extract_root(model: ModelMeta) -> Set[Index]: return roots -def compose_models(models_map: Dict[str, ModelMeta]): +def compose_models(models_map: Dict[str, ModelMeta]) -> List[dict]: + """ + Generate nested sorted models structure for internal usage. + """ root_models = ListEx() root_nested_ix = 0 structure_hash_table: Dict[Index, dict] = { @@ -81,8 +85,10 @@ def compose_models(models_map: Dict[str, ModelMeta]): else: parents = {ptr.parent.index for ptr in pointers} struct = structure_hash_table[key] + # FIXME: "Model is using by single root model" case for the time being will be disabled + # until solution to make typing ref such as 'Parent.Child' will be found # Model is using by other models - if has_root_pointers or len(parents) > 1 and len(struct["roots"]) > 1: + if has_root_pointers or len(parents) > 1: # and len(struct["roots"]) > 1 # Model is using by different root models try: root_models.insert_before( @@ -92,10 +98,10 @@ def compose_models(models_map: Dict[str, ModelMeta]): except ValueError: root_models.insert(root_nested_ix, struct) root_nested_ix += 1 - elif len(parents) > 1 and len(struct["roots"]) == 1: - # Model is using by single root model - parent = structure_hash_table[struct["roots"][0]] - parent["nested"].insert(0, struct) + # elif len(parents) > 1 and len(struct["roots"]) == 1: + # # Model is using by single root model + # parent = structure_hash_table[struct["roots"][0]] + # parent["nested"].insert(0, struct) else: # Model is using by only one model parent = structure_hash_table[next(iter(parents))] @@ -103,3 +109,31 @@ def compose_models(models_map: Dict[str, ModelMeta]): parent["nested"].append(struct) return root_models + + +def sort_fields(model_meta: ModelMeta) -> Tuple[List[str], List[str]]: + """ + Split fields into required and optional groups + + :return: two list of fields names: required fields, optional fields + """ + fields = model_meta.type + required = [] + optional = [] + for key, meta in fields.items(): + if isinstance(meta, DOptional): + optional.append(key) + else: + required.append(key) + return required, optional + + +INDENT = " " * 4 +OBJECTS_DELIMITER = "\n" * 3 # 2 blank lines + + +def indent(string: str, lvl: int = 1, indent: str = INDENT) -> str: + """ + Indent all lines of string by ``indent * lvl`` + """ + return "\n".join(indent * lvl + line for line in string.split("\n")) diff --git a/rest_client_gen/models/base.py b/rest_client_gen/models/base.py new file mode 100644 index 0000000..02bca43 --- /dev/null +++ b/rest_client_gen/models/base.py @@ -0,0 +1,157 @@ +from typing import List, Tuple, Type + +from jinja2 import Template + +from rest_client_gen.dynamic_typing import compile_imports +from rest_client_gen.models import INDENT, OBJECTS_DELIMITER +from . import indent, sort_fields +from ..dynamic_typing import ImportPathList, MetaData, ModelMeta, metadata_to_typing + + +def template(pattern: str, indent: str = INDENT) -> Template: + """ + Remove indent from triple-quotes string and return jinja2.Template instance + """ + if "\n" in pattern: + n = len(indent) + lines = pattern.split("\n") + for i in (0, -1): + if not lines[i].strip(): + del lines[i] + + pattern = "\n".join(line[n:] if line[:n] == indent else line + for line in lines) + return Template(pattern) + + +class GenericModelCodeGenerator: + """ + Core of model code generator. Extend it to customize fields of model or add some decorators. + Note that this class has nothing to do with models structure. It only can add nested models as strings. + """ + BODY = template(""" + {%- for decorator in decorators -%} + @{{ decorator }} + {% endfor -%} + class {{ name }}: + + {%- for code in nested %} + {{ code }} + {% endfor -%} + + {%- for field in fields %} + {{ field }} + {%- endfor %} + """) + + FIELD: Template = template("{{name}}: {{type}}{% if body %} = {{ body }}{% endif %}") + + def __init__(self, model: ModelMeta, **kwargs): + self.model = model + + + def generate(self, nested_classes: List[str] = None) -> Tuple[ImportPathList, str]: + """ + :param nested_classes: list of strings that contains classes code + :return: list of import data, class code + """ + imports, fields = self.fields + data = { + "decorators": self.decorators, + "name": self.model.name, + "fields": fields + } + if nested_classes: + data["nested"] = [indent(s) for s in nested_classes] + return imports, self.BODY.render(**data) + + @property + def decorators(self) -> List[str]: + """ + :return: List of decorators code (without @) + """ + return [] + + def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: + """ + Form field data for template + + :param name: Field name + :param meta: Field metadata + :param optional: Is field optional + :return: imports, field data + """ + imports, typing = metadata_to_typing(meta) + data = { + "name": name, + "type": typing + } + return imports, data + + @property + def fields(self) -> Tuple[ImportPathList, List[str]]: + """ + Generate fields strings + + :return: imports, list of fields as string + """ + required, optional = sort_fields(self.model) + imports: ImportPathList = [] + strings: List[str] = [] + for is_optional, fields in enumerate((required, optional)): + for field in fields: + field_imports, data = self.field_data(field, self.model.type[field], bool(is_optional)) + imports.extend(field_imports) + strings.append(self.FIELD.render(**data)) + return imports, strings + + +def _generate_code( + structure: List[dict], + class_generator: Type[GenericModelCodeGenerator], + class_generator_kwargs: dict, + lvl=0 +) -> Tuple[ImportPathList, List[str]]: + """ + Walk thought models structure and covert them into code + + :param structure: Result of compose_models or similar function + :param class_generator: GenericModelCodeGenerator subclass + :param class_generator_kwargs: kwags for GenericModelCodeGenerator init + :param lvl: Recursion depth + :return: imports, list of first lvl classes + """ + imports = [] + classes = [] + for data in structure: + nested_imports, nested_classes = _generate_code( + data["nested"], + class_generator, + class_generator_kwargs, + lvl=lvl + 1 + ) + imports.extend(nested_imports) + gen = class_generator(data["model"], **class_generator_kwargs) + cls_imports, cls_string = gen.generate(nested_classes) + imports.extend(cls_imports) + classes.append(cls_string) + return imports, classes + + +def generate_code(structure: List[dict], class_generator: Type[GenericModelCodeGenerator], + class_generator_kwargs: dict = None, objects_delimiter: str = OBJECTS_DELIMITER) -> str: + """ + Generate ready-to-use code + + :param structure: Result of compose_models or similar function + :param class_generator: GenericModelCodeGenerator subclass + :param class_generator_kwargs: kwags for GenericModelCodeGenerator init + :param objects_delimiter: Delimiter between root level classes + :return: Generated code + """ + imports, classes = _generate_code(structure, class_generator, class_generator_kwargs or {}) + if imports: + imports_str = compile_imports(imports) + objects_delimiter + else: + imports_str = "" + return imports_str + objects_delimiter.join(classes) + "\n" diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py new file mode 100644 index 0000000..e181c86 --- /dev/null +++ b/test/test_code_generation/test_models_code_generator.py @@ -0,0 +1,176 @@ +from typing import Dict, List + +import pytest + +from rest_client_gen.dynamic_typing import DList, DOptional, IntString, ModelMeta, compile_imports +from rest_client_gen.models import indent, sort_fields +from rest_client_gen.models.base import GenericModelCodeGenerator, generate_code + +test_indent_data = [ + pytest.param( + ("1", 1, " " * 4), + " 1" + ), + pytest.param( + ("1\n2", 1, " " * 4), + " 1\n 2" + ), + pytest.param( + ("1\n2", 2, " " * 4), + " 1\n 2" + ), + pytest.param( + ("1\n 2", 2, " " * 4), + " 1\n 2" + ), +] + + +@pytest.mark.parametrize("args,expected", test_indent_data) +def test_indent(args, expected): + assert indent(*args) == expected + + +def model_factory(name: str, metadata: dict): + model = ModelMeta(metadata, name) + model.set_raw_name(name) + return model + + +INDENT = " " * 4 * 2 + + +def trim(s: str): + if "\n" in s: + n = len(INDENT) + lines = s.split("\n") + for i in (0, -1): + if not lines[i].strip(): + del lines[i] + + s = "\n".join(line[n:] if line[:n] == INDENT else line for line in lines) + return s + + +# Data structure: +# pytest.param id -> { +# "model" -> (model_name, model_metadata), +# test_name -> expected, ... +# } +test_data = { + "base": { + "model": ("Test", { + "foo": int, + "bar": int, + "baz": float + }), + "fields_data": { + "foo": { + "name": "foo", + "type": "int" + }, + "bar": { + "name": "bar", + "type": "int" + }, + "baz": { + "name": "baz", + "type": "float" + } + }, + "fields": { + "imports": "", + "fields": [ + "foo: int", + "bar: int", + "baz: float", + ] + }, + "generated": trim(""" + class Test: + foo: int + bar: int + baz: float + """) + }, + "complex": { + "model": ("Test", { + "foo": int, + "baz": DOptional(DList(DList(str))), + "bar": IntString + }), + "fields_data": { + "foo": { + "name": "foo", + "type": "int" + }, + "baz": { + "name": "baz", + "type": "Optional[List[List[str]]]" + }, + "bar": { + "name": "bar", + "type": "IntString" + } + }, + "fields": { + "imports": "from rest_client_gen.dynamic_typing.string_serializable import IntString\n" + "from typing import List, Optional", + "fields": [ + "foo: int", + "bar: IntString", + "baz: Optional[List[List[str]]]", + ] + }, + "generated": trim(""" + from rest_client_gen.dynamic_typing.string_serializable import IntString + from typing import List, Optional + + + class Test: + foo: int + bar: IntString + baz: Optional[List[List[str]]] + """) + } +} + +test_data_unzip = { + test: [ + pytest.param( + model_factory(*data["model"]), + data[test], + id=id + ) + for id, data in test_data.items() + if test in data + ] + for test in ("fields_data", "fields", "generated") +} + + +@pytest.mark.parametrize("value,expected", test_data_unzip["fields_data"]) +def test_fields_data(value: ModelMeta, expected: Dict[str, dict]): + gen = GenericModelCodeGenerator(value) + required, optional = sort_fields(value) + for is_optional, fields in enumerate((required, optional)): + for field in fields: + field_imports, data = gen.field_data(field, value.type[field], bool(is_optional)) + assert data == expected[field] + + +@pytest.mark.parametrize("value,expected", test_data_unzip["fields"]) +def test_fields(value: ModelMeta, expected: dict): + expected_imports: str = expected["imports"] + expected_fields: List[str] = expected["fields"] + gen = GenericModelCodeGenerator(value) + imports, fields = gen.fields + imports = compile_imports(imports) + assert imports == expected_imports + assert fields == expected_fields + + +@pytest.mark.parametrize("value,expected", test_data_unzip["generated"]) +def test_generated(value: ModelMeta, expected: str): + generated = generate_code([{"model": value, "nested": []}], GenericModelCodeGenerator) + assert generated.rstrip() == expected, generated diff --git a/test/test_code_generation/test_models_composition.py b/test/test_code_generation/test_models_composition.py index 1c47695..2995443 100644 --- a/test/test_code_generation/test_models_composition.py +++ b/test/test_code_generation/test_models_composition.py @@ -195,30 +195,31 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ], id="root_order" ), - pytest.param( - [ - ("Root", { - "model_a": { - "field_a": { - "field": float - } - }, - "model_b": { - "field_b": { - "field": float - } - } - }), - ], - [ - ("Root", [ - ("FieldA_FieldB", []), - ("ModelA", []), - ("ModelB", []), - ]) - ], - id="generic_in_nested_models" - ), + # Disable until rest_client_gen/models/__init__.py:86 will be fixed + # pytest.param( + # [ + # ("Root", { + # "model_a": { + # "field_a": { + # "field": float + # } + # }, + # "model_b": { + # "field_b": { + # "field": float + # } + # } + # }), + # ], + # [ + # ("Root", [ + # ("FieldA_FieldB", []), + # ("ModelA", []), + # ("ModelB", []), + # ]) + # ], + # id="generic_in_nested_models" + # ), pytest.param( [ ("RootItem", { @@ -238,36 +239,37 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ], id="merge_with_root_model" ), - pytest.param( - [ - ("Root", { - "model_a": { - "field_a": { - "field": { - "nested_field": float - } - } - }, - "model_b": { - "field_b": { - "field": { - "nested_field": float - } - } - } - }), - ], - [ - ("Root", [ - ("FieldA_FieldB", [ - ("Field", []) - ]), - ("ModelA", []), - ("ModelB", []), - ]) - ], - id="generic_in_nested_models_with_nested_model" - ), + # Disable until rest_client_gen/models/__init__.py:86 will be fixed + # pytest.param( + # [ + # ("Root", { + # "model_a": { + # "field_a": { + # "field": { + # "nested_field": float + # } + # } + # }, + # "model_b": { + # "field_b": { + # "field": { + # "nested_field": float + # } + # } + # } + # }), + # ], + # [ + # ("Root", [ + # ("FieldA_FieldB", [ + # ("Field", []) + # ]), + # ("ModelA", []), + # ("ModelB", []), + # ]) + # ], + # id="generic_in_nested_models_with_nested_model" + # ), ] diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index 980367c..c03f3e5 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -1,11 +1,13 @@ """ Example uses Ergast Developer API (http://ergast.com/mrd/) """ + import inflection import requests from rest_client_gen.generator import MetadataGenerator from rest_client_gen.models import compose_models +from rest_client_gen.models.base import GenericModelCodeGenerator, generate_code from rest_client_gen.registry import ModelRegistry from rest_client_gen.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta @@ -34,7 +36,7 @@ def main(): drivers_data = drivers() dump_response("f1", "drivers", drivers_data) - drivers_data = ("drivers", drivers_data) + drivers_data = ("driver", drivers_data) driver_standings_data = driver_standings() dump_response("f1", "driver_standings", driver_standings_data) @@ -54,6 +56,9 @@ def main(): root = compose_models(reg.models_map) print('\n', json_format(root)) + print("=" * 20) + + print(generate_code(root, GenericModelCodeGenerator)) if __name__ == '__main__': diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index 4128d91..e458464 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -5,6 +5,7 @@ from rest_client_gen.generator import MetadataGenerator from rest_client_gen.models import compose_models +from rest_client_gen.models.base import GenericModelCodeGenerator, generate_code from rest_client_gen.registry import ModelRegistry from rest_client_gen.utils import json_format from testing_tools.pprint_meta_data import pretty_format_meta @@ -24,7 +25,7 @@ def main(): gen = MetadataGenerator() reg = ModelRegistry() fields = gen.generate(*tabs) - reg.process_meta_data(fields) + reg.process_meta_data(fields, model_name="Tab") reg.merge_models(generator=gen) reg.generate_names() @@ -33,6 +34,9 @@ def main(): root = compose_models(reg.models_map) print('\n', json_format(root)) + print("=" * 20) + + print(generate_code(root, GenericModelCodeGenerator)) if __name__ == '__main__':