diff --git a/.coveragerc b/.coveragerc index 8482a35..487d080 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,7 @@ # .coveragerc to control coverage.py [run] +omit = + rest_client_gen/lazy.py [report] exclude_lines = diff --git a/rest_client_gen/dynamic_typing/__init__.py b/rest_client_gen/dynamic_typing/__init__.py index c21ebd9..35cd6e7 100644 --- a/rest_client_gen/dynamic_typing/__init__.py +++ b/rest_client_gen/dynamic_typing/__init__.py @@ -2,7 +2,7 @@ BaseType, ImportPathList, MetaData, NoneType, Unknown, UnknownType ) from .complex import ComplexType, DList, DOptional, DTuple, DUnion, SingleType -from .models_meta import ModelMeta, ModelPtr +from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr from .string_serializable import ( BooleanString, FloatString, IntString, StringSerializable, StringSerializableRegistry, registry ) diff --git a/rest_client_gen/dynamic_typing/models_meta.py b/rest_client_gen/dynamic_typing/models_meta.py index 5239836..2acf19f 100644 --- a/rest_client_gen/dynamic_typing/models_meta.py +++ b/rest_client_gen/dynamic_typing/models_meta.py @@ -1,4 +1,5 @@ -from typing import List, Optional, Set, Tuple +import threading +from typing import Dict, List, Optional, Set, Tuple, Union import inflection @@ -126,5 +127,62 @@ def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr': return self def to_typing_code(self) -> Tuple[ImportPathList, str]: - imports, model = self.type.to_typing_code() - return imports, f"'{model}'" + return AbsoluteModelRef(self.type).to_typing_code() + + +ContextInjectionType = Dict[ModelMeta, Union[ModelMeta, str]] + + +class AbsoluteModelRef: + """ + Model forward absolute references. Using ContextManager to inject real models paths into typing code. + Forward reference is the typing string like ``List['MyModel']``. + If the model is defined as child model and is used by another nested model + than the reference to this model should be an absolute path: + + class Model: + class GenericChildModel: + ... + + class NestedModel: + data: 'Model.GenericChildModel' # <--- this + + This information is only available at the models code generation stage + while typing code is generated from raw metadata and passing this absolute path as argument + to each ModelPtr would be annoying. + """ + + class Context: + data = threading.local() + data.context: ContextInjectionType = None + + def __init__(self, patches: ContextInjectionType): + self.context: ContextInjectionType = patches + self._old: ContextInjectionType = None + + def __enter__(self): + self._old = self.data.context + self.data.context = self.context + + def __exit__(self, exc_type, exc_val, exc_tb): + self.data.context = self._old + + @classmethod + def inject(cls, patches: ContextInjectionType): + context = cls.Context(patches) + return context + + def __init__(self, model: ModelMeta): + self.model = model + + def to_typing_code(self) -> Tuple[ImportPathList, str]: + context_data = self.Context.data.context + if context_data: + model_path = context_data.get(self.model, "") + if isinstance(model_path, ModelMeta): + model_path = model_path.name + else: + model_path = "" + imports, model = self.model.to_typing_code() + s = ".".join(filter(None, (model_path, model))) + return imports, f"'{s}'" diff --git a/rest_client_gen/lazy.py b/rest_client_gen/lazy.py new file mode 100644 index 0000000..bad2410 --- /dev/null +++ b/rest_client_gen/lazy.py @@ -0,0 +1,192 @@ +""" +Fork of https://github.com/django/django/blob/2.1/django/utils/functional.py +Note: keep_lazy function is modified and returns lazy value always +even for functions without arguments. + +Copyright (c) Django Software Foundation and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of Django nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" +from functools import total_ordering, wraps + + +class Promise: + """ + Base class for the proxy class created in the closure of the lazy function. + It's used to recognize promises in code. + """ + pass + + +def lazy(func, *resultclasses): + """ + Turn any callable into a lazy evaluated callable. result classes or types + is required -- at least one is needed so that the automatic forcing of + the lazy evaluation code is triggered. Results are not memoized; the + function is evaluated on every access. + """ + + @total_ordering + class __proxy__(Promise): + """ + Encapsulate a function call and act as a proxy for methods that are + called on the result of that function. The function is not evaluated + until one of the methods on the result is called. + """ + __prepared = False + + def __init__(self, args, kw): + self.__args = args + self.__kw = kw + if not self.__prepared: + self.__prepare_class__() + self.__prepared = True + + def __reduce__(self): + return ( + _lazy_proxy_unpickle, + (func, self.__args, self.__kw) + resultclasses + ) + + def __repr__(self): + return repr(self.__cast()) + + @classmethod + def __prepare_class__(cls): + for resultclass in resultclasses: + for type_ in resultclass.mro(): + for method_name in type_.__dict__: + # All __promise__ return the same wrapper method, they + # look up the correct implementation when called. + if hasattr(cls, method_name): + continue + meth = cls.__promise__(method_name) + setattr(cls, method_name, meth) + cls._delegate_bytes = bytes in resultclasses + cls._delegate_text = str in resultclasses + assert not (cls._delegate_bytes and cls._delegate_text), ( + "Cannot call lazy() with both bytes and text return types.") + if cls._delegate_text: + cls.__str__ = cls.__text_cast + elif cls._delegate_bytes: + cls.__bytes__ = cls.__bytes_cast + + @classmethod + def __promise__(cls, method_name): + # Builds a wrapper around some magic method + def __wrapper__(self, *args, **kw): + # Automatically triggers the evaluation of a lazy value and + # applies the given magic method of the result type. + res = func(*self.__args, **self.__kw) + return getattr(res, method_name)(*args, **kw) + + return __wrapper__ + + def __text_cast(self): + return func(*self.__args, **self.__kw) + + def __bytes_cast(self): + return bytes(func(*self.__args, **self.__kw)) + + def __bytes_cast_encoded(self): + return func(*self.__args, **self.__kw).encode() + + def __cast(self): + if self._delegate_bytes: + return self.__bytes_cast() + elif self._delegate_text: + return self.__text_cast() + else: + return func(*self.__args, **self.__kw) + + def __str__(self): + # object defines __str__(), so __prepare_class__() won't overload + # a __str__() method from the proxied class. + return str(self.__cast()) + + def __eq__(self, other): + if isinstance(other, Promise): + other = other.__cast() + return self.__cast() == other + + def __lt__(self, other): + if isinstance(other, Promise): + other = other.__cast() + return self.__cast() < other + + def __hash__(self): + return hash(self.__cast()) + + def __mod__(self, rhs): + if self._delegate_text: + return str(self) % rhs + return self.__cast() % rhs + + def __deepcopy__(self, memo): + # Instances of this class are effectively immutable. It's just a + # collection of functions. So we don't need to do anything + # complicated for copying. + memo[id(self)] = self + return self + + @wraps(func) + def __wrapper__(*args, **kw): + # Creates the proxy object, instead of the actual value. + return __proxy__(args, kw) + + return __wrapper__ + + +def _lazy_proxy_unpickle(func, args, kwargs, *resultclasses): + return lazy(func, *resultclasses)(*args, **kwargs) + + +def keep_lazy(*resultclasses): + """ + A decorator that allows a function to be called with one or more lazy arguments. + Return a __proxy__ object that will evaluate the function when needed. + """ + if not resultclasses: + raise TypeError("You must pass at least one argument to keep_lazy().") + + def decorator(func): + lazy_func = lazy(func, *resultclasses) + + @wraps(func) + def wrapper(*args, **kwargs): + return lazy_func(*args, **kwargs) + + return wrapper + + return decorator + + +def keep_lazy_text(func): + """ + A decorator for functions that accept lazy arguments and return text. + """ + return keep_lazy(str)(func) diff --git a/rest_client_gen/models/__init__.py b/rest_client_gen/models/__init__.py index 95c6bc9..50bd811 100644 --- a/rest_client_gen/models/__init__.py +++ b/rest_client_gen/models/__init__.py @@ -60,9 +60,14 @@ def extract_root(model: ModelMeta) -> Set[Index]: return roots -def compose_models(models_map: Dict[str, ModelMeta]) -> List[dict]: +ModelsStructureType = Tuple[List[dict], Dict[ModelMeta, ModelMeta]] + + +def compose_models(models_map: Dict[str, ModelMeta]) -> ModelsStructureType: """ Generate nested sorted models structure for internal usage. + + :return: List of root models data, Map(child model -> root model) for absolute ref generation """ root_models = ListEx() root_nested_ix = 0 @@ -70,9 +75,11 @@ def compose_models(models_map: Dict[str, ModelMeta]) -> List[dict]: key: { "model": model, "nested": ListEx(), - "roots": list(extract_root(model)) + "roots": list(extract_root(model)), # Indexes of root level models } for key, model in models_map.items() } + # TODO: Test path_injections + path_injections: Dict[ModelMeta, ModelMeta] = {} for key, model in models_map.items(): pointers = list(filter_pointers(model)) @@ -85,10 +92,8 @@ def compose_models(models_map: Dict[str, ModelMeta]) -> List[dict]: else: parents = {ptr.parent.index for ptr in pointers} struct = structure_hash_table[key] - # FIXME: "Model is using by single root model" case for the time being will be disabled - # until solution to make typing ref such as 'Parent.Child' will be found # Model is using by other models - if has_root_pointers or len(parents) > 1: # and len(struct["roots"]) > 1 + if has_root_pointers or len(parents) > 1 and len(struct["roots"]) > 1: # Model is using by different root models try: root_models.insert_before( @@ -98,17 +103,18 @@ def compose_models(models_map: Dict[str, ModelMeta]) -> List[dict]: except ValueError: root_models.insert(root_nested_ix, struct) root_nested_ix += 1 - # elif len(parents) > 1 and len(struct["roots"]) == 1: - # # Model is using by single root model - # parent = structure_hash_table[struct["roots"][0]] - # parent["nested"].insert(0, struct) + elif len(parents) > 1 and len(struct["roots"]) == 1: + # Model is using by single root model + parent = structure_hash_table[struct["roots"][0]] + parent["nested"].insert(0, struct) + path_injections[struct["model"]] = parent["model"] else: # Model is using by only one model parent = structure_hash_table[next(iter(parents))] struct = structure_hash_table[key] parent["nested"].append(struct) - return root_models + return root_models, path_injections def sort_fields(model_meta: ModelMeta) -> Tuple[List[str], List[str]]: diff --git a/rest_client_gen/models/base.py b/rest_client_gen/models/base.py index 02bca43..ea92c0d 100644 --- a/rest_client_gen/models/base.py +++ b/rest_client_gen/models/base.py @@ -2,8 +2,8 @@ from jinja2 import Template -from rest_client_gen.dynamic_typing import compile_imports -from rest_client_gen.models import INDENT, OBJECTS_DELIMITER +from rest_client_gen.dynamic_typing import AbsoluteModelRef, compile_imports +from rest_client_gen.models import INDENT, ModelsStructureType, OBJECTS_DELIMITER from . import indent, sort_fields from ..dynamic_typing import ImportPathList, MetaData, ModelMeta, metadata_to_typing @@ -49,7 +49,6 @@ class {{ name }}: def __init__(self, model: ModelMeta, **kwargs): self.model = model - def generate(self, nested_classes: List[str] = None) -> Tuple[ImportPathList, str]: """ :param nested_classes: list of strings that contains classes code @@ -138,7 +137,7 @@ def _generate_code( return imports, classes -def generate_code(structure: List[dict], class_generator: Type[GenericModelCodeGenerator], +def generate_code(structure: ModelsStructureType, class_generator: Type[GenericModelCodeGenerator], class_generator_kwargs: dict = None, objects_delimiter: str = OBJECTS_DELIMITER) -> str: """ Generate ready-to-use code @@ -149,7 +148,9 @@ def generate_code(structure: List[dict], class_generator: Type[GenericModelCodeG :param objects_delimiter: Delimiter between root level classes :return: Generated code """ - imports, classes = _generate_code(structure, class_generator, class_generator_kwargs or {}) + root, mapping = structure + with AbsoluteModelRef.inject(mapping): + imports, classes = _generate_code(root, class_generator, class_generator_kwargs or {}) if imports: imports_str = compile_imports(imports) + objects_delimiter else: diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py index e181c86..1c10fd3 100644 --- a/test/test_code_generation/test_models_code_generator.py +++ b/test/test_code_generation/test_models_code_generator.py @@ -2,7 +2,8 @@ import pytest -from rest_client_gen.dynamic_typing import DList, DOptional, IntString, ModelMeta, compile_imports +from rest_client_gen.dynamic_typing import (AbsoluteModelRef, DList, DOptional, IntString, ModelMeta, ModelPtr, + compile_imports) from rest_client_gen.models import indent, sort_fields from rest_client_gen.models.base import GenericModelCodeGenerator, generate_code @@ -172,5 +173,25 @@ def test_fields(value: ModelMeta, expected: dict): @pytest.mark.parametrize("value,expected", test_data_unzip["generated"]) def test_generated(value: ModelMeta, expected: str): - generated = generate_code([{"model": value, "nested": []}], GenericModelCodeGenerator) + generated = generate_code(([{"model": value, "nested": []}], {}), GenericModelCodeGenerator) assert generated.rstrip() == expected, generated + + +def test_absolute_model_ref(): + test_model = ModelMeta({"field": int}, "A") + test_model.name = "test_model" + test_ptr = ModelPtr(test_model) + assert test_ptr.to_typing_code()[1] == "'TestModel'" + with AbsoluteModelRef.inject({test_model: "Parent"}): + assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + assert test_ptr.to_typing_code()[1] == "'TestModel'" + with AbsoluteModelRef.inject({test_model: "Parent"}): + assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + with AbsoluteModelRef.inject({test_model: "AnotherParent"}): + assert test_ptr.to_typing_code()[1] == "'AnotherParent.TestModel'" + assert test_ptr.to_typing_code()[1] == "'Parent.TestModel'" + + wrapper = DList(DList(test_ptr)) + assert wrapper.to_typing_code()[1] == "List[List['TestModel']]" + with AbsoluteModelRef.inject({test_model: test_model}): + assert wrapper.to_typing_code()[1] == "List[List['TestModel.TestModel']]" diff --git a/test/test_code_generation/test_models_composition.py b/test/test_code_generation/test_models_composition.py index 2995443..3bc86ff 100644 --- a/test/test_code_generation/test_models_composition.py +++ b/test/test_code_generation/test_models_composition.py @@ -2,6 +2,7 @@ import pytest +from rest_client_gen.dynamic_typing import ModelMeta from rest_client_gen.generator import MetadataGenerator from rest_client_gen.models import ListEx, compose_models, extract_root from rest_client_gen.registry import ModelRegistry @@ -127,6 +128,7 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ("Item", []) ]) ], + {}, id="basic_test" ), pytest.param( @@ -148,6 +150,7 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ("RootA", []), ("RootB", []) ], + {}, id="global_nested_model" ), pytest.param( @@ -168,6 +171,7 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ("Item", []) ]) ], + {}, id="roots_merge" ), pytest.param( @@ -193,33 +197,34 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ("RootA", []), ("RootB", []) ], + {}, id="root_order" ), - # Disable until rest_client_gen/models/__init__.py:86 will be fixed - # pytest.param( - # [ - # ("Root", { - # "model_a": { - # "field_a": { - # "field": float - # } - # }, - # "model_b": { - # "field_b": { - # "field": float - # } - # } - # }), - # ], - # [ - # ("Root", [ - # ("FieldA_FieldB", []), - # ("ModelA", []), - # ("ModelB", []), - # ]) - # ], - # id="generic_in_nested_models" - # ), + pytest.param( + [ + ("Root", { + "model_a": { + "field_a": { + "field": float + } + }, + "model_b": { + "field_b": { + "field": float + } + } + }), + ], + [ + ("Root", [ + ("FieldA_FieldB", []), + ("ModelA", []), + ("ModelB", []), + ]) + ], + {'FieldA_FieldB': 'Root'}, + id="generic_in_nested_models" + ), pytest.param( [ ("RootItem", { @@ -237,52 +242,55 @@ def test_extract_root(models_generator: MetadataGenerator, models_registry: Mode ("RootItem", []), ("RootA_RootB", []) ], + {}, id="merge_with_root_model" ), - # Disable until rest_client_gen/models/__init__.py:86 will be fixed - # pytest.param( - # [ - # ("Root", { - # "model_a": { - # "field_a": { - # "field": { - # "nested_field": float - # } - # } - # }, - # "model_b": { - # "field_b": { - # "field": { - # "nested_field": float - # } - # } - # } - # }), - # ], - # [ - # ("Root", [ - # ("FieldA_FieldB", [ - # ("Field", []) - # ]), - # ("ModelA", []), - # ("ModelB", []), - # ]) - # ], - # id="generic_in_nested_models_with_nested_model" - # ), + pytest.param( + [ + ("Root", { + "model_a": { + "field_a": { + "field": { + "nested_field": float + } + } + }, + "model_b": { + "field_b": { + "field": { + "nested_field": float + } + } + } + }), + ], + [ + ("Root", [ + ("FieldA_FieldB", [ + ("Field", []) + ]), + ("ModelA", []), + ("ModelB", []), + ]) + ], + {'FieldA_FieldB': 'Root'}, + id="generic_in_nested_models_with_nested_model" + ), ] -@pytest.mark.parametrize("value,expected", test_compose_models_data) -def test_compose_models(models_generator: MetadataGenerator, models_registry: ModelRegistry, - value: List[Tuple[str, dict]], expected: List[Tuple[str, list]]): +@pytest.mark.parametrize("value,expected,expected_mapping", test_compose_models_data) +def test_compose_models( + models_generator: MetadataGenerator, models_registry: ModelRegistry, + value: List[Tuple[str, dict]], expected: List[Tuple[str, list]], expected_mapping: Dict[str, str] +): for model_name, metadata in value: models_registry.process_meta_data(metadata, model_name=model_name) models_registry.merge_models(models_generator) models_registry.generate_names() names_map = {model.index: model.name for model in models_registry.models} names_map.update({model.name: model.index for model in models_registry.models}) - root = compose_models(models_registry.models_map) + root, mapping = compose_models(models_registry.models_map) def check(nested_value: List[dict], nested_expected: List[Tuple[str, list]]): for model_dict, (model_name, nested) in zip(nested_value, nested_expected): @@ -291,3 +299,7 @@ def check(nested_value: List[dict], nested_expected: List[Tuple[str, list]]): check(model_dict["nested"], nested) check(root, expected) + + name = lambda model: model.name if isinstance(model, ModelMeta) else model + mapping = {name(model): name(parent) for model, parent in mapping.items()} + assert mapping == expected_mapping diff --git a/test/test_code_generation/test_typing.py b/test/test_code_generation/test_typing.py index 83fb52a..01399cb 100644 --- a/test/test_code_generation/test_typing.py +++ b/test/test_code_generation/test_typing.py @@ -8,7 +8,7 @@ @pytest.mark.xfail(strict=True, raises=ValueError) def test_metadata_to_typing_with_dict(): - metadata_to_typing({'a': 1}) + assert metadata_to_typing({'a': 1}) test_imports_compiler_data = [ diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index c03f3e5..14a6aae 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -54,11 +54,11 @@ def main(): print(pretty_format_meta(model)) print("=" * 20, end='') - root = compose_models(reg.models_map) - print('\n', json_format(root)) + structure = compose_models(reg.models_map) + print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) print("=" * 20) - print(generate_code(root, GenericModelCodeGenerator)) + print(generate_code(structure, GenericModelCodeGenerator)) if __name__ == '__main__': diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index e458464..d68e490 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -32,11 +32,11 @@ def main(): print(pretty_format_meta(next(iter(reg.models)))) print("\n" + "=" * 20, end='') - root = compose_models(reg.models_map) - print('\n', json_format(root)) + structure = compose_models(reg.models_map) + print('\n', json_format([structure[0], {str(a): str(b) for a, b in structure[1].items()}])) print("=" * 20) - print(generate_code(root, GenericModelCodeGenerator)) + print(generate_code(structure, GenericModelCodeGenerator)) if __name__ == '__main__':