From 34b6da570723869c874c2ac752feb544a90ec12b Mon Sep 17 00:00:00 2001 From: bogdandm Date: Fri, 28 Sep 2018 22:09:21 +0300 Subject: [PATCH 1/5] Detect some bottleneck -> Add TODO items to fix later --- rest_client_gen/dynamic_typing/complex.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rest_client_gen/dynamic_typing/complex.py b/rest_client_gen/dynamic_typing/complex.py index 6e578e8..97710bf 100644 --- a/rest_client_gen/dynamic_typing/complex.py +++ b/rest_client_gen/dynamic_typing/complex.py @@ -21,7 +21,8 @@ def __iter__(self) -> Iterable['MetaData']: yield self.type def __eq__(self, other): - return isinstance(other, self.__class__) and self.type == other.type + # TODO: Try to cache this method (too many calls) + return other.__class__ is self.__class__ and self.type == other.type def replace(self, t: 'MetaData', **kwargs) -> 'SingleType': self.type = t @@ -45,6 +46,7 @@ def types(self, value): @property def sorted(self): + # TODO: Split into to methods and profile them """ Getter of cached sorted types list """ @@ -72,7 +74,8 @@ def __iter__(self) -> Iterable['MetaData']: yield from self.types def __eq__(self, other): - return isinstance(other, self.__class__) and self.sorted == other.sorted + # TODO: Try to cache this method (too many calls) + return other.__class__ is self.__class__ and self.sorted == other.sorted def __len__(self): return len(self.types) @@ -118,6 +121,7 @@ class DUnion(ComplexType): def __init__(self, *types: Union[type, BaseType, dict]): unique_types = [] + # TODO: Rewrite it to hash table for t in types: if isinstance(t, DUnion): # Merging nested DUnions From 3ee83134bd24cf1789b22e1268151d89730f7dd2 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Tue, 2 Oct 2018 15:12:39 +0300 Subject: [PATCH 2/5] Minor changes --- rest_client_gen/dynamic_typing/complex.py | 3 --- testing_tools/real_apis/pathofexile.py | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/rest_client_gen/dynamic_typing/complex.py b/rest_client_gen/dynamic_typing/complex.py index 97710bf..afcf2d1 100644 --- a/rest_client_gen/dynamic_typing/complex.py +++ b/rest_client_gen/dynamic_typing/complex.py @@ -21,7 +21,6 @@ def __iter__(self) -> Iterable['MetaData']: yield self.type def __eq__(self, other): - # TODO: Try to cache this method (too many calls) return other.__class__ is self.__class__ and self.type == other.type def replace(self, t: 'MetaData', **kwargs) -> 'SingleType': @@ -46,7 +45,6 @@ def types(self, value): @property def sorted(self): - # TODO: Split into to methods and profile them """ Getter of cached sorted types list """ @@ -74,7 +72,6 @@ def __iter__(self) -> Iterable['MetaData']: yield from self.types def __eq__(self, other): - # TODO: Try to cache this method (too many calls) return other.__class__ is self.__class__ and self.sorted == other.sorted def __len__(self): diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index 614a9d9..1c1e460 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -1,6 +1,8 @@ """ Path of Exile API http://www.pathofexile.com/developer/docs/api-resource-public-stash-tabs """ +from datetime import datetime + import requests from rest_client_gen.generator import MetadataGenerator @@ -23,6 +25,7 @@ def main(): tabs = tabs['stashes'] print(f"Start model generation (data len = {len(tabs)})") + start_t = datetime.now() gen = MetadataGenerator() reg = ModelRegistry() fields = gen.generate(*tabs) @@ -38,6 +41,7 @@ def main(): print("=" * 20) print(generate_code(structure, AttrsModelCodeGenerator)) + print(f"{(datetime.now() - start_t).total_seconds():.4f} seconds") if __name__ == '__main__': From 257eac19409c485677ae6506ecbcb03e95fccb9d Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 3 Oct 2018 13:44:22 +0300 Subject: [PATCH 3/5] Hash table --- rest_client_gen/dynamic_typing/base.py | 12 ++++++ rest_client_gen/dynamic_typing/complex.py | 43 ++++++++++++++----- rest_client_gen/dynamic_typing/models_meta.py | 3 ++ testing_tools/pprint_meta_data.py | 4 +- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/rest_client_gen/dynamic_typing/base.py b/rest_client_gen/dynamic_typing/base.py index b26d3f3..802bd0f 100644 --- a/rest_client_gen/dynamic_typing/base.py +++ b/rest_client_gen/dynamic_typing/base.py @@ -28,6 +28,15 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: """ raise NotImplementedError() + def to_hash_string(self) -> str: + """ + Return unique string that can be used to generate hash of type instance. + + :return: hash string + """ + # NOTE: Do not override __hash__ function as BaseType instances could mutate + raise NotImplementedError() + class UnknownType(BaseType): __slots__ = [] @@ -44,6 +53,9 @@ def replace(self, t: 'MetaData', **kwargs) -> 'UnknownType': def to_typing_code(self) -> Tuple[ImportPathList, str]: return ([('typing', 'Any')], 'Any') + def to_hash_string(self) -> str: + return "Unknown" + Unknown = UnknownType() NoneType = type(None) diff --git a/rest_client_gen/dynamic_typing/complex.py b/rest_client_gen/dynamic_typing/complex.py index afcf2d1..ec78196 100644 --- a/rest_client_gen/dynamic_typing/complex.py +++ b/rest_client_gen/dynamic_typing/complex.py @@ -1,3 +1,4 @@ +from inspect import isclass from itertools import chain from typing import Iterable, List, Tuple, Union @@ -5,6 +6,15 @@ from .typing import metadata_to_typing +def get_hash_string(t: MetaData): + if isinstance(t, dict): + return str(hash(tuple((k, get_hash_string(v)) for k, v in t.items()))) + elif isclass(t): + return str(t) + elif isinstance(t, BaseType): + return t.to_hash_string() + + class SingleType(BaseType): __slots__ = ["type"] @@ -12,21 +22,24 @@ def __init__(self, t: MetaData): self.type = t def __str__(self): - return f"{self.__class__.__name__}[{self.type}]" + return f"{type(self).__name__}[{self.type}]" def __repr__(self): - return f"<{self.__class__.__name__} [{self.type}]>" + return f"<{type(self).__name__} [{self.type}]>" def __iter__(self) -> Iterable['MetaData']: yield self.type def __eq__(self, other): - return other.__class__ is self.__class__ and self.type == other.type + return type(other) is type(self) and self.type == other.type def replace(self, t: 'MetaData', **kwargs) -> 'SingleType': self.type = t return self + def to_hash_string(self) -> str: + return f"{type(self).__name__}/{get_hash_string(self.type)}" + class ComplexType(BaseType): __slots__ = ["_types"] @@ -62,17 +75,17 @@ def _sort_key(self, item): def __str__(self): items = ', '.join(map(str, self.types)) - return f"{self.__class__.__name__}[{items}]" + return f"{type(self).__name__}[{items}]" def __repr__(self): items = ', '.join(map(str, self.types)) - return f"<{self.__class__.__name__} [{items}]>" + return f"<{type(self).__name__} [{items}]>" def __iter__(self) -> Iterable['MetaData']: yield from self.types def __eq__(self, other): - return other.__class__ is self.__class__ and self.sorted == other.sorted + return type(other) is type(self) and self.sorted == other.sorted def __len__(self): return len(self.types) @@ -97,6 +110,9 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: f"[{nested}]" ) + def to_hash_string(self) -> str: + return type(self).__name__ + "/" + ",".join(map(get_hash_string, self.types)) + class DOptional(SingleType): """ @@ -117,17 +133,22 @@ class DUnion(ComplexType): """ def __init__(self, *types: Union[type, BaseType, dict]): + hashes = set() unique_types = [] - # TODO: Rewrite it to hash table + # Ensure that types in union are unique for t in types: if isinstance(t, DUnion): # Merging nested DUnions for t2 in list(t._extract_nested_types()): - if t2 not in unique_types: + h = get_hash_string(t2) + if h not in hashes: unique_types.append(t2) - elif t not in unique_types: - # Ensure that types in union are unique - unique_types.append(t) + hashes.add(h) + else: + h = get_hash_string(t) + if h not in hashes: + hashes.add(h) + unique_types.append(t) super().__init__(*unique_types) def _extract_nested_types(self): diff --git a/rest_client_gen/dynamic_typing/models_meta.py b/rest_client_gen/dynamic_typing/models_meta.py index 2acf19f..b7c932a 100644 --- a/rest_client_gen/dynamic_typing/models_meta.py +++ b/rest_client_gen/dynamic_typing/models_meta.py @@ -129,6 +129,9 @@ def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr': def to_typing_code(self) -> Tuple[ImportPathList, str]: return AbsoluteModelRef(self.type).to_typing_code() + def to_hash_string(self) -> str: + return f"{type(self).__name__}_#{self.type.index}" + ContextInjectionType = Dict[ModelMeta, Union[ModelMeta, str]] diff --git a/testing_tools/pprint_meta_data.py b/testing_tools/pprint_meta_data.py index e2baf2f..ca09e9e 100644 --- a/testing_tools/pprint_meta_data.py +++ b/testing_tools/pprint_meta_data.py @@ -37,11 +37,11 @@ def _pprint_gen(value, key=None, lvl=0, empty_line=True, ignore_ptr=False): yield " " elif isinstance(value, SingleType): - yield f"{value.__class__.__name__}:" + yield f"{type(value).__name__}:" yield from _pprint_gen(value.type, lvl=lvl, empty_line=False, ignore_ptr=ignore_ptr) elif isinstance(value, ComplexType): - yield f"{value.__class__.__name__}:" + yield f"{type(value).__name__}:" for t in value.types: yield from _pprint_gen(t, lvl=lvl + 1, ignore_ptr=ignore_ptr) From 3a845c388d06103678601991fbdae4d59205f0f9 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 3 Oct 2018 13:52:24 +0300 Subject: [PATCH 4/5] Cache hash strings --- rest_client_gen/dynamic_typing/complex.py | 25 +++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/rest_client_gen/dynamic_typing/complex.py b/rest_client_gen/dynamic_typing/complex.py index ec78196..904d27d 100644 --- a/rest_client_gen/dynamic_typing/complex.py +++ b/rest_client_gen/dynamic_typing/complex.py @@ -16,10 +16,20 @@ def get_hash_string(t: MetaData): class SingleType(BaseType): - __slots__ = ["type"] + __slots__ = ["_type"] def __init__(self, t: MetaData): - self.type = t + self._type = t + self._hash = None + + @property + def type(self): + return self._type + + @type.setter + def type(self, t: MetaData): + self._type = t + self._hash = None def __str__(self): return f"{type(self).__name__}[{self.type}]" @@ -38,7 +48,9 @@ def replace(self, t: 'MetaData', **kwargs) -> 'SingleType': return self def to_hash_string(self) -> str: - return f"{type(self).__name__}/{get_hash_string(self.type)}" + if not self._hash: + self._hash = f"{type(self).__name__}/{get_hash_string(self.type)}" + return self._hash class ComplexType(BaseType): @@ -46,6 +58,8 @@ class ComplexType(BaseType): def __init__(self, *types: MetaData): self._types = list(types) + self._sorted = None + self._hash = None @property def types(self): @@ -55,6 +69,7 @@ def types(self): def types(self, value): self._types = value self._sorted = None + self._hash = None @property def sorted(self): @@ -111,7 +126,9 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: ) def to_hash_string(self) -> str: - return type(self).__name__ + "/" + ",".join(map(get_hash_string, self.types)) + if not self._hash: + self._hash = type(self).__name__ + "/" + ",".join(map(get_hash_string, self.types)) + return self._hash class DOptional(SingleType): From 41c6ff12f1f812b45bf2a63bbc2fb0b5746e6cd6 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Wed, 3 Oct 2018 14:25:24 +0300 Subject: [PATCH 5/5] Test to_hash_string; Rearrange and refactor code; --- rest_client_gen/dynamic_typing/__init__.py | 2 +- rest_client_gen/dynamic_typing/base.py | 24 +++++++++++++++- rest_client_gen/dynamic_typing/complex.py | 28 +++++-------------- rest_client_gen/dynamic_typing/models_meta.py | 8 +++++- .../test_dynamic_typing.py | 22 +++++++++++++-- 5 files changed, 57 insertions(+), 27 deletions(-) diff --git a/rest_client_gen/dynamic_typing/__init__.py b/rest_client_gen/dynamic_typing/__init__.py index 35cd6e7..8ef6129 100644 --- a/rest_client_gen/dynamic_typing/__init__.py +++ b/rest_client_gen/dynamic_typing/__init__.py @@ -1,5 +1,5 @@ from .base import ( - BaseType, ImportPathList, MetaData, NoneType, Unknown, UnknownType + BaseType, ImportPathList, MetaData, NoneType, Unknown, UnknownType, get_hash_string ) from .complex import ComplexType, DList, DOptional, DTuple, DUnion, SingleType from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr diff --git a/rest_client_gen/dynamic_typing/base.py b/rest_client_gen/dynamic_typing/base.py index 802bd0f..290d544 100644 --- a/rest_client_gen/dynamic_typing/base.py +++ b/rest_client_gen/dynamic_typing/base.py @@ -1,3 +1,4 @@ +from inspect import isclass from typing import Iterable, List, Tuple, Union ImportPathList = List[Tuple[str, Union[Iterable[str], str, None]]] @@ -31,10 +32,22 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: def to_hash_string(self) -> str: """ Return unique string that can be used to generate hash of type instance. + Caches hash value by default. If subclass can mutate (by default it always can) + then it should define setters to safely invalidate cached value. :return: hash string """ - # NOTE: Do not override __hash__ function as BaseType instances could mutate + # NOTE: Do not override __hash__ function because BaseType instances isn't immutable + if not self._hash: + self._hash = self._to_hash_string() + return self._hash + + def _to_hash_string(self) -> str: + """ + Hash getter method to override + + :return: + """ raise NotImplementedError() @@ -60,3 +73,12 @@ def to_hash_string(self) -> str: Unknown = UnknownType() NoneType = type(None) MetaData = Union[type, dict, BaseType] + + +def get_hash_string(t: MetaData): + if isinstance(t, dict): + return str(hash(tuple((k, get_hash_string(v)) for k, v in t.items()))) + elif isclass(t): + return str(t) + elif isinstance(t, BaseType): + return t.to_hash_string() diff --git a/rest_client_gen/dynamic_typing/complex.py b/rest_client_gen/dynamic_typing/complex.py index 904d27d..5aabe0f 100644 --- a/rest_client_gen/dynamic_typing/complex.py +++ b/rest_client_gen/dynamic_typing/complex.py @@ -1,22 +1,12 @@ -from inspect import isclass from itertools import chain from typing import Iterable, List, Tuple, Union -from .base import BaseType, ImportPathList, MetaData +from .base import BaseType, ImportPathList, MetaData, get_hash_string from .typing import metadata_to_typing -def get_hash_string(t: MetaData): - if isinstance(t, dict): - return str(hash(tuple((k, get_hash_string(v)) for k, v in t.items()))) - elif isclass(t): - return str(t) - elif isinstance(t, BaseType): - return t.to_hash_string() - - class SingleType(BaseType): - __slots__ = ["_type"] + __slots__ = ["_type", "_hash"] def __init__(self, t: MetaData): self._type = t @@ -47,14 +37,12 @@ def replace(self, t: 'MetaData', **kwargs) -> 'SingleType': self.type = t return self - def to_hash_string(self) -> str: - if not self._hash: - self._hash = f"{type(self).__name__}/{get_hash_string(self.type)}" - return self._hash + def _to_hash_string(self) -> str: + return f"{type(self).__name__}/{get_hash_string(self.type)}" class ComplexType(BaseType): - __slots__ = ["_types"] + __slots__ = ["_types", "_sorted", "_hash"] def __init__(self, *types: MetaData): self._types = list(types) @@ -125,10 +113,8 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]: f"[{nested}]" ) - def to_hash_string(self) -> str: - if not self._hash: - self._hash = type(self).__name__ + "/" + ",".join(map(get_hash_string, self.types)) - return self._hash + def _to_hash_string(self) -> str: + return type(self).__name__ + "/" + ",".join(map(get_hash_string, self.types)) class DOptional(SingleType): diff --git a/rest_client_gen/dynamic_typing/models_meta.py b/rest_client_gen/dynamic_typing/models_meta.py index b7c932a..5fd74fa 100644 --- a/rest_client_gen/dynamic_typing/models_meta.py +++ b/rest_client_gen/dynamic_typing/models_meta.py @@ -121,6 +121,7 @@ def replace(self, t: ModelMeta, **kwargs) -> 'ModelPtr': return self def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr': + self._hash = None self.parent.remove_child_ref(self) self.parent = t self.parent.add_child_ref(self) @@ -129,7 +130,7 @@ def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr': def to_typing_code(self) -> Tuple[ImportPathList, str]: return AbsoluteModelRef(self.type).to_typing_code() - def to_hash_string(self) -> str: + def _to_hash_string(self) -> str: return f"{type(self).__name__}_#{self.type.index}" @@ -153,6 +154,11 @@ class NestedModel: This information is only available at the models code generation stage while typing code is generated from raw metadata and passing this absolute path as argument to each ModelPtr would be annoying. + + Usage: + + with AbsoluteModelRef.inject({TestModel: "ParentModelName"}): + """ class Context: diff --git a/test/test_dynamic_typing/test_dynamic_typing.py b/test/test_dynamic_typing/test_dynamic_typing.py index 65e6963..b205d9a 100644 --- a/test/test_dynamic_typing/test_dynamic_typing.py +++ b/test/test_dynamic_typing/test_dynamic_typing.py @@ -1,9 +1,11 @@ +from builtins import complex + import pytest -from rest_client_gen.dynamic_typing import DUnion +from rest_client_gen.dynamic_typing import DUnion, get_hash_string # *args | MetaData -test_dunion= [ +test_dunion = [ pytest.param( [int, int], DUnion(int), @@ -21,7 +23,21 @@ ) ] + @pytest.mark.parametrize("value,expected", test_dunion) def test_dunion_creation(value, expected): result = DUnion(*value) - assert result == expected \ No newline at end of file + assert result == expected + + +def test_hash_string(): + a = {'a': int} + b = {'b': int} + c = {'a': float} + assert len(set(map(get_hash_string, (a, b, c)))) == 3 + + union = DUnion(str, float) + h1 = union.to_hash_string() + union.replace(complex, index=0) + h2 = union.to_hash_string() + assert h1 != h2, f"{h1}, {h2}"