diff --git a/.travis.yml b/.travis.yml index 8df369e..339185e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ language: python cache: pip install: + - python setup.py sdist - python setup.py install - pip install pytest pytest-cov requests coveralls codacy-coverage @@ -22,11 +23,23 @@ matrix: - python: 3.8-dev deploy: - provider: pypi - user: "bogdandm" - password: - secure: "bfIiVsBVMRAjsZYT/UtvNFiY5RU9+2b8QyveKXB3JhZCC/+S8nU/ZJkM9/3Q4V9sUVTVd+NR4Iq1OWkgFwYhJXK+eYlqWJDwfzcxTY5sXAcIxtmXGpNKJaqOF8hyCRdeusGFXR9O1RqzZVQ1ioLCj43eOkjvgFbOw2pIRtmG8Sip6oSJUmY9MSYHFzz4EoY6s+Jv9g30P7bSGY9bekOySFi7LmnX2IM+2T7AMuRvRyd4gHlLVy7pxZxj3S/jz9IiwkVq55HUl5ITWCZVrk6RnYiQhWn5D1AllrOilJGCNQ6HmKQWhDu/fNXEg24BFUTs87OymPKAFNRFniaH9PGu5v2yoN0BWN6+23c2Zc/G9QpQzqm+eVIpW5pkp25EiQw8Cz1pjDmLAVzSZuPXxJnpmkihLUAmSoktI4Zo6+QeLaBoqxds//aLqupMPhO3kzUZe1O5CrogPXTCwpHUGKzMxtjaaXdLo3z7EVT8kRDRtggpdE9KD1shDRUMBrakmuOFA+Tms+iKZSBrW5xhC2g/lFnZluZidj2ir8hyJ9lPMUmGxn/OkIQIBcMkEKCsDFC3wPD39MY/eDbkBvmK++Uhah9T0ljLOR+j1n2ZoJ+N3zD/UpucW5681dGA/sDpgjyDc9ESj14IHjTPLI7sw3IXaial4X2h0ZdfXf6NesLDraI=" - on: - tags: true - all_branches: false - skip_existing: true \ No newline at end of file + - provider: pypi + user: bogdandm + password: + secure: bfIiVsBVMRAjsZYT/UtvNFiY5RU9+2b8QyveKXB3JhZCC/+S8nU/ZJkM9/3Q4V9sUVTVd+NR4Iq1OWkgFwYhJXK+eYlqWJDwfzcxTY5sXAcIxtmXGpNKJaqOF8hyCRdeusGFXR9O1RqzZVQ1ioLCj43eOkjvgFbOw2pIRtmG8Sip6oSJUmY9MSYHFzz4EoY6s+Jv9g30P7bSGY9bekOySFi7LmnX2IM+2T7AMuRvRyd4gHlLVy7pxZxj3S/jz9IiwkVq55HUl5ITWCZVrk6RnYiQhWn5D1AllrOilJGCNQ6HmKQWhDu/fNXEg24BFUTs87OymPKAFNRFniaH9PGu5v2yoN0BWN6+23c2Zc/G9QpQzqm+eVIpW5pkp25EiQw8Cz1pjDmLAVzSZuPXxJnpmkihLUAmSoktI4Zo6+QeLaBoqxds//aLqupMPhO3kzUZe1O5CrogPXTCwpHUGKzMxtjaaXdLo3z7EVT8kRDRtggpdE9KD1shDRUMBrakmuOFA+Tms+iKZSBrW5xhC2g/lFnZluZidj2ir8hyJ9lPMUmGxn/OkIQIBcMkEKCsDFC3wPD39MY/eDbkBvmK++Uhah9T0ljLOR+j1n2ZoJ+N3zD/UpucW5681dGA/sDpgjyDc9ESj14IHjTPLI7sw3IXaial4X2h0ZdfXf6NesLDraI= + on: + tags: true + all_branches: false + + - provider: releases + api_key: + secure: AalzD76UPcpYdW0EqIbIi0prnxj1rsyRI6S4EBx91XGFY+BRJV+wJ99MiCE36A4q+cerDCxLmosiAQdzYkJ6/1ageezytJYD0FiC8GxvXtWoutySi0Ka1UTxvIQY22680lh5Xj4vki8tmQAMiLc9gTtwkk+e1J6bPgbQVNQtd5HD3697fdbeBrK70kzHUt93nW1x3N6Ers+WuLtZzV8ft1QzKGQYqgEGigV3rfCM6BThhOXq8B5DnzwwRHg/4TbHvtjyqKchPSzj9+zxGrInJ4dMAut8sGbvFRypDS8Y/I0ODveEBbbmi1SQkoRTH7OuP76T1qAPurmYR2nhq1owxQ4OLjDLgtSiFyzLTEXTjFAOENa8vlZcwC75iCyun/jj2lBVFvW/1rAIiogHxwLsKELbRZQzfY8pTJK5D5KPLNMtNDYecrxI0Bg0z7RPRjk5cpT3gAIyRHuVqCeXZUh+bP87Ev632jtK0thilF67vJNfSamHxLJZH6hEQNKOMjztcOdRDlY+vNbIVwl/RScrhSi+dQCH6FAINXNL9a6TENkwxYjgHYhB9xxNQXq6p+miSID7V9ptBTDVsNIGcv8xLzaChVWExzkFR/BflObjccKmiN9hTlzHSEUaRBpw3EX/L9YNKtPHvPSo/1KxZPt+AB/z0OgvWqSdNaVVtopooQg= + file_glob: true + file: dist/* + skip_cleanup: true + overwrite: true + draft: true + on: + repo: bogdandm/json2python-models + tags: true + all_branches: false \ No newline at end of file diff --git a/json_to_models/__init__.py b/json_to_models/__init__.py index ae763aa..6edeefc 100644 --- a/json_to_models/__init__.py +++ b/json_to_models/__init__.py @@ -1,4 +1,4 @@ from pkg_resources import parse_version -__version__ = "0.1a1" +__version__ = "0.1b1" VERSION = parse_version(__version__) diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 812952d..04b78ce 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -3,6 +3,7 @@ import itertools import json import os.path +import re from collections import defaultdict from datetime import datetime from pathlib import Path @@ -79,26 +80,21 @@ def parse_args(self, args: List[str] = None): framework = namespace.framework code_generator = namespace.code_generator code_generator_kwargs_raw: List[str] = namespace.code_generator_kwargs - code_generator_kwargs = {} - if code_generator_kwargs_raw: - for item in code_generator_kwargs_raw: - if item[0] == '"': - item = item[1:] - if item[-1] == '"': - item = item[:-1] - name, value = item.split("=", 1) - code_generator_kwargs[name] = value + dict_keys_regex: List[str] = namespace.dict_keys_regex + dict_keys_fields: List[str] = namespace.dict_keys_fields self.validate(models, models_lists, merge_policy, framework, code_generator) self.setup_models_data(models, models_lists) - self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs) + self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw, + dict_keys_regex, dict_keys_fields) def run(self): if self.enable_datetime: register_datetime_classes() - - # TODO: Inject dict_keys_regex and dict_keys_fields - generator = MetadataGenerator() + generator = MetadataGenerator( + dict_keys_regex=self.dict_keys_regex, + dict_keys_fields=self.dict_keys_fields + ) registry = ModelRegistry(*self.merge_policy) for name, data in self.models_data.items(): meta = generator.generate(*data) @@ -152,7 +148,8 @@ def setup_models_data(self, models: Iterable[Tuple[str, Iterable[Path]]], } def set_args(self, merge_policy: List[Union[List[str], str]], - structure: str, framework: str, code_generator: str, code_generator_kwargs: dict): + structure: str, framework: str, code_generator: str, code_generator_kwargs_raw: List[str], + dict_keys_regex: List[str], dict_keys_fields: List[str]): """ Convert CLI args to python representation and set them to appropriate object attributes """ @@ -175,8 +172,18 @@ def set_args(self, merge_policy: List[Union[List[str], str]], m = importlib.import_module(module) self.model_generator = getattr(m, cls) - if code_generator_kwargs: - self.model_generator_kwargs = code_generator_kwargs + self.model_generator_kwargs = {} + if code_generator_kwargs_raw: + for item in code_generator_kwargs_raw: + if item[0] == '"': + item = item[1:] + if item[-1] == '"': + item = item[:-1] + name, value = item.split("=", 1) + self.model_generator_kwargs[name] = value + + self.dict_keys_regex = [re.compile(rf"^{r}$") for r in dict_keys_regex] if dict_keys_regex else () + self.dict_keys_fields = dict_keys_fields or () self.initialize = True @@ -185,7 +192,6 @@ def _create_argparser() -> argparse.ArgumentParser: """ ArgParser factory """ - # TODO: dict_keys_regex and dict_keys_fields arguments parser = argparse.ArgumentParser( formatter_class=argparse.RawTextHelpFormatter, description="Convert given json files into Python models." @@ -215,6 +221,21 @@ def _create_argparser() -> argparse.ArgumentParser: "Warn.: This can lead to 6-7 times slowdown on large datasets.\n" " Be sure that you really need this option.\n\n" ) + parser.add_argument( + "--dict-keys-regex", "--dkr", + nargs="+", metavar="RegEx", + help="List of regular expressions (Python syntax).\n" + "If all keys of some dict are match one of them\n" + "then this dict will be marked as dict field but not nested model.\n" + "Note: ^ and $ tokens will be added automatically but you have to\n" + "escape other special characters manually.\n" + ) + parser.add_argument( + "--dict-keys-fields", "--dkf", + nargs="+", metavar="FIELD NAME", + help="List of model fields names that will be marked as dict fields\n\n" + ) + default_percent = f"{ModelFieldsPercentMatch.DEFAULT * 100:.0f}" default_number = f"{ModelFieldsNumberMatch.DEFAULT:.0f}" parser.add_argument( diff --git a/json_to_models/generator.py b/json_to_models/generator.py index e094700..ccbcfe6 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -182,6 +182,12 @@ def optimize_type(self, meta: MetaData, process_model_ptr=False) -> MetaData: elif isinstance(meta, DUnion): return self._optimize_union(meta) + elif isinstance(meta, DOptional): + t = self.optimize_type(meta.type) + if isinstance(t, DOptional): + t = t.type + return meta.replace(t) + elif isinstance(meta, SingleType) and (process_model_ptr or not isinstance(meta, ModelPtr)): # Optimize nested type return meta.replace(self.optimize_type(meta.type)) @@ -193,21 +199,27 @@ def optimize_type(self, meta: MetaData, process_model_ptr=False) -> MetaData: def _optimize_union(self, t: DUnion): # Replace DUnion of 1 element with this element - if len(t) == 1: - return t.types[0] + # if len(t) == 1: + # return t.types[0] # Split nested types into categories str_types: List[Union[type, StringSerializable]] = [] types_to_merge: List[dict] = [] list_types: List[DList] = [] + dict_types: List[DList] = [] other_types: List[MetaData] = [] for item in t.types: + if isinstance(item, DOptional): + item = item.type + other_types.append(NoneType) if isinstance(item, dict): types_to_merge.append(item) elif item in self.str_types_registry or item is str: str_types.append(item) elif isinstance(item, DList): list_types.append(item) + elif isinstance(item, DDict): + dict_types.append(item) else: other_types.append(item) @@ -217,10 +229,11 @@ def _optimize_union(self, t: DUnion): if types_to_merge: other_types.append(self.merge_field_sets(types_to_merge)) - if list_types: - other_types.append(DList(DUnion(*( - t.type for t in list_types - )))) + for cls, iterable_types in ((DList, list_types), (DDict, dict_types)): + if iterable_types: + other_types.append(cls(DUnion(*( + t.type for t in iterable_types + )))) if str in str_types: other_types.append(str) @@ -234,14 +247,20 @@ def _optimize_union(self, t: DUnion): if Unknown in types: types.remove(Unknown) - if len(types) > 1: - if NoneType in types: + optional = False + if NoneType in types: + optional = True + while NoneType in types: types.remove(NoneType) - if len(types) > 1: - return DOptional(DUnion(*types)) - else: - return DOptional(types[0]) - return DUnion(*types) + if len(types) > 1: + meta_type = DUnion(*types) + if len(meta_type.types) == 1: + meta_type = meta_type.types[0] + else: + meta_type = types[0] + + if optional: + return DOptional(meta_type) else: - return types[0] + return meta_type diff --git a/json_to_models/registry.py b/json_to_models/registry.py index 1653601..a466bf0 100644 --- a/json_to_models/registry.py +++ b/json_to_models/registry.py @@ -162,10 +162,16 @@ def merge_models(self, generator, strict=False) -> List[Tuple[ModelMeta, Set[Mod groups = new_groups replaces = [] + replaces_ids = set() for group in groups: model_meta = self._merge(generator, *group) generator.optimize_type(model_meta) + replaces_ids.add(model_meta.index) replaces.append((model_meta, group)) + + for model_meta in self.models: + if model_meta.index not in replaces_ids: + generator.optimize_type(model_meta) return replaces def _merge(self, generator, *models: ModelMeta): diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 4b68b9a..7177b87 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -62,11 +62,11 @@ def test_help(): -l User - "{test_data_path / 'users.json'}" """, id="list1_list2"), - pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" """, id="gists"), - pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --datetime""", id="gists_datetime"), - pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --merge percent number_10""", + pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files""", id="gists"), + pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --datetime""", id="gists_datetime"), + pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --merge percent number_10""", id="gists_merge_policy"), - pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --merge exact""", + pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --merge exact""", id="gists_no_merge"), ] diff --git a/test/test_generator/test_optimize_type.py b/test/test_generator/test_optimize_type.py index 1f172fa..6f68fb0 100644 --- a/test/test_generator/test_optimize_type.py +++ b/test/test_generator/test_optimize_type.py @@ -1,7 +1,7 @@ import pytest -from json_to_models.dynamic_typing import (BooleanString, DList, DOptional, DTuple, DUnion, FloatString, IntString, - NoneType, Unknown) +from json_to_models.dynamic_typing import (BooleanString, DDict, DList, DOptional, DTuple, DUnion, FloatString, + IntString, NoneType, Unknown) from json_to_models.generator import MetadataGenerator # MetaData | Optimized MetaData @@ -106,6 +106,26 @@ DList(DUnion(str, int)), id="union_of_str_int_FloatString" ), + pytest.param( + DOptional(DUnion(DOptional(str), str)), + DOptional(str), + id="optional_union_nested" + ), + pytest.param( + DUnion(NoneType, str, NoneType), + DOptional(str), + id="optional_str" + ), + pytest.param( + DUnion(DDict(str), DDict(str), DDict(str)), + DDict(str), + id="dict_union" + ), + pytest.param( + DUnion(DDict(str), DDict(int), DDict(str)), + DDict(DUnion(str, int)), + id="dict_union_2" + ), ]