From 2536cfd134e42d1f0501e6de47763cc57c9f278f Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Fri, 9 Sep 2022 18:10:31 +0300 Subject: [PATCH 1/6] Handle list data with regular --model argument; Deprecate --list argument --- json_to_models/cli.py | 136 +++++++++++++++++------------------- test/test_cli/test_utils.py | 4 +- 2 files changed, 68 insertions(+), 72 deletions(-) diff --git a/json_to_models/cli.py b/json_to_models/cli.py index c6986cb..05d90a1 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -81,14 +81,6 @@ def parse_args(self, args: List[str] = None): namespace = parser.parse_args(args) # Extract args - models: List[Tuple[str, Iterable[Path]]] = [ - (model_name, itertools.chain(*map(_process_path, paths))) - for model_name, *paths in namespace.model or () - ] - models_lists: List[Tuple[str, Tuple[str, Path]]] = [ - (model_name, (lookup, Path(path))) - for model_name, lookup, path in namespace.list or () - ] parser = getattr(FileLoaders, namespace.input_format) self.output_file = namespace.output self.enable_datetime = namespace.datetime @@ -104,8 +96,8 @@ def parse_args(self, args: List[str] = None): dict_keys_fields: List[str] = namespace.dict_keys_fields preamble: str = namespace.preamble - self.validate(models_lists, merge_policy, framework, code_generator) - self.setup_models_data(models, models_lists, parser) + self.setup_models_data(namespace.model or (), namespace.list or (), parser) + self.validate(merge_policy, framework, code_generator) self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw, dict_keys_regex, dict_keys_fields, disable_unicode_conversion, preamble) @@ -144,20 +136,15 @@ def version_string(self): '"""\n' ) - def validate(self, models_list, merge_policy, framework, code_generator): + def validate(self, merge_policy, framework, code_generator): """ Validate parsed args - :param models_list: List of pairs (model name, list of lookup expr and filesystem path) :param merge_policy: List of merge policies. Each merge policy is either string or string and policy arguments :param framework: Framework name (predefined code generator) :param code_generator: Code generator import string :return: """ - names = {name for name, _ in models_list} - if len(names) != len(models_list): - raise ValueError("Model names under -l flag should be unique") - for m in merge_policy: if isinstance(m, list): if m[0] not in self.MODEL_CMP_MAPPING: @@ -172,23 +159,33 @@ def validate(self, models_list, merge_policy, framework, code_generator): def setup_models_data( self, - models: Iterable[Tuple[str, Iterable[Path]]], - models_lists: Iterable[Tuple[str, Tuple[str, Path]]], + models: Iterable[Union[ + Tuple[str, str], + Tuple[str, str, str], + ]], + models_lists: Iterable[Tuple[str, str, str]], parser: 'FileLoaders.T' ): """ Initialize lazy loaders for models data """ - models_dict: Dict[str, List[Iterable[dict]]] = defaultdict(list) - for model_name, paths in models: - models_dict[model_name].append(parser(path) for path in paths) - for model_name, (lookup, path) in models_lists: - models_dict[model_name].append(iter_json_file(parser(path), lookup)) + models_dict: Dict[str, List[dict]] = defaultdict(list) + + models = list(models) + list(models_lists) + for model_tuple in models: + if len(model_tuple) == 2: + model_name, path_raw = model_tuple + lookup = '-' + elif len(model_tuple) == 3: + model_name, lookup, path_raw = model_tuple + else: + raise RuntimeError('`--model` argument should contain exactly 2 or 3 strings') - self.models_data = { - model_name: itertools.chain(*list_of_gen) - for model_name, list_of_gen in models_dict.items() - } + for real_path in process_path(path_raw): + iterator = iter_json_file(parser(real_path), lookup) + models_dict[model_name].extend(iterator) + + self.models_data = models_dict def set_args( self, @@ -257,20 +254,13 @@ def _create_argparser(cls) -> argparse.ArgumentParser: parser.add_argument( "-m", "--model", - nargs="+", action="append", metavar=("", ""), + nargs="+", action="append", metavar=(" [] ", ""), help="Model name and its JSON data as path or unix-like path pattern.\n" "'*', '**' or '?' patterns symbols are supported.\n\n" - ) - parser.add_argument( - "-l", "--list", - nargs=3, action="append", metavar=("", "", ""), - help="Like -m but given json file should contain list of model data.\n" + "JSON data could be array of models or single model\n\n" "If this file contains dict with nested list than you can pass\n" - " to lookup. Deep lookups are supported by dot-separated path.\n" - "If no lookup needed pass '-' as \n\n" - - "I.e. for file that contains dict {\"a\": {\"b\": [model_data, ...]}} you should\n" - "pass 'a.b' as .\n\n" + ". Deep lookups are supported by dot-separated path.\n" + "If no lookup needed pass '-' as (default)\n\n" ) parser.add_argument( "-i", "--input-format", @@ -377,6 +367,11 @@ def _create_argparser(cls) -> argparse.ArgumentParser: type=str, help="Code to insert into the generated file after the imports and before the list of classes\n\n" ) + parser.add_argument( + "-l", "--list", + nargs=3, action="append", metavar=("", "", ""), + help="DEPRECATED, use --model argument instead" + ) return parser @@ -395,27 +390,6 @@ def main(): print(cli.run()) -def path_split(path: str) -> List[str]: - """ - Split path into list of components - - :param path: string path - :return: List of files/patterns - """ - folders = [] - while True: - path, folder = os.path.split(path) - - if folder: - folders.append(folder) - else: - if path: - folders.append(path) - break - folders.reverse() - return folders - - class FileLoaders: T = Callable[[Path], Union[dict, list]] @@ -442,7 +416,7 @@ def ini(path: Path) -> dict: def dict_lookup(d: Union[dict, list], lookup: str) -> Union[dict, list]: """ - Extract nested dictionary value from key path. + Extract nested value from key path. If lookup is "-" returns dict as is. :param d: Nested dict @@ -460,25 +434,26 @@ def dict_lookup(d: Union[dict, list], lookup: str) -> Union[dict, list]: def iter_json_file(data: Union[dict, list], lookup: str) -> Generator[Union[dict, list], Any, None]: """ - Loads given 'path' file, perform lookup and return generator over json list. + Perform lookup and return generator over json list. Does not open file until iteration is started. - :param path: File Path instance + :param data: JSON data :param lookup: Dot separated lookup path - :return: + :return: Generator of the model data """ - l = dict_lookup(data, lookup) - assert isinstance(l, list), f"Dict lookup return {type(l)} but list is expected, check your lookup path" - yield from l + item = dict_lookup(data, lookup) + if isinstance(item, list): + yield from item + elif isinstance(item, dict): + yield item + else: + raise TypeError(f'dict or list is expected at {lookup if lookup != "-" else "JSON root"}, not {type(item)}') -def _process_path(path: str) -> Iterable[Path]: +def process_path(path: str) -> Iterable[Path]: """ Convert path pattern into path iterable. If non-pattern path is given return tuple of one element: (path,) - - :param path: - :return: """ split_path = path_split(path) clean_path = list(itertools.takewhile( @@ -502,3 +477,24 @@ def _process_path(path: str) -> Iterable[Path]: return path.glob(pattern_path) else: return path, + + +def path_split(path: str) -> List[str]: + """ + Split path into list of components + + :param path: string path + :return: List of files/patterns + """ + folders = [] + while True: + path, folder = os.path.split(path) + + if folder: + folders.append(folder) + else: + if path: + folders.append(path) + break + folders.reverse() + return folders diff --git a/test/test_cli/test_utils.py b/test/test_cli/test_utils.py index f8318de..6392c98 100644 --- a/test/test_cli/test_utils.py +++ b/test/test_cli/test_utils.py @@ -4,7 +4,7 @@ import pytest -from json_to_models.cli import _process_path, dict_lookup, iter_json_file, path_split +from json_to_models.cli import dict_lookup, iter_json_file, path_split, process_path from json_to_models.utils import convert_args echo = lambda *args, **kwargs: (args, kwargs) @@ -158,5 +158,5 @@ def test_iter_json_file(value, expected): @pytest.mark.parametrize("value,expected", test_process_path_data) def test_process_path(value, expected): - result = set(str(p).replace("\\", "/") for p in _process_path(value)) + result = set(str(p).replace("\\", "/") for p in process_path(value)) assert result == expected, f"(in value: {value})" From 4f1caae32a3e9f613d7f795b5c1463d84764a906 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Fri, 16 Sep 2022 17:55:43 +0300 Subject: [PATCH 2/6] Update tests --- test/test_cli/test_script.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 6e86d7f..036632f 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -52,16 +52,18 @@ def test_help(): test_commands = [ - pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" """, id="list1"), - pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" """, id="list2"), + pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" """, id="list1"), + pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" """, id="list1_legacy"), + pytest.param(f"""{executable} -m User "{test_data_path / 'users.json'}" """, id="list2"), + pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" """, id="list2_legacy"), pytest.param(f"""{executable} -m Photos "{test_data_path / 'photos.json'}" """, id="model1"), - pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" \ + pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" \ -m Photos "{test_data_path / 'photos.json'}" """, id="list1_model1"), - pytest.param(f"""{executable} -l Photo items "{test_data_path / 'photos.json'}" \ - -l User - "{test_data_path / 'users.json'}" """, + pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" \ + -m User "{test_data_path / 'users.json'}" """, id="list1_list2"), pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files""", @@ -75,7 +77,7 @@ def test_help(): pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --datetime --strings-converters""", id="gists_strings_converters"), - pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" --strings-converters""", + pytest.param(f"""{executable} -m User "{test_data_path / 'users.json'}" --strings-converters""", id="users_strings_converters"), pytest.param(f"""{executable} -m SomeUnicode "{test_data_path / 'unicode.json'}" """, id="convert_unicode"), @@ -216,15 +218,15 @@ def trim_header(line_string): wrong_arguments_commands = [ - pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" \ - -l Model - "{test_data_path / 'users.json'}" """, id="duplicate_name"), - pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" --merge unknown""", + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" \ + -m Model - "{test_data_path / 'users.json'}" """, id="duplicate_name"), + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --merge unknown""", id="wrong_merge_policy"), - pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" --merge unknown_10""", + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --merge unknown_10""", id="wrong_merge_policy"), - pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" -f custom""", + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" -f custom""", id="custom_model_generator_without_class_link"), - pytest.param(f"""{executable} -l Model items "{test_data_path / 'photos.json'}" --code-generator test""", + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --code-generator test""", id="class_link_without_custom_model_generator_enabled"), ] From 176c1757b453e0d754bb8cbd07f56ff450f30539 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Fri, 16 Sep 2022 17:56:44 +0300 Subject: [PATCH 3/6] Remove unneeded logic from #46 --- json_to_models/generator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/json_to_models/generator.py b/json_to_models/generator.py index 82ca79b..4dfc42f 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -15,7 +15,7 @@ StringSerializable, StringSerializableRegistry, Unknown, - registry + registry, ) _static_types = {float, bool, int} @@ -46,8 +46,6 @@ def generate(self, *data_variants: dict) -> dict: """ Convert given list of data variants to metadata dict """ - if isinstance(data_variants[0], list): - data_variants = [item for sublist in data_variants for item in sublist] fields_sets = [self._convert(data) for data in data_variants] fields = self.merge_field_sets(fields_sets) return self.optimize_type(fields) From 266f7f2e083f083bda2ebf82016518e1944ac098 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Fri, 16 Sep 2022 18:22:41 +0300 Subject: [PATCH 4/6] Update README.md and tests --- README.md | 30 +++-- test/test_cli/test_script.py | 6 +- .../test_models_composition.py | 105 +++++++++--------- 3 files changed, 68 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index b958e61..b5b491b 100644 --- a/README.md +++ b/README.md @@ -103,13 +103,13 @@ driver_standings.json ``` ``` -json2models -f pydantic -l DriverStandings - driver_standings.json +json2models -f pydantic -m DriverStandings driver_standings.json ``` ```python r""" generated by json2python-models v0.2.0 at Mon May 4 17:46:30 2020 -command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -l DriverStandings - driver_standings.json +command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -m DriverStandings driver_standings.json """ from pydantic import BaseModel, Field from typing import List @@ -469,27 +469,25 @@ json2models -m Car car_*.json -f attrs > car.py ``` Arguments: -* `-h`, `--help` - Show help message and exit -* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. `*`, `**` or `?` patterns symbols - are supported. - * **Format**: `-m [ ...]` - * **Example**: `-m Car audi.json reno.json` or `-m Car audi.json -m Car reno.json` (results will be the same) +* `-h`, `--help` - Show help message and exit -* `-l`, `--list` - Like `-m` but given json file should contain list of model data (dataset). If this file contains dict - with nested list than you can pass `` to lookup. Deep lookups are supported by dot-separated path. If no - lookup needed pass `-` as ``. - * **Format**: `-l ` - * **Example**: `-l Car - cars.json -l Person fetch_results.items.persons result.json` - * **Note**: Models names under these arguments should be unique. +* `-m`, `--model` - Model name and its JSON data as path or unix-like path pattern. + `*`, `**` or `?` patterns symbols are supported. + JSON data could be an array of models or single model. + If this file contains dict with nested list than you can pass + . Deep lookups are supported by dot-separated path. + If no lookup needed pass '-' as (default) + * **Format**: `-m [] ` + * **Example**: `-m Car audi.json -m Car results reno.json` * `-i`, `--input-format` - Input file format (parser). Default is JSON parser. Yaml parser requires PyYaml or ruamel.yaml to be installed. Ini parser uses builtin [configparser](https://docs.python.org/3/library/configparser.html). To implement new one - add new method to `cli.FileLoaders` (and create pull request :) ) - * **Format**: `-i {json, yaml, ini}` - * **Example**: `-i yaml` - * **Default**: `-i json` + * **Format**: `-i {json, yaml, ini}` + * **Example**: `-i yaml` + * **Default**: `-i json` * `-o`, `--output` - Output file * **Format**: `-o ` diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 036632f..4cb841f 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -57,6 +57,8 @@ def test_help(): pytest.param(f"""{executable} -m User "{test_data_path / 'users.json'}" """, id="list2"), pytest.param(f"""{executable} -l User - "{test_data_path / 'users.json'}" """, id="list2_legacy"), pytest.param(f"""{executable} -m Photos "{test_data_path / 'photos.json'}" """, id="model1"), + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" \ + -m Model - "{test_data_path / 'users.json'}" """, id="duplicate_name"), pytest.param(f"""{executable} -m Photo items "{test_data_path / 'photos.json'}" \ -m Photos "{test_data_path / 'photos.json'}" """, @@ -218,8 +220,6 @@ def trim_header(line_string): wrong_arguments_commands = [ - pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" \ - -m Model - "{test_data_path / 'users.json'}" """, id="duplicate_name"), pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --merge unknown""", id="wrong_merge_policy"), pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --merge unknown_10""", @@ -231,7 +231,7 @@ def trim_header(line_string): ] -@pytest.mark.xfail +@pytest.mark.xfail(strict=True) @pytest.mark.parametrize("command", wrong_arguments_commands) def test_wrong_arguments(command): print("Command:", command) diff --git a/test/test_code_generation/test_models_composition.py b/test/test_code_generation/test_models_composition.py index fa3c872..6aa89a9 100644 --- a/test/test_code_generation/test_models_composition.py +++ b/test/test_code_generation/test_models_composition.py @@ -1,5 +1,5 @@ -from typing import Dict, List, Set, Tuple, Union, Any -from copy import deepcopy +from typing import Dict, List, Set, Tuple, Union + import pytest from json_to_models.dynamic_typing import ModelMeta @@ -21,62 +21,59 @@ def test_list_ex(): assert l == [0, 'a', *range(1, 6), 'b', *range(6, 10)] -def generate_list_input(input_dict: Dict[str, Any]) -> Dict[str, Any]: - """ - Convert input into a list format. - - Mimics the case where the JSON in a file has a list - at the top level, rather than a dictionary. - - :param input_dict: dict with keys 'value', 'expected', and 'id' - :type input_dict: dict - :return: duplicate of the input structure but with the 'value' value as a list - :rtype: dict - """ - outputs = { - "expected": deepcopy(input_dict["expected"]), - "id": input_dict["id"] + "_list", - "value": [] - } - - for item in input_dict["value"]: - # item is a tuple of model name and model data - model = [{key: deepcopy(value)} for key, value in item[1].items()] - outputs["value"].append((item[0], model)) - - return outputs - - # This test relies on model names as a some sort of models ids # and may fail if some logic of their generation will be changed # List of Tuple[root_model_name, JSON data] | Dict[model_name, Set[root_model_names]] -extract_root_data_input = [ - { - "value": [ - ("TestModelA", {"count": 1, "items": [{"x": 0.5, "y": 0.1}]}), - ( - "TestModelB", +test_extract_root_data = [ + pytest.param([ + ("TestModelA", { + "count": 1, + "items": [ { - "next": "some_url", - "prev": None, - "count": 2000, - "items": [{"x": 0.5, "y": 0.1}], - }, - ), - ], - "expected": {"Item": {"TestModelA", "TestModelB"}, "TestModelA": set()}, - "id": "separate_roots" - },{ - "value": [ - ("TestModelA", {"count": 1, "items": [{"x": 0.5, "y": 0.1}]}), - ("TestModelB", {"count": 1, "items": [{"x": 0.5, "y": 0.1}]}), - ], - "expected": {"Item": {"TestModelA_TestModelB"}, "TestModelA_TestModelB": set()}, - "id": "merge_root", - }] - -extract_root_data_input_list = [generate_list_input(i) for i in extract_root_data_input] -test_extract_root_data = [pytest.param(inpt["value"], inpt["expected"], id=inpt["id"]) for inpt in extract_root_data_input + extract_root_data_input_list] + "x": .5, + "y": .1 + } + ] + }), + ("TestModelB", { + "next": "some_url", + "prev": None, + "count": 2000, + "items": [ + { + "x": .5, + "y": .1 + } + ] + }), + ], { + 'Item': {'TestModelA', 'TestModelB'}, + 'TestModelA': set() + }), + pytest.param([ + ("TestModelA", { + "count": 1, + "items": [ + { + "x": .5, + "y": .1 + } + ] + }), + ("TestModelB", { + "count": 1, + "items": [ + { + "x": .5, + "y": .1 + } + ] + }), + ], { + 'Item': {'TestModelA_TestModelB'}, + 'TestModelA_TestModelB': set() + }, id="merge_root") +] @pytest.mark.parametrize("value,expected", test_extract_root_data) def test_extract_root( From bbcbe3553a5260a6d3b401ddb2336caed2072e6a Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Fri, 16 Sep 2022 18:31:13 +0300 Subject: [PATCH 5/6] Improve code coverage --- test/test_cli/test_script.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 4cb841f..0d30d0f 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -228,6 +228,10 @@ def trim_header(line_string): id="custom_model_generator_without_class_link"), pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" --code-generator test""", id="class_link_without_custom_model_generator_enabled"), + pytest.param(f"""{executable} -m Model items "{test_data_path / 'photos.json'}" another_arg --code-generator test""", + id="4_args_model"), + pytest.param(f"""{executable} -m Model total "{test_data_path / 'photos.json'}" --code-generator test""", + id="non_dict_or_list_data"), ] From e785649d2a400017f7138e884e54a6147f58f0c9 Mon Sep 17 00:00:00 2001 From: bogdan_dm Date: Fri, 16 Sep 2022 18:51:23 +0300 Subject: [PATCH 6/6] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b5b491b..5adc36f 100644 --- a/README.md +++ b/README.md @@ -476,8 +476,8 @@ Arguments: `*`, `**` or `?` patterns symbols are supported. JSON data could be an array of models or single model. If this file contains dict with nested list than you can pass - . Deep lookups are supported by dot-separated path. - If no lookup needed pass '-' as (default) + ``. Deep lookups are supported by dot-separated path. + If no lookup needed pass '-' as `` (default) * **Format**: `-m [] ` * **Example**: `-m Car audi.json -m Car results reno.json`