diff --git a/README.md b/README.md index 2838458..b958e61 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ ![Example](/etc/convert.png) -json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes -([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs)) -from JSON dataset. +json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes +([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs)) +from JSON datasets. ## Features @@ -155,9 +155,9 @@ class Constructor(BaseModel): `swagger.json` from any online API (I tested file generated by drf-yasg and another one for Spotify API) -It requires a lit bit of tweaking: +It requires a bit of tweaking: * Some fields store routes/models specs as dicts -* There is a lot of optinal fields so we reduce merging threshold +* There are a lot of optinal fields so we reduce merging threshold * Disable string literals ``` @@ -405,9 +405,45 @@ class Run(BaseModel):

+### Example with preamble + +
----- Show ----- +

+A simple example to demonstrate adding extra code before the class list. + +```sh +json2models -f pydantic --preamble "# set up defaults +USERNAME = 'user' +SERVER_IP = '127.0.0.1' +" -m Swagger testing_tools/swagger.json +``` + +```py +r""" +generated by json2python-models v0.2.5 at Tue Aug 23 08:55:09 2022 +command: json2models -f pydantic --preamble # set up defaults +USERNAME = 'user' +SERVER_IP = '127.0.0.1' + -m Swagger testing_tools/swagger.json -o output.py +""" +from pydantic import BaseModel, Field +from typing import Any, List, Literal, Optional, Union + + +# set up defaults +USERNAME = 'user' +SERVER_IP = '127.0.0.1' + + + +class Swagger(BaseModel): + # etc. +``` +

+ ## Installation -| **Be ware**: this project supports only `python3.7` and higher. | +| **Beware**: this project supports only `python3.7` and higher. | | --- | To install it, use `pip`: @@ -426,7 +462,7 @@ python setup.py install ### CLI -For regular usage CLI tool is the best option. After you install this package you could use it as `json2models ` +For regular usage CLI tool is the best option. After you install this package you can use it as `json2models ` or `python -m json_to_models `. I.e.: ``` json2models -m Car car_*.json -f attrs > car.py @@ -464,61 +500,71 @@ Arguments: * **Format**: `-f {base, pydantic, attrs, dataclasses, custom}` * **Example**: `-f pydantic` * **Default**: `-f base` - + * `-s`, `--structure` - Models composition style. - * **Format**: `-s {flat, nested}` + * **Format**: `-s {flat, nested}` * **Example**: `-s nested` * **Default**: `-s flat` - + +* `--preamble` - Additional material to be + * **Format**: `--preamble ""` + * **Example**: + ```sh + --preamble "# set up defaults + USERNAME = 'user' + SERVER = '127.0.0.1'" + ``` + * **Optional** + * `--datetime` - Enable datetime/date/time strings parsing. * **Default**: disabled * **Warning**: This can lead to 6-7 times slowdown on large datasets. Be sure that you really need this option. - + * `--disable-unicode-conversion`, `--no-unidecode` - Disable unicode conversion in field labels and class names * **Default**: enabled - + * `--strings-converters` - Enable generation of string types converters (i.e. `IsoDatetimeString` or `BooleanString`). * **Default**: disabled - + * `--max-strings-literals` - Generate `Literal['foo', 'bar']` when field have less than NUMBER string constants as values. - * **Format**: `--max-strings-literals ` + * **Format**: `--max-strings-literals ` * **Default**: 10 (generator classes could override it) * **Example**: `--max-strings-literals 5` - only 5 literals will be saved and used to code generation * **Note**: There could not be more than **15** literals per field (for performance reasons) * **Note**: `attrs` code generator do not use Literals and just generate `str` fields instead -* `--merge` - Merge policy settings. Possible values are: +* `--merge` - Merge policy settings. Possible values are: * **Format**: `--merge MERGE_POLICY [MERGE_POLICY ...]` * **Possible values** (MERGE_POLICY): - * `percent[_]` - two models had a certain percentage of matched field names. - Custom value could be i.e. `percent_95`. - * `number[_]` - two models had a certain number of matched field names. + * `percent[_]` - two models had a certain percentage of matched field names. + Custom value could be i.e. `percent_95`. + * `number[_]` - two models had a certain number of matched field names. * `exact` - two models should have exact same field names to merge. * **Example**: `--merge percent_95 number_20` - merge if 95% of fields are matched or 20 of fields are matched * **Default**: `--merge percent_70 number_10` - + * `--dict-keys-regex`, `--dkr` - List of regular expressions (Python syntax). - If all keys of some dict are match one of the pattern then + If all keys of some dict are match one of the pattern then this dict will be marked as dict field but not nested model. * **Format**: `--dkr RegEx [RegEx ...]` * **Example**: `--dkr node_\d+ \d+_\d+_\d+` - * **Note**: `^` and `$` (string borders) tokens will be added automatically but you + * **Note**: `^` and `$` (string borders) tokens will be added automatically but you have to escape other special characters manually. * **Optional** - + * `--dict-keys-fields`, `--dkf` - List of model fields names that will be marked as dict fields * **Format**: `--dkf FIELD_NAME [FIELD_NAME ...]` * **Example**: `--dkf "dict_data" "mapping"` * **Optional** - + * `--code-generator` - Absolute import path to `GenericModelCodeGenerator` subclass. * **Format**: `--code-generator CODE_GENERATOR` * **Example**: `-f mypackage.mymodule.DjangoModelsGenerator` * **Note**: Is ignored without `-f custom` but is required with it. - -* `--code-generator-kwargs` - List of GenericModelCodeGenerator subclass arguments (for `__init__` method, - see docs of specific subclass). - Each argument should be in following format: `argument_name=value` or `"argument_name=value with space"`. + +* `--code-generator-kwargs` - List of GenericModelCodeGenerator subclass arguments (for `__init__` method, + see docs of specific subclass). + Each argument should be in following format: `argument_name=value` or `"argument_name=value with space"`. Boolean values should be passed in JS style: `true` or `false` * **Format**: `--code-generator-kwargs [NAME=VALUE [NAME=VALUE ...]]` * **Example**: `--code-generator-kwargs kwarg1=true kwarg2=10 "kwarg3=It is string with spaces"` diff --git a/json_to_models/cli.py b/json_to_models/cli.py index 20222c1..c6986cb 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -102,11 +102,12 @@ def parse_args(self, args: List[str] = None): code_generator_kwargs_raw: List[str] = namespace.code_generator_kwargs dict_keys_regex: List[str] = namespace.dict_keys_regex dict_keys_fields: List[str] = namespace.dict_keys_fields + preamble: str = namespace.preamble - self.validate(models, models_lists, merge_policy, framework, code_generator) + self.validate(models_lists, merge_policy, framework, code_generator) self.setup_models_data(models, models_lists, parser) self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw, - dict_keys_regex, dict_keys_fields, disable_unicode_conversion) + dict_keys_regex, dict_keys_fields, disable_unicode_conversion, preamble) def run(self): if self.enable_datetime: @@ -122,8 +123,11 @@ def run(self): registry.merge_models(generator) registry.generate_names() structure = self.structure_fn(registry.models_map) - output = self.version_string + \ - generate_code(structure, self.model_generator, class_generator_kwargs=self.model_generator_kwargs) + output = self.version_string + generate_code( + structure, + self.model_generator, + class_generator_kwargs=self.model_generator_kwargs, + preamble=self.preamble) if self.output_file: with open(self.output_file, "w", encoding="utf-8") as f: f.write(output) @@ -140,11 +144,10 @@ def version_string(self): '"""\n' ) - def validate(self, models, models_list, merge_policy, framework, code_generator): + def validate(self, models_list, merge_policy, framework, code_generator): """ Validate parsed args - :param models: List of pairs (model name, list of filesystem path) :param models_list: List of pairs (model name, list of lookup expr and filesystem path) :param merge_policy: List of merge policies. Each merge policy is either string or string and policy arguments :param framework: Framework name (predefined code generator) @@ -196,7 +199,8 @@ def set_args( code_generator_kwargs_raw: List[str], dict_keys_regex: List[str], dict_keys_fields: List[str], - disable_unicode_conversion: bool + disable_unicode_conversion: bool, + preamble: str, ): """ Convert CLI args to python representation and set them to appropriate object attributes @@ -236,7 +240,9 @@ def set_args( self.dict_keys_regex = [re.compile(rf"^{r}$") for r in dict_keys_regex] if dict_keys_regex else () self.dict_keys_fields = dict_keys_fields or () - + if preamble: + preamble = preamble.strip() + self.preamble = preamble or None self.initialized = True @classmethod @@ -366,6 +372,11 @@ def _create_argparser(cls) -> argparse.ArgumentParser: "Boolean values should be passed in JS style: true | false" "\n\n" ) + parser.add_argument( + "--preamble", + type=str, + help="Code to insert into the generated file after the imports and before the list of classes\n\n" + ) return parser diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index 80c82ea..6102cf1 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -54,11 +54,11 @@ class GenericModelCodeGenerator: @{{ decorator }} {% endfor -%} class {{ name }}{% if bases %}({{ bases }}){% endif %}: - + {%- for code in nested %} {{ code }} {% endfor -%} - + {%- if fields -%} {%- for field in fields %} {{ field }} @@ -66,7 +66,7 @@ class {{ name }}{% if bases %}({{ bases }}){% endif %}: {%- else %} pass {%- endif -%} - {%- if extra %} + {%- if extra %} {{ extra }} {%- endif -%} """) @@ -210,7 +210,7 @@ def _generate_code( lvl=0 ) -> Tuple[ImportPathList, List[str]]: """ - Walk thought models structure and covert them into code + Walk through the model structures and convert them into code :param structure: Result of compose_models or similar function :param class_generator: GenericModelCodeGenerator subclass @@ -241,7 +241,9 @@ def _generate_code( def generate_code(structure: ModelsStructureType, class_generator: Type[GenericModelCodeGenerator], - class_generator_kwargs: dict = None, objects_delimiter: str = OBJECTS_DELIMITER) -> str: + class_generator_kwargs: dict = None, + objects_delimiter: str = OBJECTS_DELIMITER, + preamble: str = None) -> str: """ Generate ready-to-use code @@ -249,15 +251,18 @@ def generate_code(structure: ModelsStructureType, class_generator: Type[GenericM :param class_generator: GenericModelCodeGenerator subclass :param class_generator_kwargs: kwags for GenericModelCodeGenerator init :param objects_delimiter: Delimiter between root level classes + :param preamble: code to insert after the imports and before the classes :return: Generated code """ root, mapping = structure with AbsoluteModelRef.inject(mapping): imports, classes = _generate_code(root, class_generator, class_generator_kwargs or {}) + imports_str = "" if imports: imports_str = compile_imports(imports) + objects_delimiter - else: - imports_str = "" + if preamble: + imports_str += preamble + objects_delimiter + return imports_str + objects_delimiter.join(classes) + "\n" diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index aee4844..6e86d7f 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -90,7 +90,8 @@ def test_help(): ] -def _validate_result(proc: subprocess.Popen, output=None, output_file: Path = None) -> Tuple[str, str]: +def execute_test(command, output_file: Path = None, output=None) -> Tuple[str, str]: + proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = map(bytes.decode, proc.communicate()) if output_file: assert output is None @@ -108,31 +109,27 @@ def _validate_result(proc: subprocess.Popen, output=None, output_file: Path = No exec(compile(stdout, "test_model.py", "exec"), module.__dict__) except Exception as e: assert not e, stdout - return stdout, stderr + + print(stdout) + return stdout @pytest.mark.parametrize("command", test_commands) def test_script(command): - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) - print(stdout) + execute_test(command) @pytest.mark.parametrize("command", test_commands) def test_script_flat(command): command += " -s flat" - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) - print(stdout) + execute_test(command) @pytest.mark.parametrize("command", test_commands) def test_script_attrs(command): command += " -f attrs" - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) + stdout = execute_test(command) assert "@attr.s" in stdout - print(stdout) @pytest.mark.parametrize("command", test_commands) @@ -140,10 +137,8 @@ def test_script_pydantic(command): command += " -f pydantic" # Pydantic has native (str) -> (builtin_type) converters command = command.replace('--strings-converters', '') - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) + stdout = execute_test(command) assert "(BaseModel):" in stdout - print(stdout) @pytest.mark.parametrize("command", test_commands) @@ -151,30 +146,73 @@ def test_script_pydantic_disable_literals(command): command += " -f pydantic --code-generator-kwargs max_literals=0" # Pydantic has native (str) -> (builtin_type) converters command = command.replace('--strings-converters', '') - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) + stdout = execute_test(command) assert "(BaseModel):" in stdout assert "Literal" not in stdout - print(stdout) @pytest.mark.parametrize("command", test_commands) def test_script_dataclasses(command): command += " -f dataclasses" - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) + stdout = execute_test(command) assert "@dataclass" in stdout - print(stdout) @pytest.mark.parametrize("command", test_commands) def test_script_custom(command): command += " -f custom --code-generator json_to_models.models.attr.AttrsModelCodeGenerator" command += ' --code-generator-kwargs "meta=true"' - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc) + stdout = execute_test(command) assert "@attr.s" in stdout - print(stdout) + + +@pytest.mark.parametrize("command", test_commands) +def test_add_preamble(command): + + PREAMBLE_TEXT = """ +# this is some test code +# to be added to the file + + +# let's see if it works + + + """ + stdout = execute_test(command + ' --preamble "' + PREAMBLE_TEXT + '"') + assert "let's see if it works" in stdout + + +@pytest.mark.parametrize("command", test_commands) +def test_add_trim_preamble(command): + + def trim_header(line_string): + """remove the quoted command and everything from the first class declaration onwards""" + lines = line_string.splitlines() + start = 0 + end = 0 + line_no = 0 + for l in lines: + if l.startswith('"""'): + start = line_no + if l.startswith('class '): + end = line_no + break + line_no += 1 + + return lines[start:end] + + expected_result = execute_test(command) + + BLANK_SPACE = """ + + + + + """ + # ensure blank space does not get propagated + stdout = execute_test(command + ' --preamble "' + BLANK_SPACE + '"') + + assert trim_header(expected_result) == trim_header(stdout) wrong_arguments_commands = [ @@ -195,14 +233,11 @@ def test_script_custom(command): @pytest.mark.parametrize("command", wrong_arguments_commands) def test_wrong_arguments(command): print("Command:", command) - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _validate_result(proc) + execute_test(command) @pytest.mark.parametrize("command", test_commands) def test_script_output_file(command): file = tmp_path / 'out.py' command += f" -o {file}" - proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = _validate_result(proc, output_file=file) - print(stdout) + execute_test(command, output_file=file)