diff --git a/TODO.md b/TODO.md index 5418ade..1899ebe 100644 --- a/TODO.md +++ b/TODO.md @@ -11,7 +11,8 @@ - [X] typing code generation - [ ] (Maybe in future) Extract to another module (by serializers for each dynamic typing class) - [X] attrs - - [ ] dataclasses + - [X] dataclasses + - [ ] post_init converters for StringSerializable types - [ ] generate from_json/to_json converters - [ ] Model class -> Meta format converter - [ ] attrs @@ -36,24 +37,10 @@ - [X] ISO date - [X] ISO time - [X] ISO datetime + - [ ] Don't create metadata (RCG_ORIGINAL_FIELD) if original_field == generated_field - [X] Cli tool - Testing - - Models layer - - [X] Create and register models - - [X] Test pointers in the models registry - - [ ] Test whats going on with strict/non-strict merging - - [ ] Save meta-models as python code - - [X] typing code generation - - [X] attrs - - [ ] dataclasses - - [ ] generate from_json/to_json converters - - [ ] Model class -> Meta format converter - - [ ] attrs - - [ ] dataclasses - - [ ] Implement existing models registration - - [ ] attrs - - [ ] dataclasses - Build, Deploy, CI - [X] setup.py diff --git a/json_to_models/cli.py b/json_to_models/cli.py index ee5d74a..5790f5c 100644 --- a/json_to_models/cli.py +++ b/json_to_models/cli.py @@ -15,6 +15,7 @@ from json_to_models.models import ModelsStructureType, compose_models from json_to_models.models.attr import AttrsModelCodeGenerator from json_to_models.models.base import GenericModelCodeGenerator, generate_code +from json_to_models.models.dataclasses import DataclassModelCodeGenerator from json_to_models.registry import ( ModelCmp, ModelFieldsEquals, ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry ) @@ -40,8 +41,7 @@ class Cli: MODEL_GENERATOR_MAPPING: Dict[str, Type[GenericModelCodeGenerator]] = { "base": convert_args(GenericModelCodeGenerator), "attrs": convert_args(AttrsModelCodeGenerator, meta=bool_js_style), - # TODO: vvvv - "dataclasses": None + "dataclasses": convert_args(DataclassModelCodeGenerator, meta=bool_js_style, post_init_converters=bool_js_style) } def __init__(self): diff --git a/json_to_models/models/attr.py b/json_to_models/models/attr.py index fe337b6..29b60c2 100644 --- a/json_to_models/models/attr.py +++ b/json_to_models/models/attr.py @@ -1,14 +1,8 @@ from inspect import isclass -from typing import Iterable, List, Tuple +from typing import List, Tuple -from .base import GenericModelCodeGenerator, template -from ..dynamic_typing import DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable - -METADATA_FIELD_NAME = "RCG_ORIGINAL_FIELD" -KWAGRS_TEMPLATE = "{% for key, value in kwargs.items() %}" \ - "{{ key }}={{ value }}" \ - "{% if not loop.last %}, {% endif %}" \ - "{% endfor %}" +from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template +from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable DEFAULT_ORDER = ( ("default", "converter", "factory"), @@ -17,24 +11,6 @@ ) -def sort_kwargs(kwargs: dict, ordering: Iterable[Iterable[str]] = DEFAULT_ORDER) -> dict: - sorted_dict_1 = {} - sorted_dict_2 = {} - current = sorted_dict_1 - for group in ordering: - if isinstance(group, str): - if group != "*": - raise ValueError(f"Unknown kwarg group: {group}") - current = sorted_dict_2 - else: - for item in group: - if item in kwargs: - value = kwargs.pop(item) - current[item] = value - sorted_dict = {**sorted_dict_1, **kwargs, **sorted_dict_2} - return sorted_dict - - class AttrsModelCodeGenerator(GenericModelCodeGenerator): ATTRS = template("attr.s" "{% if kwargs %}" @@ -45,7 +21,7 @@ class AttrsModelCodeGenerator(GenericModelCodeGenerator): def __init__(self, model: ModelMeta, meta=False, attrs_kwargs: dict = None, **kwargs): """ :param model: ModelMeta instance - :param no_meta: Disable generation of metadata as attrib argument + :param meta: Enable generation of metadata as attrib argument :param attrs_kwargs: kwargs for @attr.s() decorators :param kwargs: """ @@ -84,6 +60,8 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP meta: DOptional if isinstance(meta.type, DList): body_kwargs["factory"] = "list" + elif isinstance(meta.type, DDict): + body_kwargs["factory"] = "dict" else: body_kwargs["default"] = "None" if isclass(meta.type) and issubclass(meta.type, StringSerializable): @@ -94,5 +72,5 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP if not self.no_meta: body_kwargs["metadata"] = {METADATA_FIELD_NAME: name} - data["body"] = self.ATTRIB.render(kwargs=sort_kwargs(body_kwargs)) + data["body"] = self.ATTRIB.render(kwargs=sort_kwargs(body_kwargs, DEFAULT_ORDER)) return imports, data diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index 03d8fde..ba02497 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Type +from typing import Iterable, List, Tuple, Type import inflection from jinja2 import Template @@ -6,6 +6,12 @@ from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER, indent, sort_fields from ..dynamic_typing import AbsoluteModelRef, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing +METADATA_FIELD_NAME = "RCG_ORIGINAL_FIELD" +KWAGRS_TEMPLATE = "{% for key, value in kwargs.items() %}" \ + "{{ key }}={{ value }}" \ + "{% if not loop.last %}, {% endif %}" \ + "{% endfor %}" + def template(pattern: str, indent: str = INDENT) -> Template: """ @@ -159,3 +165,21 @@ def generate_code(structure: ModelsStructureType, class_generator: Type[GenericM else: imports_str = "" return imports_str + objects_delimiter.join(classes) + "\n" + + +def sort_kwargs(kwargs: dict, ordering: Iterable[Iterable[str]]) -> dict: + sorted_dict_1 = {} + sorted_dict_2 = {} + current = sorted_dict_1 + for group in ordering: + if isinstance(group, str): + if group != "*": + raise ValueError(f"Unknown kwarg group: {group}") + current = sorted_dict_2 + else: + for item in group: + if item in kwargs: + value = kwargs.pop(item) + current[item] = value + sorted_dict = {**sorted_dict_1, **kwargs, **sorted_dict_2} + return sorted_dict diff --git a/json_to_models/models/dataclass.py b/json_to_models/models/dataclass.py deleted file mode 100644 index e69de29..0000000 diff --git a/json_to_models/models/dataclasses.py b/json_to_models/models/dataclasses.py new file mode 100644 index 0000000..c4db650 --- /dev/null +++ b/json_to_models/models/dataclasses.py @@ -0,0 +1,81 @@ +from inspect import isclass +from typing import List, Tuple + +from .base import GenericModelCodeGenerator, KWAGRS_TEMPLATE, METADATA_FIELD_NAME, sort_kwargs, template +from ..dynamic_typing import DDict, DList, DOptional, ImportPathList, MetaData, ModelMeta, StringSerializable + +DEFAULT_ORDER = ( + ("default", "default_factory"), + "*", + ("metadata",) +) + + +class DataclassModelCodeGenerator(GenericModelCodeGenerator): + DC_DECORATOR = template("dataclass" + "{% if kwargs %}" + f"({KWAGRS_TEMPLATE})" + "{% endif %}") + DC_FIELD = template(f"field({KWAGRS_TEMPLATE})") + + def __init__(self, model: ModelMeta, meta=False, post_init_converters=False, dataclass_kwargs: dict = None, + **kwargs): + """ + :param model: ModelMeta instance + :param meta: Enable generation of metadata as attrib argument + :param post_init_converters: Enable generation of type converters in __post_init__ methods + :param dataclass_kwargs: kwargs for @dataclass() decorators + :param kwargs: + """ + super().__init__(model, **kwargs) + self.post_init_converters = post_init_converters + self.no_meta = not meta + self.dataclass_kwargs = dataclass_kwargs or {} + + def generate(self, nested_classes: List[str] = None) -> Tuple[ImportPathList, str]: + """ + :param nested_classes: list of strings that contains classes code + :return: list of import data, class code + """ + imports, code = super().generate(nested_classes) + imports.append(('dataclasses', ['dataclass, field'])) + return imports, code + + @property + def decorators(self) -> List[str]: + """ + :return: List of decorators code (without @) + """ + return [self.DC_DECORATOR.render(kwargs=self.dataclass_kwargs)] + + def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: + """ + Form field data for template + + :param name: Field name + :param meta: Field metadata + :param optional: Is field optional + :return: imports, field data + """ + imports, data = super().field_data(name, meta, optional) + body_kwargs = {} + if optional: + meta: DOptional + if isinstance(meta.type, DList): + body_kwargs["default_factory"] = "list" + elif isinstance(meta.type, DDict): + body_kwargs["default_factory"] = "dict" + else: + body_kwargs["default"] = "None" + if isclass(meta.type) and issubclass(meta.type, StringSerializable): + pass + elif isclass(meta) and issubclass(meta, StringSerializable): + pass + + if not self.no_meta: + body_kwargs["metadata"] = {METADATA_FIELD_NAME: name} + if len(body_kwargs) == 1 and next(iter(body_kwargs.keys())) == "default": + data["body"] = body_kwargs["default"] + elif body_kwargs: + data["body"] = self.DC_FIELD.render(kwargs=sort_kwargs(body_kwargs, DEFAULT_ORDER)) + return imports, data diff --git a/test/test_cli/test_script.py b/test/test_cli/test_script.py index 0832b69..cefae48 100644 --- a/test/test_cli/test_script.py +++ b/test/test_cli/test_script.py @@ -79,8 +79,9 @@ def _validate_result(proc: subprocess.Popen) -> Tuple[str, str]: assert stdout, stdout assert proc.returncode == 0 # Note: imp package is deprecated but I can't find a way to create dummy module using importlib - module = imp.new_module("model") - exec(compile(stdout, "model.py", "exec"), module.__dict__) + module = imp.new_module("test_model") + sys.modules["test_model"] = module + exec(compile(stdout, "test_model.py", "exec"), module.__dict__) return stdout, stderr @@ -100,6 +101,15 @@ def test_script_attrs(command): print(stdout) +@pytest.mark.parametrize("command", test_commands) +def test_script_dataclasses(command): + command += " -f dataclasses" + proc = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = _validate_result(proc) + assert "@dataclass" in stdout + print(stdout) + + @pytest.mark.parametrize("command", test_commands) def test_script_custom(command): command += " -f custom --code-generator json_to_models.models.attr.AttrsModelCodeGenerator" diff --git a/test/test_code_generation/test_attrs_generation.py b/test/test_code_generation/test_attrs_generation.py index 5408284..692542e 100644 --- a/test/test_code_generation/test_attrs_generation.py +++ b/test/test_code_generation/test_attrs_generation.py @@ -4,8 +4,8 @@ from json_to_models.dynamic_typing import (DDict, DList, DOptional, FloatString, IntString, ModelMeta, compile_imports) from json_to_models.models import sort_fields -from json_to_models.models.attr import AttrsModelCodeGenerator, METADATA_FIELD_NAME, sort_kwargs -from json_to_models.models.base import generate_code +from json_to_models.models.attr import AttrsModelCodeGenerator, DEFAULT_ORDER +from json_to_models.models.base import METADATA_FIELD_NAME, generate_code, sort_kwargs from test.test_code_generation.test_models_code_generator import model_factory, trim @@ -16,7 +16,7 @@ def test_attrib_kwargs_sort(): converter='a', default=None, x=1, - )) + ), DEFAULT_ORDER) expected = ['default', 'converter', 'y', 'x', 'metadata'] for k1, k2 in zip(sorted_kwargs.keys(), expected): assert k1 == k2 diff --git a/test/test_code_generation/test_dataclasses_generation.py b/test/test_code_generation/test_dataclasses_generation.py new file mode 100644 index 0000000..28048aa --- /dev/null +++ b/test/test_code_generation/test_dataclasses_generation.py @@ -0,0 +1,162 @@ +from typing import Dict, List + +import pytest + +from json_to_models.dynamic_typing import (DDict, DList, DOptional, FloatString, IntString, ModelMeta, compile_imports) +from json_to_models.models import sort_fields +from json_to_models.models.base import METADATA_FIELD_NAME, generate_code +from json_to_models.models.dataclasses import DataclassModelCodeGenerator +from test.test_code_generation.test_models_code_generator import model_factory, trim + + +def field_meta(original_name): + return f"metadata={{'{METADATA_FIELD_NAME}': '{original_name}'}}" + + +# Data structure: +# pytest.param id -> { +# "model" -> (model_name, model_metadata), +# test_name -> expected, ... +# } +test_data = { + "base": { + "model": ("Test", { + "foo": int, + "bar": int, + "baz": float + }), + "fields_data": { + "foo": { + "name": "foo", + "type": "int", + "body": f"field({field_meta('foo')})" + }, + "bar": { + "name": "bar", + "type": "int", + "body": f"field({field_meta('bar')})" + }, + "baz": { + "name": "baz", + "type": "float", + "body": f"field({field_meta('baz')})" + } + }, + "fields": { + "imports": "", + "fields": [ + f"foo: int = field({field_meta('foo')})", + f"bar: int = field({field_meta('bar')})", + f"baz: float = field({field_meta('baz')})", + ] + }, + "generated": trim(f""" + from dataclasses import dataclass, field + + + @dataclass + class Test: + foo: int = field({field_meta('foo')}) + bar: int = field({field_meta('bar')}) + baz: float = field({field_meta('baz')}) + """) + }, + "complex": { + "model": ("Test", { + "foo": int, + "baz": DOptional(DList(DList(str))), + "bar": DOptional(IntString), + "qwerty": FloatString, + "asdfg": DOptional(int), + "dict": DDict(int) + }), + "fields_data": { + "foo": { + "name": "foo", + "type": "int", + "body": f"field({field_meta('foo')})" + }, + "baz": { + "name": "baz", + "type": "Optional[List[List[str]]]", + "body": f"field(default_factory=list, {field_meta('baz')})" + }, + "bar": { + "name": "bar", + "type": "Optional[IntString]", + "body": f"field(default=None, {field_meta('bar')})" + }, + "qwerty": { + "name": "qwerty", + "type": "FloatString", + "body": f"field({field_meta('qwerty')})" + }, + "asdfg": { + "name": "asdfg", + "type": "Optional[int]", + "body": f"field(default=None, {field_meta('asdfg')})" + }, + "dict": { + "name": "dict", + "type": "Dict[str, int]", + "body": f"field({field_meta('dict')})" + } + }, + "generated": trim(f""" + from dataclasses import dataclass, field + from json_to_models.dynamic_typing import FloatString, IntString + from typing import Dict, List, Optional + + + @dataclass + class Test: + foo: int = field({field_meta('foo')}) + qwerty: FloatString = field({field_meta('qwerty')}) + dict: Dict[str, int] = field({field_meta('dict')}) + baz: Optional[List[List[str]]] = field(default_factory=list, {field_meta('baz')}) + bar: Optional[IntString] = field(default=None, {field_meta('bar')}) + asdfg: Optional[int] = field(default=None, {field_meta('asdfg')}) + """) + } +} + +test_data_unzip = { + test: [ + pytest.param( + model_factory(*data["model"]), + data[test], + id=id + ) + for id, data in test_data.items() + if test in data + ] + for test in ("fields_data", "fields", "generated") +} + + +@pytest.mark.parametrize("value,expected", test_data_unzip["fields_data"]) +def test_fields_data_attr(value: ModelMeta, expected: Dict[str, dict]): + gen = DataclassModelCodeGenerator(value, meta=True) + required, optional = sort_fields(value) + for is_optional, fields in enumerate((required, optional)): + for field in fields: + field_imports, data = gen.field_data(field, value.type[field], bool(is_optional)) + assert data == expected[field] + + +@pytest.mark.parametrize("value,expected", test_data_unzip["fields"]) +def test_fields_attr(value: ModelMeta, expected: dict): + expected_imports: str = expected["imports"] + expected_fields: List[str] = expected["fields"] + gen = DataclassModelCodeGenerator(value, meta=True) + imports, fields = gen.fields + imports = compile_imports(imports) + assert imports == expected_imports + assert fields == expected_fields + + +@pytest.mark.parametrize("value,expected", test_data_unzip["generated"]) +def test_generated_attr(value: ModelMeta, expected: str): + generated = generate_code(([{"model": value, "nested": []}], {}), DataclassModelCodeGenerator, + class_generator_kwargs={'meta': True}) + assert generated.rstrip() == expected, generated