From 00f08aa8aba7795a759a0239e0dc8aa84663e0ce Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sun, 2 Dec 2018 18:39:28 +0300 Subject: [PATCH 1/2] Decode unicode and remove non-words characters in models fields names --- json_to_models/generator.py | 4 --- json_to_models/models/base.py | 19 +++++++++- .../test_attrs_generation.py | 35 +++++++++++++++---- test/test_generator/test_detect_type.py | 2 +- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/json_to_models/generator.py b/json_to_models/generator.py index 6921f95..d67de0c 100644 --- a/json_to_models/generator.py +++ b/json_to_models/generator.py @@ -1,4 +1,3 @@ -import keyword import re from collections import OrderedDict from typing import Any, Callable, List, Optional, Pattern, Union @@ -8,7 +7,6 @@ from .dynamic_typing import (ComplexType, DDict, DList, DOptional, DUnion, MetaData, ModelPtr, Null, SingleType, StringSerializable, StringSerializableRegistry, Unknown, registry) -keywords_set = set(keyword.kwlist) _static_types = {float, bool, int} class MetadataGenerator: @@ -51,8 +49,6 @@ def _convert(self, data: dict): # Crash does not produce any useful logs and can occur any time after bad string was processed # It can be reproduced on real_apis tests (openlibrary API) convert_dict = key not in self.dict_keys_fields - if key in keywords_set: - key += "_" fields[key] = self._detect_type(value if not isinstance(value, str) else unidecode(value), convert_dict) return fields diff --git a/json_to_models/models/base.py b/json_to_models/models/base.py index 6eba4e3..af78bfe 100644 --- a/json_to_models/models/base.py +++ b/json_to_models/models/base.py @@ -1,7 +1,10 @@ +import keyword +import re from typing import Iterable, List, Tuple, Type import inflection from jinja2 import Template +from unidecode import unidecode from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER, indent, sort_fields from ..dynamic_typing import AbsoluteModelRef, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing @@ -12,6 +15,8 @@ "{% if not loop.last %}, {% endif %}" \ "{% endfor %}" +keywords_set = set(keyword.kwlist) +ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine'] def template(pattern: str, indent: str = INDENT) -> Template: """ @@ -80,6 +85,17 @@ def decorators(self) -> List[str]: """ return [] + @classmethod + def convert_field_name(cls, name): + if name in keywords_set: + name += "_" + name = unidecode(name) + name = re.sub(r"\W", "", name) + if not ('a' <= name[0].lower() <= 'z'): + if '0' <= name[0] <= '9': + name = ones[int(name[0])] + "_" + name[1:] + return inflection.underscore(name) + def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]: """ Form field data for template @@ -90,8 +106,9 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP :return: imports, field data """ imports, typing = metadata_to_typing(meta) + data = { - "name": inflection.underscore(name), + "name": self.convert_field_name(name), "type": typing } return imports, data diff --git a/test/test_code_generation/test_attrs_generation.py b/test/test_code_generation/test_attrs_generation.py index 12f90a8..5d641f4 100644 --- a/test/test_code_generation/test_attrs_generation.py +++ b/test/test_code_generation/test_attrs_generation.py @@ -88,38 +88,56 @@ class Test: "bar": DOptional(IntString), "qwerty": FloatString, "asdfg": DOptional(int), - "dict": DDict(int) + "dict": DDict(int), + "not": bool, + "1day": int, + "день_недели": str, }), "fields_data": { "foo": { "name": "foo", "type": "int", - "body": f"attr.ib()" + "body": "attr.ib()" }, "baz": { "name": "baz", "type": "Optional[List[List[str]]]", - "body": f"attr.ib(factory=list)" + "body": "attr.ib(factory=list)" }, "bar": { "name": "bar", "type": "Optional[IntString]", - "body": f"attr.ib(default=None, converter=optional(IntString))" + "body": "attr.ib(default=None, converter=optional(IntString))" }, "qwerty": { "name": "qwerty", "type": "FloatString", - "body": f"attr.ib(converter=FloatString)" + "body": "attr.ib(converter=FloatString)" }, "asdfg": { "name": "asdfg", "type": "Optional[int]", - "body": f"attr.ib(default=None)" + "body": "attr.ib(default=None)" }, "dict": { "name": "dict", "type": "Dict[str, int]", - "body": f"attr.ib()" + "body": "attr.ib()" + }, + "not": { + "name": "not_", + "type": "bool", + "body": f"attr.ib({field_meta('not')})" + }, + "1day": { + "name": "one_day", + "type": "int", + "body": f"attr.ib({field_meta('1day')})" + }, + "день_недели": { + "name": "den_nedeli", + "type": "str", + "body": f"attr.ib({field_meta('день_недели')})" } }, "generated": trim(f""" @@ -134,6 +152,9 @@ class Test: foo: int = attr.ib() qwerty: FloatString = attr.ib(converter=FloatString) dict: Dict[str, int] = attr.ib() + not_: bool = attr.ib({field_meta('not')}) + one_day: int = attr.ib({field_meta('1day')}) + den_nedeli: str = attr.ib({field_meta('день_недели')}) baz: Optional[List[List[str]]] = attr.ib(factory=list) bar: Optional[IntString] = attr.ib(default=None, converter=optional(IntString)) asdfg: Optional[int] = attr.ib(default=None) diff --git a/test/test_generator/test_detect_type.py b/test/test_generator/test_detect_type.py index 1e9fb4d..e1589c7 100644 --- a/test/test_generator/test_detect_type.py +++ b/test/test_generator/test_detect_type.py @@ -53,5 +53,5 @@ def test_convert(models_generator: MetadataGenerator): "another_dict_field_2": DDict(int), "another_dict_field_3": DDict(int), "int_field": int, - "not_": bool + "not": bool } From bfeef0a1f1541fd6e5e9f5fce9e9d55a883dba77 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Sun, 2 Dec 2018 18:46:17 +0300 Subject: [PATCH 2/2] Update TODO list --- TODO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index 80c648d..1a71360 100644 --- a/TODO.md +++ b/TODO.md @@ -22,7 +22,6 @@ - [ ] dataclasses - [ ] Decorator to mark class as exclude from models merge - Other features - - [ ] Decode unicode in keys - [ ] Nesting models generation - [X] Cascade (default) - [X] Flat @@ -38,6 +37,7 @@ - [X] ISO time - [X] ISO datetime - [X] Don't create metadata (J2M_ORIGINAL_FIELD) if original_field == generated_field + - [X] Decode unicode in keys - [X] Cli tool - Testing