From b5e54adab1edf962e76f9aedd640e144a4acccf7 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Thu, 4 Oct 2018 15:23:55 +0300 Subject: [PATCH 1/4] ISO date time parsing --- TODO.md | 12 +- rest_client_gen/dynamic_typing/__init__.py | 1 + .../dynamic_typing/string_datetime.py | 116 ++++++++++++++++++ .../dynamic_typing/string_serializable.py | 13 +- .../test_attrs_generation.py | 2 +- .../test_models_code_generator.py | 4 +- test/test_code_generation/test_typing.py | 4 +- testing_tools/real_apis/f1.py | 2 + testing_tools/real_apis/pathofexile.py | 1 + 9 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 rest_client_gen/dynamic_typing/string_datetime.py diff --git a/TODO.md b/TODO.md index 597c0fc..656cddf 100644 --- a/TODO.md +++ b/TODO.md @@ -32,10 +32,10 @@ - [ ] Complex python types annotations - [ ] Decorator to specify field metatype - [ ] Specify metatype in attr/dataclass argument (if dataclasses has such) - - [ ] String based types - - [ ] ISO date - - [ ] ISO time - - [ ] ISO datetime + - [ ] String based types (Warning: 6 times slow down) + - [X] ISO date + - [X] ISO time + - [X] ISO datetime - API Layer - [ ] Route object - [ ] Register model as route in/out data spec @@ -68,6 +68,10 @@ - Generate OpenAPI spec - [ ] Meta-model -> OpenAPI model converter - [ ] Route -> OpenAPI converter + - [ ] String based types + - [ ] ISO date + - [ ] ISO time + - [ ] ISO datetime - Build, Deploy, CI - [ ] setup.py diff --git a/rest_client_gen/dynamic_typing/__init__.py b/rest_client_gen/dynamic_typing/__init__.py index 8ef6129..9dd4f7a 100644 --- a/rest_client_gen/dynamic_typing/__init__.py +++ b/rest_client_gen/dynamic_typing/__init__.py @@ -3,6 +3,7 @@ ) from .complex import ComplexType, DList, DOptional, DTuple, DUnion, SingleType from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr +from .string_datetime import IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes from .string_serializable import ( BooleanString, FloatString, IntString, StringSerializable, StringSerializableRegistry, registry ) diff --git a/rest_client_gen/dynamic_typing/string_datetime.py b/rest_client_gen/dynamic_typing/string_datetime.py new file mode 100644 index 0000000..ea4357d --- /dev/null +++ b/rest_client_gen/dynamic_typing/string_datetime.py @@ -0,0 +1,116 @@ +import operator +from datetime import date, datetime, time +from typing import Optional + +import dateutil.parser + +from .string_serializable import StringSerializable, StringSerializableRegistry, registry + +_dt_args_getter = operator.attrgetter('year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond', 'tzinfo') +_d_args_getter = operator.attrgetter('year', 'month', 'day') +_t_args_getter = operator.attrgetter('hour', 'minute', 'second', 'microsecond', 'tzinfo') + + +def _extend_datetime(d, cls: type): + if isinstance(d, datetime): + args = _dt_args_getter + elif isinstance(d, time): + args = _t_args_getter + else: + args = _d_args_getter + return cls(*args(d)) + + +_check_values_date = ( + datetime(2018, 1, 2, 0, 4, 5, 678, tzinfo=None), + datetime(2018, 1, 2, 9, 4, 5, 678, tzinfo=None) +) + + +def is_date(s: str) -> Optional[date]: + d1 = dateutil.parser.parse(s, default=_check_values_date[0]) + d2 = dateutil.parser.parse(s, default=_check_values_date[1]) + return None if d1 == d2 else d1.date() + + +_check_values_time = ( + datetime(2018, 10, 11, 0, 4, 5, 678, tzinfo=None), + datetime(2018, 12, 30, 0, 4, 5, 678, tzinfo=None) +) + + +def is_time(s: str) -> Optional[time]: + d1 = dateutil.parser.parse(s, default=_check_values_time[0]) + d2 = dateutil.parser.parse(s, default=_check_values_time[1]) + return None if d1 == d2 else d1.time() + + +class IsoDateString(StringSerializable, date): + """ + Parse date using dateutil.parser.isoparse. Representation format always is ``YYYY-MM-DD``. + You can override to_representation method to customize it. Just don't forget to call registry.remove(YourCls) + """ + + @classmethod + def to_internal_value(cls, value: str) -> 'IsoDateString': + if not is_date(value): + raise ValueError(f"'{value}' is not valid date") + dt = dateutil.parser.isoparse(value) + return _extend_datetime(dt.date(), cls) + + def to_representation(self): + return self.isoformat() + + def replace(self, *args, **kwargs) -> 'IsoDateString': + # noinspection PyTypeChecker + return date.replace(self, *args, **kwargs) + + +class IsoTimeString(StringSerializable, time): + """ + Parse time using dateutil.parser.parse. Representation format always is ``hh:mm:ss.ms``. + You can override to_representation method to customize it. + """ + + @classmethod + def to_internal_value(cls, value: str) -> 'IsoTimeString': + t = is_time(value) + if not t: + raise ValueError(f"'{value}' is not valid time") + return _extend_datetime(t, cls) + + def to_representation(self): + return self.isoformat() + + def replace(self, *args, **kwargs) -> 'IsoTimeString': + # noinspection PyTypeChecker + return time.replace(self, *args, **kwargs) + + +class IsoDatetimeString(StringSerializable, datetime): + """ + Parse datetime using dateutil.parser.isoparse. + Representation format always is ``YYYY-MM-DDThh:mm:ss.ms`` (datetime.isoformat method). + """ + + @classmethod + def to_internal_value(cls, value: str) -> 'IsoDatetimeString': + dt = dateutil.parser.isoparse(value) + return _extend_datetime(dt, cls) + + def to_representation(self): + return self.isoformat() + + def replace(self, *args, **kwargs) -> 'IsoDatetimeString': + # noinspection PyTypeChecker + return datetime.replace(self, *args, **kwargs) + + +def register_datetime_classes(registry: StringSerializableRegistry = registry): + """ + Register datetime classes in given registry (using default registry if no arguments is passed). + Date parsing is expensive operation so this classes are disabled by default + """ + registry.add(cls=IsoDateString) + registry.add(cls=IsoTimeString) + registry.add(cls=IsoDatetimeString) diff --git a/rest_client_gen/dynamic_typing/string_serializable.py b/rest_client_gen/dynamic_typing/string_serializable.py index 7be6de2..88f79bf 100644 --- a/rest_client_gen/dynamic_typing/string_serializable.py +++ b/rest_client_gen/dynamic_typing/string_serializable.py @@ -34,7 +34,7 @@ def to_typing_code(cls) -> Tuple[ImportPathList, str]: as a metadata instance but contains actual data """ cls_name = cls.__name__ - return [('rest_client_gen.dynamic_typing.string_serializable', cls_name)], cls_name + return [('rest_client_gen.dynamic_typing', cls_name)], cls_name T_StringSerializable = Type[StringSerializable] @@ -71,6 +71,17 @@ def decorator(cls): return decorator + def remove(self, cls: type): + """ + Unregister given class + + :param cls: StringSerializable class + """ + self.types.remove(cls) + for base, replace in list(self.replaces): + if replace is cls: + self.replaces.remove((base, replace)) + def resolve(self, *types: T_StringSerializable) -> Collection[T_StringSerializable]: """ Return set of StringSerializable classes which can represent all classes from types argument. diff --git a/test/test_code_generation/test_attrs_generation.py b/test/test_code_generation/test_attrs_generation.py index 0d0988a..1bc13a7 100644 --- a/test/test_code_generation/test_attrs_generation.py +++ b/test/test_code_generation/test_attrs_generation.py @@ -119,7 +119,7 @@ class Test: "generated": trim(f""" import attr from attr.converter import optional - from rest_client_gen.dynamic_typing.string_serializable import FloatString, IntString + from rest_client_gen.dynamic_typing import FloatString, IntString from typing import List, Optional diff --git a/test/test_code_generation/test_models_code_generator.py b/test/test_code_generation/test_models_code_generator.py index 4807dd7..40b06a0 100644 --- a/test/test_code_generation/test_models_code_generator.py +++ b/test/test_code_generation/test_models_code_generator.py @@ -118,7 +118,7 @@ class Test: } }, "fields": { - "imports": "from rest_client_gen.dynamic_typing.string_serializable import IntString\n" + "imports": "from rest_client_gen.dynamic_typing import IntString\n" "from typing import List, Optional", "fields": [ "foo: int", @@ -127,7 +127,7 @@ class Test: ] }, "generated": trim(""" - from rest_client_gen.dynamic_typing.string_serializable import IntString + from rest_client_gen.dynamic_typing import IntString from typing import List, Optional diff --git a/test/test_code_generation/test_typing.py b/test/test_code_generation/test_typing.py index 41eda00..3cf0b6f 100644 --- a/test/test_code_generation/test_typing.py +++ b/test/test_code_generation/test_typing.py @@ -104,12 +104,12 @@ class TestModel: ), pytest.param( FloatString, - ('from rest_client_gen.dynamic_typing.string_serializable import FloatString', FloatString), + ('from rest_client_gen.dynamic_typing import FloatString', FloatString), id="string_serializable" ), pytest.param( DOptional(IntString), - ('from rest_client_gen.dynamic_typing.string_serializable import IntString\n' + ('from rest_client_gen.dynamic_typing import IntString\n' 'from typing import Optional', Optional[IntString]), id="complex_string_serializable" ), diff --git a/testing_tools/real_apis/f1.py b/testing_tools/real_apis/f1.py index cb88752..34bbef0 100644 --- a/testing_tools/real_apis/f1.py +++ b/testing_tools/real_apis/f1.py @@ -5,6 +5,7 @@ import inflection import requests +from rest_client_gen.dynamic_typing import register_datetime_classes from rest_client_gen.generator import MetadataGenerator from rest_client_gen.models import compose_models from rest_client_gen.models.attr import AttrsModelCodeGenerator @@ -43,6 +44,7 @@ def main(): dump_response("f1", "driver_standings", driver_standings_data) driver_standings_data = ("driver_standings", driver_standings_data) + register_datetime_classes() gen = MetadataGenerator() reg = ModelRegistry() for name, data in (results_data, drivers_data, driver_standings_data): diff --git a/testing_tools/real_apis/pathofexile.py b/testing_tools/real_apis/pathofexile.py index 1c1e460..517cf49 100644 --- a/testing_tools/real_apis/pathofexile.py +++ b/testing_tools/real_apis/pathofexile.py @@ -26,6 +26,7 @@ def main(): print(f"Start model generation (data len = {len(tabs)})") start_t = datetime.now() + # register_datetime_classes() gen = MetadataGenerator() reg = ModelRegistry() fields = gen.generate(*tabs) From 7de12d76284be933dc7ef98d11374db5f2a4ae0c Mon Sep 17 00:00:00 2001 From: bogdandm Date: Fri, 5 Oct 2018 15:55:34 +0300 Subject: [PATCH 2/4] Docstrings --- .../dynamic_typing/string_datetime.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/rest_client_gen/dynamic_typing/string_datetime.py b/rest_client_gen/dynamic_typing/string_datetime.py index ea4357d..2701006 100644 --- a/rest_client_gen/dynamic_typing/string_datetime.py +++ b/rest_client_gen/dynamic_typing/string_datetime.py @@ -1,6 +1,6 @@ import operator from datetime import date, datetime, time -from typing import Optional +from typing import Any, Optional, Type, Union import dateutil.parser @@ -11,7 +11,14 @@ _t_args_getter = operator.attrgetter('hour', 'minute', 'second', 'microsecond', 'tzinfo') -def _extend_datetime(d, cls: type): +def extend_datetime(d: Union[date, time, datetime], cls: Union[Type[date], Type[time], Type[datetime]]) -> Any: + """ + Wrap datetime object into datetime subclass + + :param d: date/time/datetime instance + :param cls: datetime subclass + :return: + """ if isinstance(d, datetime): args = _dt_args_getter elif isinstance(d, time): @@ -28,6 +35,14 @@ def _extend_datetime(d, cls: type): def is_date(s: str) -> Optional[date]: + """ + Return date instance if given string is a date and None otherwise + + :param s: string + :return: date or None + """ + # dateutil.parser.parse replaces missing parts of datetime with values from default value + # so if there is hour part in given string then d1 and d2 would be equal and string is not pure date d1 = dateutil.parser.parse(s, default=_check_values_date[0]) d2 = dateutil.parser.parse(s, default=_check_values_date[1]) return None if d1 == d2 else d1.date() @@ -40,6 +55,12 @@ def is_date(s: str) -> Optional[date]: def is_time(s: str) -> Optional[time]: + """ + Return time instance if given string is a time and None otherwise + + :param s: string + :return: time or None + """ d1 = dateutil.parser.parse(s, default=_check_values_time[0]) d2 = dateutil.parser.parse(s, default=_check_values_time[1]) return None if d1 == d2 else d1.time() @@ -48,7 +69,7 @@ def is_time(s: str) -> Optional[time]: class IsoDateString(StringSerializable, date): """ Parse date using dateutil.parser.isoparse. Representation format always is ``YYYY-MM-DD``. - You can override to_representation method to customize it. Just don't forget to call registry.remove(YourCls) + You can override to_representation method to customize it. Just don't forget to call registry.remove(IsoDateString) """ @classmethod @@ -56,7 +77,7 @@ def to_internal_value(cls, value: str) -> 'IsoDateString': if not is_date(value): raise ValueError(f"'{value}' is not valid date") dt = dateutil.parser.isoparse(value) - return _extend_datetime(dt.date(), cls) + return extend_datetime(dt.date(), cls) def to_representation(self): return self.isoformat() @@ -77,7 +98,7 @@ def to_internal_value(cls, value: str) -> 'IsoTimeString': t = is_time(value) if not t: raise ValueError(f"'{value}' is not valid time") - return _extend_datetime(t, cls) + return extend_datetime(t, cls) def to_representation(self): return self.isoformat() @@ -96,7 +117,7 @@ class IsoDatetimeString(StringSerializable, datetime): @classmethod def to_internal_value(cls, value: str) -> 'IsoDatetimeString': dt = dateutil.parser.isoparse(value) - return _extend_datetime(dt, cls) + return extend_datetime(dt, cls) def to_representation(self): return self.isoformat() From ebd7fd4038b6cca09ba28bbb8653f5589d7c94a4 Mon Sep 17 00:00:00 2001 From: bogdandm Date: Fri, 5 Oct 2018 16:45:23 +0300 Subject: [PATCH 3/4] Tests --- .../dynamic_typing/string_datetime.py | 4 +- .../test_string_datetime.py | 94 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 test/test_dynamic_typing/test_string_datetime.py diff --git a/rest_client_gen/dynamic_typing/string_datetime.py b/rest_client_gen/dynamic_typing/string_datetime.py index 2701006..19db008 100644 --- a/rest_client_gen/dynamic_typing/string_datetime.py +++ b/rest_client_gen/dynamic_typing/string_datetime.py @@ -49,8 +49,8 @@ def is_date(s: str) -> Optional[date]: _check_values_time = ( - datetime(2018, 10, 11, 0, 4, 5, 678, tzinfo=None), - datetime(2018, 12, 30, 0, 4, 5, 678, tzinfo=None) + datetime(2018, 10, 11), + datetime(2018, 12, 30) ) diff --git a/test/test_dynamic_typing/test_string_datetime.py b/test/test_dynamic_typing/test_string_datetime.py new file mode 100644 index 0000000..594d15b --- /dev/null +++ b/test/test_dynamic_typing/test_string_datetime.py @@ -0,0 +1,94 @@ +import datetime + +import pytest + +from rest_client_gen.dynamic_typing import ( + FloatString, IntString, IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes +) +from rest_client_gen.generator import MetadataGenerator + +register_datetime_classes() + +test_detect_type_data = [ + # Check that string datetime doesn't break default string types + pytest.param( + "1", + IntString, + id="default_check_int" + ), + pytest.param( + "1.5", + FloatString, + id="default_check_float" + ), + pytest.param( + "2018-12-31", + IsoDateString, + id="date" + ), + pytest.param( + "12:58", + IsoTimeString, + id="time" + ), + pytest.param( + "2018-12-31T12:58:12Z", + IsoDatetimeString, + id="datetime" + ) +] + + +@pytest.mark.parametrize("value,expected", test_detect_type_data) +def test_detect_type(models_generator: MetadataGenerator, value, expected): + result = models_generator._detect_type(value) + assert result == expected + + +test_parse_data = [ + pytest.param( + "2018-12-31", + IsoDateString(2018, 12, 31), + id="date" + ), + pytest.param( + "12:13", + IsoTimeString(12, 13), + id="time" + ), + pytest.param( + "04:15:34", + IsoTimeString(4, 15, 34), + id="time_seconds" + ), + pytest.param( + "04:15:34.034", + IsoTimeString(4, 15, 34, 34000), + id="time_ms" + ), + pytest.param( + "2018-12-04T04:15:34.034000+00:00", + IsoDatetimeString(2018, 12, 4, 4, 15, 34, 34000, tzinfo=datetime.timezone.utc), + id="datetime_full" + ), + pytest.param( + "2018-12-04T04:15", + IsoDatetimeString(2018, 12, 4, 4, 15), + id="datetime_partial" + ) +] + + +@pytest.mark.parametrize("value,expected", test_parse_data) +def test_parse(models_generator: MetadataGenerator, value, expected): + cls = models_generator._detect_type(value) + result = cls.to_internal_value(value) + assert result == expected + assert value in result.to_representation() + + +def test_replace(): + assert IsoTimeString(14, 12, 57).replace(minute=58, second=32) == IsoTimeString(14, 58, 32) + assert IsoDateString(2014, 12, 5).replace(day=4, month=5) == IsoDateString(2014, 5, 4) + assert IsoDatetimeString(2014, 12, 5, 14, 12, 57).replace(minute=58, second=32, day=4, month=5) \ + == IsoDatetimeString(2014, 5, 4, 14, 58, 32) From 44007a465159b24bc3a401516ddc0f5e0af781ed Mon Sep 17 00:00:00 2001 From: bogdandm Date: Fri, 5 Oct 2018 17:10:47 +0300 Subject: [PATCH 4/4] Test StringSerializableRegistry.remove --- .../dynamic_typing/string_serializable.py | 2 +- .../test_string_serializable_registry.py | 30 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/rest_client_gen/dynamic_typing/string_serializable.py b/rest_client_gen/dynamic_typing/string_serializable.py index 88f79bf..bcb5d69 100644 --- a/rest_client_gen/dynamic_typing/string_serializable.py +++ b/rest_client_gen/dynamic_typing/string_serializable.py @@ -79,7 +79,7 @@ def remove(self, cls: type): """ self.types.remove(cls) for base, replace in list(self.replaces): - if replace is cls: + if replace is cls or base is cls: self.replaces.remove((base, replace)) def resolve(self, *types: T_StringSerializable) -> Collection[T_StringSerializable]: diff --git a/test/test_dynamic_typing/test_string_serializable_registry.py b/test/test_dynamic_typing/test_string_serializable_registry.py index b405af7..b221958 100644 --- a/test/test_dynamic_typing/test_string_serializable_registry.py +++ b/test/test_dynamic_typing/test_string_serializable_registry.py @@ -1,6 +1,9 @@ import pytest -from rest_client_gen.dynamic_typing.string_serializable import StringSerializable, StringSerializableRegistry +from rest_client_gen.dynamic_typing import IsoTimeString +from rest_client_gen.dynamic_typing.string_serializable import (FloatString, IntString, StringSerializable, + StringSerializableRegistry) +from rest_client_gen.generator import MetadataGenerator r = StringSerializableRegistry() @@ -54,7 +57,30 @@ class Y(StringSerializable): pytest.param((X, B), {B, X}), ] + @pytest.mark.parametrize("value,expected", test_data) def test_string_serializable_registry(value, expected): result = r.resolve(*value) - assert result == expected \ No newline at end of file + assert result == expected + + +r2 = StringSerializableRegistry() +gen = MetadataGenerator(r2) + +r2.add(cls=IsoTimeString) +r2.add(cls=IntString) +r2.add(replace_types=(IntString,), cls=FloatString) + + +@pytest.mark.xfail() +def test_without_remove(): + assert gen._detect_type("12") == IntString + + +r2.remove(IsoTimeString) +r2.add(cls=IsoTimeString) + + +def test_remove(): + assert gen._detect_type("12") == IntString + assert gen._detect_type("12:14") == IsoTimeString