From 49d6100183fbdf790be6c8c9f97bf9ceb429b19f Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 22 Jul 2019 16:39:44 +0300 Subject: [PATCH 01/25] API Core: simplify methods Two similar methods combined into one. --- api_core/google/api_core/datetime_helpers.py | 70 +++++++++++--------- api_core/tests/unit/test_datetime_helpers.py | 24 ++----- 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 84c1bb7f512c..be7e8ebbafae 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -17,6 +17,7 @@ import calendar import datetime import re +import warnings import pytz @@ -115,58 +116,67 @@ def from_iso8601_time(value): def from_rfc3339(value): - """Convert a microsecond-precision timestamp to datetime. + """Convert a nanosecond-precision or if nanoseconds are missing + microsecond-precision timestamp to a native datetime. + + .. note:: + Python datetimes do not support nanosecond precision; this function + therefore truncates such values to microseconds. Args: - value (str): The RFC3339 string to convert. + value (str): The RFC 3339 string to convert. Returns: - datetime.datetime: The datetime object equivalent to the timestamp in - UTC. + datetime.datetime: The datetime object equivalent to the timestamp + in UTC. + """ - return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + with_nanos = _RFC3339_NANOS.match(value) + + if with_nanos is None: + return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + else: + bare_seconds = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") + + if fraction is None: + micros = 0 + else: + scale = 9 - len(fraction) + nanos = int(fraction) * (10 ** scale) + micros = nanos // 1000 + + return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) def from_rfc3339_nanos(value): """Convert a nanosecond-precision timestamp to a native datetime. .. note:: - Python datetimes do not support nanosecond precision; this function - therefore truncates such values to microseconds. + Python datetimes do not support nanosecond precision; this + function therefore truncates such values to microseconds. Args: value (str): The RFC3339 string to convert. Returns: - datetime.datetime: The datetime object equivalent to the timestamp in - UTC. + datetime.datetime: The datetime object equivalent to the + timestamp in UTC. Raises: ValueError: If the timestamp does not match the RFC 3339 regular expression. """ - with_nanos = _RFC3339_NANOS.match(value) - - if with_nanos is None: - raise ValueError( - "Timestamp: {!r}, does not match pattern: {!r}".format( - value, _RFC3339_NANOS.pattern - ) - ) - - bare_seconds = datetime.datetime.strptime( - with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + # Raise deprecation warnings for things we want to go away. + warnings.warn( + "The `from_rfc3339_nanos` method will be deprecated in" + "future versions; use `from_rfc3339` instead.", + PendingDeprecationWarning, + stacklevel=2, ) - fraction = with_nanos.group("nanos") - - if fraction is None: - micros = 0 - else: - scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) - micros = nanos // 1000 - - return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + return from_rfc3339(value) def to_rfc3339(value, ignore_zone=True): diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index 4d138c88a80c..fb615a5c872d 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -83,33 +83,19 @@ def test_from_rfc3339(): def test_from_rfc3339_with_bad_tz(): - value = "2009-12-17T12:44:32.123456BAD" - - with pytest.raises(ValueError): - datetime_helpers.from_rfc3339(value) - - -def test_from_rfc3339_with_nanos(): - value = "2009-12-17T12:44:32.123456789Z" + value = "2009-12-17T12:44:32.123456789BAD" with pytest.raises(ValueError): datetime_helpers.from_rfc3339(value) -def test_from_rfc3339_nanos_without_nanos(): +def test_from_rfc3339_without_nanos(): value = "2009-12-17T12:44:32Z" - assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + assert datetime_helpers.from_rfc3339(value) == datetime.datetime( 2009, 12, 17, 12, 44, 32, 0, pytz.utc ) -def test_from_rfc3339_nanos_with_bad_tz(): - value = "2009-12-17T12:44:32.123456789BAD" - - with pytest.raises(ValueError): - datetime_helpers.from_rfc3339_nanos(value) - - @pytest.mark.parametrize( "truncated, micros", [ @@ -123,9 +109,9 @@ def test_from_rfc3339_nanos_with_bad_tz(): ("1", 100000), ], ) -def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): +def test_from_rfc3339_with_truncated_nanos(truncated, micros): value = "2009-12-17T12:44:32.{}Z".format(truncated) - assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + assert datetime_helpers.from_rfc3339(value) == datetime.datetime( 2009, 12, 17, 12, 44, 32, micros, pytz.utc ) From ddbcb99573431b00b7dc1de361f4fd73cdd40d96 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 24 Jul 2019 12:25:22 +0300 Subject: [PATCH 02/25] Update test_datetime_helpers.py Re-write tests for the 'from_rfc3339_nanos' method. --- api_core/tests/unit/test_datetime_helpers.py | 37 +++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index fb615a5c872d..e06daf88bfcb 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -82,11 +82,11 @@ def test_from_rfc3339(): ) -def test_from_rfc3339_with_bad_tz(): - value = "2009-12-17T12:44:32.123456789BAD" - - with pytest.raises(ValueError): - datetime_helpers.from_rfc3339(value) +def test_from_rfc3339_nanos(): + value = "2009-12-17T12:44:32.123456Z" + assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + 2009, 12, 17, 12, 44, 32, 123456, pytz.utc + ) def test_from_rfc3339_without_nanos(): @@ -96,6 +96,13 @@ def test_from_rfc3339_without_nanos(): ) +def test_from_rfc3339_nanos_without_nanos(): + value = "2009-12-17T12:44:32Z" + assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + 2009, 12, 17, 12, 44, 32, 0, pytz.utc + ) + + @pytest.mark.parametrize( "truncated, micros", [ @@ -116,6 +123,26 @@ def test_from_rfc3339_with_truncated_nanos(truncated, micros): ) +@pytest.mark.parametrize( + "truncated, micros", + [ + ("12345678", 123456), + ("1234567", 123456), + ("123456", 123456), + ("12345", 123450), + ("1234", 123400), + ("123", 123000), + ("12", 120000), + ("1", 100000), + ], +) +def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): + value = "2009-12-17T12:44:32.{}Z".format(truncated) + assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + 2009, 12, 17, 12, 44, 32, micros, pytz.utc + ) + + def test_to_rfc3339(): value = datetime.datetime(2016, 4, 5, 13, 30, 0) expected = "2016-04-05T13:30:00.000000Z" From 1cfaf618f76e654c8aabc081cbbe538aeed128dd Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 25 Jul 2019 13:13:25 +0300 Subject: [PATCH 03/25] Update datetime_helpers.py Fix for coverage and lint failures. --- api_core/google/api_core/datetime_helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index be7e8ebbafae..c01c9c067af7 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -131,11 +131,11 @@ def from_rfc3339(value): in UTC. """ - with_nanos = _RFC3339_NANOS.match(value) - - if with_nanos is None: + try: return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) - else: + except ValueError: + with_nanos = _RFC3339_NANOS.match(value) + bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) @@ -155,14 +155,14 @@ def from_rfc3339_nanos(value): """Convert a nanosecond-precision timestamp to a native datetime. .. note:: - Python datetimes do not support nanosecond precision; this + Python datetimes do not support nanosecond precision; this function therefore truncates such values to microseconds. Args: value (str): The RFC3339 string to convert. Returns: - datetime.datetime: The datetime object equivalent to the + datetime.datetime: The datetime object equivalent to the timestamp in UTC. Raises: @@ -172,7 +172,7 @@ def from_rfc3339_nanos(value): # Raise deprecation warnings for things we want to go away. warnings.warn( "The `from_rfc3339_nanos` method will be deprecated in" - "future versions; use `from_rfc3339` instead.", + " future versions; use `from_rfc3339` instead.", PendingDeprecationWarning, stacklevel=2, ) From 59a49b944751dace4e3a0096b84acfb9378a6832 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 26 Jul 2019 10:22:47 +0300 Subject: [PATCH 04/25] Update datetime_helpers.py restore previous version with testing regex first + cosmetic chgs for the descriptions --- api_core/google/api_core/datetime_helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index c01c9c067af7..476110668658 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -131,11 +131,11 @@ def from_rfc3339(value): in UTC. """ - try: - return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) - except ValueError: - with_nanos = _RFC3339_NANOS.match(value) + with_nanos = _RFC3339_NANOS.match(value) + if with_nanos is None: + return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + else: bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) @@ -152,7 +152,7 @@ def from_rfc3339(value): def from_rfc3339_nanos(value): - """Convert a nanosecond-precision timestamp to a native datetime. + """DEPRECATED. Convert a nanosecond-precision timestamp to a native datetime. .. note:: Python datetimes do not support nanosecond precision; this @@ -171,8 +171,8 @@ def from_rfc3339_nanos(value): """ # Raise deprecation warnings for things we want to go away. warnings.warn( - "The `from_rfc3339_nanos` method will be deprecated in" - " future versions; use `from_rfc3339` instead.", + "The `from_rfc3339_nanos` function is deprecated" + " use `from_rfc3339` instead.", PendingDeprecationWarning, stacklevel=2, ) From 0481ad45b311573a221cd52d34d9c14e1a672432 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 26 Jul 2019 15:07:35 +0300 Subject: [PATCH 05/25] Update test_datetime_helpers.py added test to check the deprecation warning --- api_core/tests/unit/test_datetime_helpers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index e06daf88bfcb..c1096cc914a6 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -15,6 +15,7 @@ import calendar import datetime +import mock import pytest import pytz @@ -23,6 +24,8 @@ ONE_MINUTE_IN_MICROSECONDS = 60 * 1e6 +MESSAGE = ("The `from_rfc3339_nanos` function is deprecated" + " use `from_rfc3339` instead.") def test_utcnow(): @@ -123,6 +126,19 @@ def test_from_rfc3339_with_truncated_nanos(truncated, micros): ) +def test_from_rfc3339_nanos_is_deprecated(): + from_rfc3339_patch = mock.patch("google.api_core.datetime_helpers.from_rfc3339") + warnings_patch = mock.patch("warnings.warn") + value = "2009-12-17T12:44:32.123456Z" + + with from_rfc3339_patch as from_rfc3339, warnings_patch as warn: + result = datetime_helpers.from_rfc3339_nanos(value) + + assert result is from_rfc3339.return_value + from_rfc3339.assert_called_once_with(value) + warn.assert_called_once_with(MESSAGE, DeprecationWarning, stacklevel=2) + + @pytest.mark.parametrize( "truncated, micros", [ From 710978bf70a4c7e6a374d7080b8bc904fa1dd660 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 26 Jul 2019 15:09:28 +0300 Subject: [PATCH 06/25] Update datetime_helpers.py adjusted the condition for the 'from_rfc3339' method + warning fix --- api_core/google/api_core/datetime_helpers.py | 26 +++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 476110668658..4368df89c3b5 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -132,23 +132,19 @@ def from_rfc3339(value): """ with_nanos = _RFC3339_NANOS.match(value) + bare_seconds = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") - if with_nanos is None: - return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + if fraction is None: + micros = 0 else: - bare_seconds = datetime.datetime.strptime( - with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION - ) - fraction = with_nanos.group("nanos") - - if fraction is None: - micros = 0 - else: - scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) - micros = nanos // 1000 + scale = 9 - len(fraction) + nanos = int(fraction) * (10 ** scale) + micros = nanos // 1000 - return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) def from_rfc3339_nanos(value): @@ -173,7 +169,7 @@ def from_rfc3339_nanos(value): warnings.warn( "The `from_rfc3339_nanos` function is deprecated" " use `from_rfc3339` instead.", - PendingDeprecationWarning, + DeprecationWarning, stacklevel=2, ) return from_rfc3339(value) From 70bf8bcafee5ad64a1a6268ef2177324d3c79071 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 10:56:38 +0300 Subject: [PATCH 07/25] Update datetime_helpers.py * deleted extra spaces --- api_core/google/api_core/datetime_helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 4368df89c3b5..310240f4d4b7 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -124,7 +124,7 @@ def from_rfc3339(value): therefore truncates such values to microseconds. Args: - value (str): The RFC 3339 string to convert. + value (str): The RFC3339 string to convert. Returns: datetime.datetime: The datetime object equivalent to the timestamp @@ -162,7 +162,7 @@ def from_rfc3339_nanos(value): timestamp in UTC. Raises: - ValueError: If the timestamp does not match the RFC 3339 + ValueError: If the timestamp does not match the RFC3339 regular expression. """ # Raise deprecation warnings for things we want to go away. @@ -221,10 +221,10 @@ def nanosecond(self): return self._nanosecond def rfc3339(self): - """Return an RFC 3339-compliant timestamp. + """Return an RFC3339-compliant timestamp. Returns: - (str): Timestamp string according to RFC 3339 spec. + (str): Timestamp string according to RFC3339 spec. """ if self._nanosecond == 0: return to_rfc3339(self) @@ -233,10 +233,10 @@ def rfc3339(self): @classmethod def from_rfc3339(cls, stamp): - """Parse RFC 3339-compliant timestamp, preserving nanoseconds. + """Parse RFC3339-compliant timestamp, preserving nanoseconds. Args: - stamp (str): RFC 3339 stamp, with up to nanosecond precision + stamp (str): RFC3339 stamp, with up to nanosecond precision Returns: :class:`DatetimeWithNanoseconds`: @@ -286,7 +286,7 @@ def timestamp_pb(self): @classmethod def from_timestamp_pb(cls, stamp): - """Parse RFC 3339-compliant timestamp, preserving nanoseconds. + """Parse RFC3339-compliant timestamp, preserving nanoseconds. Args: stamp (:class:`~google.protobuf.timestamp_pb2.Timestamp`): timestamp message From 06bee67b097e485a58415d218f3ac421a04d3d6a Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 5 Sep 2019 20:52:35 +0300 Subject: [PATCH 08/25] update datetime_helpers * 'if-else' statement restored to throw an exception * wrote additional tests to cover the exception --- api_core/google/api_core/datetime_helpers.py | 10 ++++- api_core/tests/unit/test_datetime_helpers.py | 43 ++++++++++++++------ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 310240f4d4b7..9bc6a910974b 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -132,6 +132,14 @@ def from_rfc3339(value): """ with_nanos = _RFC3339_NANOS.match(value) + + if with_nanos is None: + raise ValueError( + "Timestamp: {!r}, does not match pattern: {!r}".format( + value, _RFC3339_NANOS.pattern + ) + ) + bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) @@ -228,7 +236,7 @@ def rfc3339(self): """ if self._nanosecond == 0: return to_rfc3339(self) - nanos = str(self._nanosecond).rjust(9, '0').rstrip("0") + nanos = str(self._nanosecond).rjust(9, "0").rstrip("0") return "{}.{}Z".format(self.strftime(_RFC3339_NO_FRACTION), nanos) @classmethod diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index c1096cc914a6..948723f96ead 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -24,8 +24,9 @@ ONE_MINUTE_IN_MICROSECONDS = 60 * 1e6 -MESSAGE = ("The `from_rfc3339_nanos` function is deprecated" - " use `from_rfc3339` instead.") +MESSAGE = ( + "The `from_rfc3339_nanos` function is deprecated" " use `from_rfc3339` instead." +) def test_utcnow(): @@ -159,6 +160,18 @@ def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): ) +def test_from_rfc3339_wo_nanos_raise_exception(): + value = "2009-12-17T12:44:32" + with pytest.raises(ValueError): + datetime_helpers.from_rfc3339(value) + + +def test_from_rfc3339_w_nanos_raise_exception(): + value = "2009-12-17T12:44:32.123456" + with pytest.raises(ValueError): + datetime_helpers.from_rfc3339(value) + + def test_to_rfc3339(): value = datetime.datetime(2016, 4, 5, 13, 30, 0) expected = "2016-04-05T13:30:00.000000Z" @@ -186,10 +199,11 @@ def test_to_rfc3339_with_non_utc_ignore_zone(): class Test_DateTimeWithNanos(object): - @staticmethod def test_ctor_wo_nanos(): - stamp = datetime_helpers.DatetimeWithNanoseconds(2016, 12, 20, 21, 13, 47, 123456) + stamp = datetime_helpers.DatetimeWithNanoseconds( + 2016, 12, 20, 21, 13, 47, 123456 + ) assert stamp.year == 2016 assert stamp.month == 12 assert stamp.day == 20 @@ -229,7 +243,9 @@ def test_ctor_w_micros_keyword_and_nanos(): @staticmethod def test_rfc3339_wo_nanos(): - stamp = datetime_helpers.DatetimeWithNanoseconds(2016, 12, 20, 21, 13, 47, 123456) + stamp = datetime_helpers.DatetimeWithNanoseconds( + 2016, 12, 20, 21, 13, 47, 123456 + ) assert stamp.rfc3339() == "2016-12-20T21:13:47.123456Z" @staticmethod @@ -314,12 +330,16 @@ def test_from_rfc3339_w_full_precision(): ) def test_from_rfc3339_test_nanoseconds(fractional, nanos): value = "2009-12-17T12:44:32.{}Z".format(fractional) - assert datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(value).nanosecond == nanos + assert ( + datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(value).nanosecond + == nanos + ) @staticmethod def test_timestamp_pb_wo_nanos_naive(): stamp = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, 123456) + 2016, 12, 20, 21, 13, 47, 123456 + ) delta = stamp.replace(tzinfo=pytz.UTC) - datetime_helpers._UTC_EPOCH seconds = int(delta.total_seconds()) nanos = 123456000 @@ -333,7 +353,8 @@ def test_timestamp_pb_w_nanos(): ) delta = stamp - datetime_helpers._UTC_EPOCH timestamp = timestamp_pb2.Timestamp( - seconds=int(delta.total_seconds()), nanos=123456789) + seconds=int(delta.total_seconds()), nanos=123456789 + ) assert stamp.timestamp_pb() == timestamp @staticmethod @@ -343,8 +364,7 @@ def test_from_timestamp_pb_wo_nanos(): seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds) - stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb( - timestamp) + stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb(timestamp) assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 0 @@ -358,8 +378,7 @@ def test_from_timestamp_pb_w_nanos(): seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=123456789) - stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb( - timestamp) + stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb(timestamp) assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 123456 From 873a60dbc64b4ae3ef585f289d25eade0369ea72 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 22 Jul 2019 16:39:44 +0300 Subject: [PATCH 09/25] API Core: simplify methods Two similar methods combined into one. --- api_core/google/api_core/datetime_helpers.py | 70 +++++++++++--------- api_core/tests/unit/test_datetime_helpers.py | 24 ++----- 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 84c1bb7f512c..be7e8ebbafae 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -17,6 +17,7 @@ import calendar import datetime import re +import warnings import pytz @@ -115,58 +116,67 @@ def from_iso8601_time(value): def from_rfc3339(value): - """Convert a microsecond-precision timestamp to datetime. + """Convert a nanosecond-precision or if nanoseconds are missing + microsecond-precision timestamp to a native datetime. + + .. note:: + Python datetimes do not support nanosecond precision; this function + therefore truncates such values to microseconds. Args: - value (str): The RFC3339 string to convert. + value (str): The RFC 3339 string to convert. Returns: - datetime.datetime: The datetime object equivalent to the timestamp in - UTC. + datetime.datetime: The datetime object equivalent to the timestamp + in UTC. + """ - return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + with_nanos = _RFC3339_NANOS.match(value) + + if with_nanos is None: + return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + else: + bare_seconds = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") + + if fraction is None: + micros = 0 + else: + scale = 9 - len(fraction) + nanos = int(fraction) * (10 ** scale) + micros = nanos // 1000 + + return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) def from_rfc3339_nanos(value): """Convert a nanosecond-precision timestamp to a native datetime. .. note:: - Python datetimes do not support nanosecond precision; this function - therefore truncates such values to microseconds. + Python datetimes do not support nanosecond precision; this + function therefore truncates such values to microseconds. Args: value (str): The RFC3339 string to convert. Returns: - datetime.datetime: The datetime object equivalent to the timestamp in - UTC. + datetime.datetime: The datetime object equivalent to the + timestamp in UTC. Raises: ValueError: If the timestamp does not match the RFC 3339 regular expression. """ - with_nanos = _RFC3339_NANOS.match(value) - - if with_nanos is None: - raise ValueError( - "Timestamp: {!r}, does not match pattern: {!r}".format( - value, _RFC3339_NANOS.pattern - ) - ) - - bare_seconds = datetime.datetime.strptime( - with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + # Raise deprecation warnings for things we want to go away. + warnings.warn( + "The `from_rfc3339_nanos` method will be deprecated in" + "future versions; use `from_rfc3339` instead.", + PendingDeprecationWarning, + stacklevel=2, ) - fraction = with_nanos.group("nanos") - - if fraction is None: - micros = 0 - else: - scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) - micros = nanos // 1000 - - return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + return from_rfc3339(value) def to_rfc3339(value, ignore_zone=True): diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index 4d138c88a80c..fb615a5c872d 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -83,33 +83,19 @@ def test_from_rfc3339(): def test_from_rfc3339_with_bad_tz(): - value = "2009-12-17T12:44:32.123456BAD" - - with pytest.raises(ValueError): - datetime_helpers.from_rfc3339(value) - - -def test_from_rfc3339_with_nanos(): - value = "2009-12-17T12:44:32.123456789Z" + value = "2009-12-17T12:44:32.123456789BAD" with pytest.raises(ValueError): datetime_helpers.from_rfc3339(value) -def test_from_rfc3339_nanos_without_nanos(): +def test_from_rfc3339_without_nanos(): value = "2009-12-17T12:44:32Z" - assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + assert datetime_helpers.from_rfc3339(value) == datetime.datetime( 2009, 12, 17, 12, 44, 32, 0, pytz.utc ) -def test_from_rfc3339_nanos_with_bad_tz(): - value = "2009-12-17T12:44:32.123456789BAD" - - with pytest.raises(ValueError): - datetime_helpers.from_rfc3339_nanos(value) - - @pytest.mark.parametrize( "truncated, micros", [ @@ -123,9 +109,9 @@ def test_from_rfc3339_nanos_with_bad_tz(): ("1", 100000), ], ) -def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): +def test_from_rfc3339_with_truncated_nanos(truncated, micros): value = "2009-12-17T12:44:32.{}Z".format(truncated) - assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + assert datetime_helpers.from_rfc3339(value) == datetime.datetime( 2009, 12, 17, 12, 44, 32, micros, pytz.utc ) From 796204cb3a8ce747b180e0fbd7bd54408013f037 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Wed, 24 Jul 2019 12:25:22 +0300 Subject: [PATCH 10/25] Update test_datetime_helpers.py Re-write tests for the 'from_rfc3339_nanos' method. --- api_core/tests/unit/test_datetime_helpers.py | 37 +++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index fb615a5c872d..e06daf88bfcb 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -82,11 +82,11 @@ def test_from_rfc3339(): ) -def test_from_rfc3339_with_bad_tz(): - value = "2009-12-17T12:44:32.123456789BAD" - - with pytest.raises(ValueError): - datetime_helpers.from_rfc3339(value) +def test_from_rfc3339_nanos(): + value = "2009-12-17T12:44:32.123456Z" + assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + 2009, 12, 17, 12, 44, 32, 123456, pytz.utc + ) def test_from_rfc3339_without_nanos(): @@ -96,6 +96,13 @@ def test_from_rfc3339_without_nanos(): ) +def test_from_rfc3339_nanos_without_nanos(): + value = "2009-12-17T12:44:32Z" + assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + 2009, 12, 17, 12, 44, 32, 0, pytz.utc + ) + + @pytest.mark.parametrize( "truncated, micros", [ @@ -116,6 +123,26 @@ def test_from_rfc3339_with_truncated_nanos(truncated, micros): ) +@pytest.mark.parametrize( + "truncated, micros", + [ + ("12345678", 123456), + ("1234567", 123456), + ("123456", 123456), + ("12345", 123450), + ("1234", 123400), + ("123", 123000), + ("12", 120000), + ("1", 100000), + ], +) +def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): + value = "2009-12-17T12:44:32.{}Z".format(truncated) + assert datetime_helpers.from_rfc3339_nanos(value) == datetime.datetime( + 2009, 12, 17, 12, 44, 32, micros, pytz.utc + ) + + def test_to_rfc3339(): value = datetime.datetime(2016, 4, 5, 13, 30, 0) expected = "2016-04-05T13:30:00.000000Z" From 0503e92f6eb975c3711455df301e797cee9d6bf2 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 25 Jul 2019 13:13:25 +0300 Subject: [PATCH 11/25] Update datetime_helpers.py Fix for coverage and lint failures. --- api_core/google/api_core/datetime_helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index be7e8ebbafae..c01c9c067af7 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -131,11 +131,11 @@ def from_rfc3339(value): in UTC. """ - with_nanos = _RFC3339_NANOS.match(value) - - if with_nanos is None: + try: return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) - else: + except ValueError: + with_nanos = _RFC3339_NANOS.match(value) + bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) @@ -155,14 +155,14 @@ def from_rfc3339_nanos(value): """Convert a nanosecond-precision timestamp to a native datetime. .. note:: - Python datetimes do not support nanosecond precision; this + Python datetimes do not support nanosecond precision; this function therefore truncates such values to microseconds. Args: value (str): The RFC3339 string to convert. Returns: - datetime.datetime: The datetime object equivalent to the + datetime.datetime: The datetime object equivalent to the timestamp in UTC. Raises: @@ -172,7 +172,7 @@ def from_rfc3339_nanos(value): # Raise deprecation warnings for things we want to go away. warnings.warn( "The `from_rfc3339_nanos` method will be deprecated in" - "future versions; use `from_rfc3339` instead.", + " future versions; use `from_rfc3339` instead.", PendingDeprecationWarning, stacklevel=2, ) From dc4f896f7ef5bde0e6c0ecb6e2306d77bafc4877 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 26 Jul 2019 10:22:47 +0300 Subject: [PATCH 12/25] Update datetime_helpers.py restore previous version with testing regex first + cosmetic chgs for the descriptions --- api_core/google/api_core/datetime_helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index c01c9c067af7..476110668658 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -131,11 +131,11 @@ def from_rfc3339(value): in UTC. """ - try: - return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) - except ValueError: - with_nanos = _RFC3339_NANOS.match(value) + with_nanos = _RFC3339_NANOS.match(value) + if with_nanos is None: + return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + else: bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) @@ -152,7 +152,7 @@ def from_rfc3339(value): def from_rfc3339_nanos(value): - """Convert a nanosecond-precision timestamp to a native datetime. + """DEPRECATED. Convert a nanosecond-precision timestamp to a native datetime. .. note:: Python datetimes do not support nanosecond precision; this @@ -171,8 +171,8 @@ def from_rfc3339_nanos(value): """ # Raise deprecation warnings for things we want to go away. warnings.warn( - "The `from_rfc3339_nanos` method will be deprecated in" - " future versions; use `from_rfc3339` instead.", + "The `from_rfc3339_nanos` function is deprecated" + " use `from_rfc3339` instead.", PendingDeprecationWarning, stacklevel=2, ) From 4bcf6518adb27a6e6e5c10d83cdde558b494f896 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 26 Jul 2019 15:07:35 +0300 Subject: [PATCH 13/25] Update test_datetime_helpers.py added test to check the deprecation warning --- api_core/tests/unit/test_datetime_helpers.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index e06daf88bfcb..c1096cc914a6 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -15,6 +15,7 @@ import calendar import datetime +import mock import pytest import pytz @@ -23,6 +24,8 @@ ONE_MINUTE_IN_MICROSECONDS = 60 * 1e6 +MESSAGE = ("The `from_rfc3339_nanos` function is deprecated" + " use `from_rfc3339` instead.") def test_utcnow(): @@ -123,6 +126,19 @@ def test_from_rfc3339_with_truncated_nanos(truncated, micros): ) +def test_from_rfc3339_nanos_is_deprecated(): + from_rfc3339_patch = mock.patch("google.api_core.datetime_helpers.from_rfc3339") + warnings_patch = mock.patch("warnings.warn") + value = "2009-12-17T12:44:32.123456Z" + + with from_rfc3339_patch as from_rfc3339, warnings_patch as warn: + result = datetime_helpers.from_rfc3339_nanos(value) + + assert result is from_rfc3339.return_value + from_rfc3339.assert_called_once_with(value) + warn.assert_called_once_with(MESSAGE, DeprecationWarning, stacklevel=2) + + @pytest.mark.parametrize( "truncated, micros", [ From 8151cae2d1230880245b651888c2134752cd15a0 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 26 Jul 2019 15:09:28 +0300 Subject: [PATCH 14/25] Update datetime_helpers.py adjusted the condition for the 'from_rfc3339' method + warning fix --- api_core/google/api_core/datetime_helpers.py | 26 +++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 476110668658..4368df89c3b5 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -132,23 +132,19 @@ def from_rfc3339(value): """ with_nanos = _RFC3339_NANOS.match(value) + bare_seconds = datetime.datetime.strptime( + with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION + ) + fraction = with_nanos.group("nanos") - if with_nanos is None: - return datetime.datetime.strptime(value, _RFC3339_MICROS).replace(tzinfo=pytz.utc) + if fraction is None: + micros = 0 else: - bare_seconds = datetime.datetime.strptime( - with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION - ) - fraction = with_nanos.group("nanos") - - if fraction is None: - micros = 0 - else: - scale = 9 - len(fraction) - nanos = int(fraction) * (10 ** scale) - micros = nanos // 1000 + scale = 9 - len(fraction) + nanos = int(fraction) * (10 ** scale) + micros = nanos // 1000 - return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) + return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) def from_rfc3339_nanos(value): @@ -173,7 +169,7 @@ def from_rfc3339_nanos(value): warnings.warn( "The `from_rfc3339_nanos` function is deprecated" " use `from_rfc3339` instead.", - PendingDeprecationWarning, + DeprecationWarning, stacklevel=2, ) return from_rfc3339(value) From 447ed3e294ab4a1396728f80390186e9b03dfaa2 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 23 Aug 2019 10:56:38 +0300 Subject: [PATCH 15/25] Update datetime_helpers.py * deleted extra spaces --- api_core/google/api_core/datetime_helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 4368df89c3b5..310240f4d4b7 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -124,7 +124,7 @@ def from_rfc3339(value): therefore truncates such values to microseconds. Args: - value (str): The RFC 3339 string to convert. + value (str): The RFC3339 string to convert. Returns: datetime.datetime: The datetime object equivalent to the timestamp @@ -162,7 +162,7 @@ def from_rfc3339_nanos(value): timestamp in UTC. Raises: - ValueError: If the timestamp does not match the RFC 3339 + ValueError: If the timestamp does not match the RFC3339 regular expression. """ # Raise deprecation warnings for things we want to go away. @@ -221,10 +221,10 @@ def nanosecond(self): return self._nanosecond def rfc3339(self): - """Return an RFC 3339-compliant timestamp. + """Return an RFC3339-compliant timestamp. Returns: - (str): Timestamp string according to RFC 3339 spec. + (str): Timestamp string according to RFC3339 spec. """ if self._nanosecond == 0: return to_rfc3339(self) @@ -233,10 +233,10 @@ def rfc3339(self): @classmethod def from_rfc3339(cls, stamp): - """Parse RFC 3339-compliant timestamp, preserving nanoseconds. + """Parse RFC3339-compliant timestamp, preserving nanoseconds. Args: - stamp (str): RFC 3339 stamp, with up to nanosecond precision + stamp (str): RFC3339 stamp, with up to nanosecond precision Returns: :class:`DatetimeWithNanoseconds`: @@ -286,7 +286,7 @@ def timestamp_pb(self): @classmethod def from_timestamp_pb(cls, stamp): - """Parse RFC 3339-compliant timestamp, preserving nanoseconds. + """Parse RFC3339-compliant timestamp, preserving nanoseconds. Args: stamp (:class:`~google.protobuf.timestamp_pb2.Timestamp`): timestamp message From 1fc8661cc3ea18de692370c476c2bf77873bff7a Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Thu, 5 Sep 2019 20:52:35 +0300 Subject: [PATCH 16/25] update datetime_helpers * 'if-else' statement restored to throw an exception * wrote additional tests to cover the exception --- api_core/google/api_core/datetime_helpers.py | 10 ++++- api_core/tests/unit/test_datetime_helpers.py | 43 ++++++++++++++------ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 310240f4d4b7..9bc6a910974b 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -132,6 +132,14 @@ def from_rfc3339(value): """ with_nanos = _RFC3339_NANOS.match(value) + + if with_nanos is None: + raise ValueError( + "Timestamp: {!r}, does not match pattern: {!r}".format( + value, _RFC3339_NANOS.pattern + ) + ) + bare_seconds = datetime.datetime.strptime( with_nanos.group("no_fraction"), _RFC3339_NO_FRACTION ) @@ -228,7 +236,7 @@ def rfc3339(self): """ if self._nanosecond == 0: return to_rfc3339(self) - nanos = str(self._nanosecond).rjust(9, '0').rstrip("0") + nanos = str(self._nanosecond).rjust(9, "0").rstrip("0") return "{}.{}Z".format(self.strftime(_RFC3339_NO_FRACTION), nanos) @classmethod diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index c1096cc914a6..948723f96ead 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -24,8 +24,9 @@ ONE_MINUTE_IN_MICROSECONDS = 60 * 1e6 -MESSAGE = ("The `from_rfc3339_nanos` function is deprecated" - " use `from_rfc3339` instead.") +MESSAGE = ( + "The `from_rfc3339_nanos` function is deprecated" " use `from_rfc3339` instead." +) def test_utcnow(): @@ -159,6 +160,18 @@ def test_from_rfc3339_nanos_with_truncated_nanos(truncated, micros): ) +def test_from_rfc3339_wo_nanos_raise_exception(): + value = "2009-12-17T12:44:32" + with pytest.raises(ValueError): + datetime_helpers.from_rfc3339(value) + + +def test_from_rfc3339_w_nanos_raise_exception(): + value = "2009-12-17T12:44:32.123456" + with pytest.raises(ValueError): + datetime_helpers.from_rfc3339(value) + + def test_to_rfc3339(): value = datetime.datetime(2016, 4, 5, 13, 30, 0) expected = "2016-04-05T13:30:00.000000Z" @@ -186,10 +199,11 @@ def test_to_rfc3339_with_non_utc_ignore_zone(): class Test_DateTimeWithNanos(object): - @staticmethod def test_ctor_wo_nanos(): - stamp = datetime_helpers.DatetimeWithNanoseconds(2016, 12, 20, 21, 13, 47, 123456) + stamp = datetime_helpers.DatetimeWithNanoseconds( + 2016, 12, 20, 21, 13, 47, 123456 + ) assert stamp.year == 2016 assert stamp.month == 12 assert stamp.day == 20 @@ -229,7 +243,9 @@ def test_ctor_w_micros_keyword_and_nanos(): @staticmethod def test_rfc3339_wo_nanos(): - stamp = datetime_helpers.DatetimeWithNanoseconds(2016, 12, 20, 21, 13, 47, 123456) + stamp = datetime_helpers.DatetimeWithNanoseconds( + 2016, 12, 20, 21, 13, 47, 123456 + ) assert stamp.rfc3339() == "2016-12-20T21:13:47.123456Z" @staticmethod @@ -314,12 +330,16 @@ def test_from_rfc3339_w_full_precision(): ) def test_from_rfc3339_test_nanoseconds(fractional, nanos): value = "2009-12-17T12:44:32.{}Z".format(fractional) - assert datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(value).nanosecond == nanos + assert ( + datetime_helpers.DatetimeWithNanoseconds.from_rfc3339(value).nanosecond + == nanos + ) @staticmethod def test_timestamp_pb_wo_nanos_naive(): stamp = datetime_helpers.DatetimeWithNanoseconds( - 2016, 12, 20, 21, 13, 47, 123456) + 2016, 12, 20, 21, 13, 47, 123456 + ) delta = stamp.replace(tzinfo=pytz.UTC) - datetime_helpers._UTC_EPOCH seconds = int(delta.total_seconds()) nanos = 123456000 @@ -333,7 +353,8 @@ def test_timestamp_pb_w_nanos(): ) delta = stamp - datetime_helpers._UTC_EPOCH timestamp = timestamp_pb2.Timestamp( - seconds=int(delta.total_seconds()), nanos=123456789) + seconds=int(delta.total_seconds()), nanos=123456789 + ) assert stamp.timestamp_pb() == timestamp @staticmethod @@ -343,8 +364,7 @@ def test_from_timestamp_pb_wo_nanos(): seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds) - stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb( - timestamp) + stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb(timestamp) assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 0 @@ -358,8 +378,7 @@ def test_from_timestamp_pb_w_nanos(): seconds = int(delta.total_seconds()) timestamp = timestamp_pb2.Timestamp(seconds=seconds, nanos=123456789) - stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb( - timestamp) + stamp = datetime_helpers.DatetimeWithNanoseconds.from_timestamp_pb(timestamp) assert _to_seconds(when) == _to_seconds(stamp) assert stamp.microsecond == 123456 From 9dcffee18ed9cf8c61c2992dae81c7e927e43db4 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 11:15:40 +0300 Subject: [PATCH 17/25] update comments --- api_core/google/api_core/datetime_helpers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 9bc6a910974b..d670a12bd3b7 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -116,8 +116,15 @@ def from_iso8601_time(value): def from_rfc3339(value): +<<<<<<< HEAD + """Convert an RFC3339-format timestamp to a native datetime. + + Supported formats include those without fractional seconds, or with + any fraction up to nanosecond precision. +======= """Convert a nanosecond-precision or if nanoseconds are missing microsecond-precision timestamp to a native datetime. +>>>>>>> 06bee67b097e485a58415d218f3ac421a04d3d6a .. note:: Python datetimes do not support nanosecond precision; this function @@ -173,6 +180,8 @@ def from_rfc3339_nanos(value): ValueError: If the timestamp does not match the RFC3339 regular expression. """ +<<<<<<< HEAD +======= # Raise deprecation warnings for things we want to go away. warnings.warn( "The `from_rfc3339_nanos` function is deprecated" @@ -180,6 +189,7 @@ def from_rfc3339_nanos(value): DeprecationWarning, stacklevel=2, ) +>>>>>>> 06bee67b097e485a58415d218f3ac421a04d3d6a return from_rfc3339(value) From bc18ddbcda37b6a8788c4fbda3bbe118b7b13e96 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 11:25:29 +0300 Subject: [PATCH 18/25] update as requested --- api_core/google/api_core/datetime_helpers.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index d670a12bd3b7..a7ee91b9365e 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -116,15 +116,10 @@ def from_iso8601_time(value): def from_rfc3339(value): -<<<<<<< HEAD """Convert an RFC3339-format timestamp to a native datetime. Supported formats include those without fractional seconds, or with any fraction up to nanosecond precision. -======= - """Convert a nanosecond-precision or if nanoseconds are missing - microsecond-precision timestamp to a native datetime. ->>>>>>> 06bee67b097e485a58415d218f3ac421a04d3d6a .. note:: Python datetimes do not support nanosecond precision; this function @@ -180,16 +175,6 @@ def from_rfc3339_nanos(value): ValueError: If the timestamp does not match the RFC3339 regular expression. """ -<<<<<<< HEAD -======= - # Raise deprecation warnings for things we want to go away. - warnings.warn( - "The `from_rfc3339_nanos` function is deprecated" - " use `from_rfc3339` instead.", - DeprecationWarning, - stacklevel=2, - ) ->>>>>>> 06bee67b097e485a58415d218f3ac421a04d3d6a return from_rfc3339(value) From fcd3e6ae34d2b54c38f87fb15c7582fa71bf3b1f Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 11:29:07 +0300 Subject: [PATCH 19/25] black --- api_core/google/api_core/bidi.py | 10 +++++----- api_core/google/api_core/iam.py | 1 + api_core/google/api_core/protobuf_helpers.py | 1 + api_core/google/api_core/retry.py | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/api_core/google/api_core/bidi.py b/api_core/google/api_core/bidi.py index b171a4112a31..be52d97d42f9 100644 --- a/api_core/google/api_core/bidi.py +++ b/api_core/google/api_core/bidi.py @@ -172,7 +172,9 @@ def __init__(self, access_limit, time_window): self._time_window = time_window self._access_limit = access_limit - self._past_entries = collections.deque(maxlen=access_limit) # least recent first + self._past_entries = collections.deque( + maxlen=access_limit + ) # least recent first self._entry_lock = threading.Lock() def __enter__(self): @@ -198,9 +200,7 @@ def __exit__(self, *_): def __repr__(self): return "{}(access_limit={}, time_window={})".format( - self.__class__.__name__, - self._access_limit, - repr(self._time_window), + self.__class__.__name__, self._access_limit, repr(self._time_window) ) @@ -423,7 +423,7 @@ def __init__( if throttle_reopen: self._reopen_throttle = _Throttle( - access_limit=5, time_window=datetime.timedelta(seconds=10), + access_limit=5, time_window=datetime.timedelta(seconds=10) ) else: self._reopen_throttle = None diff --git a/api_core/google/api_core/iam.py b/api_core/google/api_core/iam.py index 0e108a30679a..df10cf6e7735 100644 --- a/api_core/google/api_core/iam.py +++ b/api_core/google/api_core/iam.py @@ -34,6 +34,7 @@ """ import collections + try: from collections import abc as collections_abc except ImportError: # Python 2.7 diff --git a/api_core/google/api_core/protobuf_helpers.py b/api_core/google/api_core/protobuf_helpers.py index aec6b3620743..92e7844f3e3f 100644 --- a/api_core/google/api_core/protobuf_helpers.py +++ b/api_core/google/api_core/protobuf_helpers.py @@ -15,6 +15,7 @@ """Helpers for :mod:`protobuf`.""" import collections + try: from collections import abc as collections_abc except ImportError: # Python 2.7 diff --git a/api_core/google/api_core/retry.py b/api_core/google/api_core/retry.py index 79343bad3471..5962a68b2a87 100644 --- a/api_core/google/api_core/retry.py +++ b/api_core/google/api_core/retry.py @@ -237,7 +237,7 @@ def __init__( maximum=_DEFAULT_MAXIMUM_DELAY, multiplier=_DEFAULT_DELAY_MULTIPLIER, deadline=_DEFAULT_DEADLINE, - on_error=None + on_error=None, ): self._predicate = predicate self._initial = initial From ed1ac2f9a37a1f9fe425abad02c21742400d5f41 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 12:05:12 +0300 Subject: [PATCH 20/25] flake8 + test fix --- api_core/google/api_core/datetime_helpers.py | 1 - api_core/tests/unit/test_datetime_helpers.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index a7ee91b9365e..1f65446a72e3 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -17,7 +17,6 @@ import calendar import datetime import re -import warnings import pytz diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index 948723f96ead..a84598112668 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -24,9 +24,6 @@ ONE_MINUTE_IN_MICROSECONDS = 60 * 1e6 -MESSAGE = ( - "The `from_rfc3339_nanos` function is deprecated" " use `from_rfc3339` instead." -) def test_utcnow(): @@ -137,7 +134,6 @@ def test_from_rfc3339_nanos_is_deprecated(): assert result is from_rfc3339.return_value from_rfc3339.assert_called_once_with(value) - warn.assert_called_once_with(MESSAGE, DeprecationWarning, stacklevel=2) @pytest.mark.parametrize( From 3b8f432443fbec06fb407c99f88488b7b3de3b6b Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 12:29:12 +0300 Subject: [PATCH 21/25] remove warning test --- api_core/tests/unit/test_datetime_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index a84598112668..2c53e4dc9b72 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -126,7 +126,6 @@ def test_from_rfc3339_with_truncated_nanos(truncated, micros): def test_from_rfc3339_nanos_is_deprecated(): from_rfc3339_patch = mock.patch("google.api_core.datetime_helpers.from_rfc3339") - warnings_patch = mock.patch("warnings.warn") value = "2009-12-17T12:44:32.123456Z" with from_rfc3339_patch as from_rfc3339, warnings_patch as warn: From c7e5b06111e23fc50d92426c0e79087215504cb8 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 12:57:51 +0300 Subject: [PATCH 22/25] test fix --- api_core/tests/unit/test_datetime_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index 2c53e4dc9b72..49f514b82795 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -128,7 +128,7 @@ def test_from_rfc3339_nanos_is_deprecated(): from_rfc3339_patch = mock.patch("google.api_core.datetime_helpers.from_rfc3339") value = "2009-12-17T12:44:32.123456Z" - with from_rfc3339_patch as from_rfc3339, warnings_patch as warn: + with from_rfc3339_patch as from_rfc3339: result = datetime_helpers.from_rfc3339_nanos(value) assert result is from_rfc3339.return_value From 0951c9a0a066d799a65c0d6d2c17de45f95f9e75 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Fri, 8 Nov 2019 14:00:31 +0300 Subject: [PATCH 23/25] try flake8 I202 fix --- api_core/google/api_core/iam.py | 2 +- api_core/google/api_core/protobuf_helpers.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api_core/google/api_core/iam.py b/api_core/google/api_core/iam.py index df10cf6e7735..04680eb58869 100644 --- a/api_core/google/api_core/iam.py +++ b/api_core/google/api_core/iam.py @@ -34,12 +34,12 @@ """ import collections +import warnings try: from collections import abc as collections_abc except ImportError: # Python 2.7 import collections as collections_abc -import warnings # Generic IAM roles diff --git a/api_core/google/api_core/protobuf_helpers.py b/api_core/google/api_core/protobuf_helpers.py index 92e7844f3e3f..365ef25c6784 100644 --- a/api_core/google/api_core/protobuf_helpers.py +++ b/api_core/google/api_core/protobuf_helpers.py @@ -15,11 +15,6 @@ """Helpers for :mod:`protobuf`.""" import collections - -try: - from collections import abc as collections_abc -except ImportError: # Python 2.7 - import collections as collections_abc import copy import inspect @@ -27,6 +22,12 @@ from google.protobuf import message from google.protobuf import wrappers_pb2 +try: + from collections import abc as collections_abc +except ImportError: # Python 2.7 + import collections as collections_abc + + _SENTINEL = object() _WRAPPER_TYPES = ( wrappers_pb2.BoolValue, From 05630fc154d221de2a2a2573062c858f64064934 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 11 Nov 2019 12:18:43 +0300 Subject: [PATCH 24/25] remove _nanos method + modified tests --- api_core/google/api_core/datetime_helpers.py | 23 ++++---------------- api_core/tests/unit/test_datetime_helpers.py | 8 +++---- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/api_core/google/api_core/datetime_helpers.py b/api_core/google/api_core/datetime_helpers.py index 1f65446a72e3..e52fb1dd3bcf 100644 --- a/api_core/google/api_core/datetime_helpers.py +++ b/api_core/google/api_core/datetime_helpers.py @@ -131,6 +131,9 @@ def from_rfc3339(value): datetime.datetime: The datetime object equivalent to the timestamp in UTC. + Raises: + ValueError: If the timestamp does not match the RFC3339 + regular expression. """ with_nanos = _RFC3339_NANOS.match(value) @@ -156,25 +159,7 @@ def from_rfc3339(value): return bare_seconds.replace(microsecond=micros, tzinfo=pytz.utc) -def from_rfc3339_nanos(value): - """DEPRECATED. Convert a nanosecond-precision timestamp to a native datetime. - - .. note:: - Python datetimes do not support nanosecond precision; this - function therefore truncates such values to microseconds. - - Args: - value (str): The RFC3339 string to convert. - - Returns: - datetime.datetime: The datetime object equivalent to the - timestamp in UTC. - - Raises: - ValueError: If the timestamp does not match the RFC3339 - regular expression. - """ - return from_rfc3339(value) +from_rfc3339_nanos = from_rfc3339 # from_rfc3339_nanos method was deprecated. def to_rfc3339(value, ignore_zone=True): diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index 49f514b82795..7b7544b0c64d 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -125,14 +125,12 @@ def test_from_rfc3339_with_truncated_nanos(truncated, micros): def test_from_rfc3339_nanos_is_deprecated(): - from_rfc3339_patch = mock.patch("google.api_core.datetime_helpers.from_rfc3339") value = "2009-12-17T12:44:32.123456Z" - with from_rfc3339_patch as from_rfc3339: - result = datetime_helpers.from_rfc3339_nanos(value) + result = datetime_helpers.from_rfc3339(value) + result_nanos = datetime_helpers.from_rfc3339_nanos(value) - assert result is from_rfc3339.return_value - from_rfc3339.assert_called_once_with(value) + assert result == result_nanos @pytest.mark.parametrize( From 97e866276d07349d5f7cd4971843408efcc28168 Mon Sep 17 00:00:00 2001 From: Leonid Emar-Kar Date: Mon, 11 Nov 2019 12:42:51 +0300 Subject: [PATCH 25/25] flake8 update --- api_core/tests/unit/test_datetime_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api_core/tests/unit/test_datetime_helpers.py b/api_core/tests/unit/test_datetime_helpers.py index 7b7544b0c64d..4ddcf36103f2 100644 --- a/api_core/tests/unit/test_datetime_helpers.py +++ b/api_core/tests/unit/test_datetime_helpers.py @@ -15,7 +15,6 @@ import calendar import datetime -import mock import pytest import pytz