From 79c29d5e3082e1779f468865ee056d593b169bab Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Fri, 14 May 2021 12:28:29 -0700 Subject: [PATCH 1/6] fix: add ConnectionError to default retry --- google/cloud/storage/retry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/storage/retry.py b/google/cloud/storage/retry.py index e17f3d5a0..c9b49ca83 100644 --- a/google/cloud/storage/retry.py +++ b/google/cloud/storage/retry.py @@ -22,6 +22,7 @@ _RETRYABLE_TYPES = ( + ConnectionError, api_exceptions.TooManyRequests, # 429 api_exceptions.InternalServerError, # 500 api_exceptions.BadGateway, # 502 From 67c2b22791f2e434c7d8c0bc129e06809a0a7d67 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Tue, 18 May 2021 13:48:59 -0700 Subject: [PATCH 2/6] add _RETRYABLE_STDLIB_TYPES to address ConnectionError not in python2 --- google/cloud/storage/retry.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/retry.py b/google/cloud/storage/retry.py index c9b49ca83..0683fff59 100644 --- a/google/cloud/storage/retry.py +++ b/google/cloud/storage/retry.py @@ -22,7 +22,6 @@ _RETRYABLE_TYPES = ( - ConnectionError, api_exceptions.TooManyRequests, # 429 api_exceptions.InternalServerError, # 500 api_exceptions.BadGateway, # 502 @@ -31,6 +30,13 @@ requests.ConnectionError, ) + +try: + _RETRYABLE_STDLIB_TYPES = (ConnectionError,) +except NameError: + _RETRYABLE_STDLIB_TYPES = () + + # Some retriable errors don't have their own custom exception in api_core. _ADDITIONAL_RETRYABLE_STATUS_CODES = (408,) @@ -39,6 +45,8 @@ def _should_retry(exc): """Predicate for determining when to retry.""" if isinstance(exc, _RETRYABLE_TYPES): return True + elif isinstance(exc, _RETRYABLE_STDLIB_TYPES): + return True elif isinstance(exc, api_exceptions.GoogleAPICallError): return exc.code in _ADDITIONAL_RETRYABLE_STATUS_CODES elif isinstance(exc, auth_exceptions.TransportError): From 4477757ea7cce949325ba594b1c3c21a9cdd077a Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Tue, 18 May 2021 15:21:18 -0700 Subject: [PATCH 3/6] add test for ConnectionError default retries --- google/cloud/storage/retry.py | 17 ++++++++--------- tests/unit/test_retry.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/google/cloud/storage/retry.py b/google/cloud/storage/retry.py index 0683fff59..7b9626ed5 100644 --- a/google/cloud/storage/retry.py +++ b/google/cloud/storage/retry.py @@ -21,7 +21,14 @@ import json -_RETRYABLE_TYPES = ( +# ConnectionError is a built-in exception only in Python3 and not in Python2. +try: + _RETRYABLE_STDLIB_TYPES = (ConnectionError,) +except NameError: + _RETRYABLE_STDLIB_TYPES = () + + +_RETRYABLE_TYPES = _RETRYABLE_STDLIB_TYPES + ( api_exceptions.TooManyRequests, # 429 api_exceptions.InternalServerError, # 500 api_exceptions.BadGateway, # 502 @@ -31,12 +38,6 @@ ) -try: - _RETRYABLE_STDLIB_TYPES = (ConnectionError,) -except NameError: - _RETRYABLE_STDLIB_TYPES = () - - # Some retriable errors don't have their own custom exception in api_core. _ADDITIONAL_RETRYABLE_STATUS_CODES = (408,) @@ -45,8 +46,6 @@ def _should_retry(exc): """Predicate for determining when to retry.""" if isinstance(exc, _RETRYABLE_TYPES): return True - elif isinstance(exc, _RETRYABLE_STDLIB_TYPES): - return True elif isinstance(exc, api_exceptions.GoogleAPICallError): return exc.code in _ADDITIONAL_RETRYABLE_STATUS_CODES elif isinstance(exc, auth_exceptions.TransportError): diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 582fa8097..fe2434a74 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -59,6 +59,16 @@ def test_w_requests_connection_error(self): exc = ValueError("testing") self.assertFalse(self._call_fut(exc)) + def test_w_stdlib_connection_error(self): + from google.cloud.storage import retry + + try: + exc = ConnectionError() + self.assertTrue(self._call_fut(exc)) + self.assertTrue(ConnectionError in retry._RETRYABLE_TYPES) + except NameError: + pass + class TestConditionalRetryPolicy(unittest.TestCase): def _make_one(self, retry_policy, conditional_predicate, required_kwargs): From e7cc7598ec4ae486157e7db4b1bc2dd86e277b0f Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Wed, 19 May 2021 13:55:27 -0700 Subject: [PATCH 4/6] rename test_miss_w_stdlib_error and update test_w_requests_connection_error --- tests/unit/test_retry.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index fe2434a74..600896b7b 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -56,6 +56,12 @@ def test_w_google_api_call_error_miss(self): self.assertFalse(self._call_fut(exc)) def test_w_requests_connection_error(self): + import requests + + exc = requests.ConnectionError() + self.assertTrue(self._call_fut(exc)) + + def test_miss_w_stdlib_error(self): exc = ValueError("testing") self.assertFalse(self._call_fut(exc)) From c37d6bb18667139fa7cff2fda91662a3544d3fac Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Tue, 18 May 2021 06:00:08 +0200 Subject: [PATCH 5/6] chore(deps): update precommit hook pre-commit/pre-commit-hooks to v4 (#444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [pre-commit/pre-commit-hooks](https://togithub.com/pre-commit/pre-commit-hooks) | repository | major | `v3.4.0` -> `v4.0.1` | --- ### Release Notes
pre-commit/pre-commit-hooks ### [`v4.0.1`](https://togithub.com/pre-commit/pre-commit-hooks/releases/v4.0.1) [Compare Source](https://togithub.com/pre-commit/pre-commit-hooks/compare/v4.0.0...v4.0.1) ##### Fixes - `check-shebang-scripts-are-executable` fix entry point. - [#​602](https://togithub.com/pre-commit/pre-commit-hooks/issues/602) issue by [@​Person-93](https://togithub.com/Person-93). - [#​603](https://togithub.com/pre-commit/pre-commit-hooks/issues/603) PR by [@​scop](https://togithub.com/scop). ### [`v4.0.0`](https://togithub.com/pre-commit/pre-commit-hooks/releases/v4.0.0) [Compare Source](https://togithub.com/pre-commit/pre-commit-hooks/compare/v3.4.0...v4.0.0) ##### Features - `check-json`: report duplicate keys. - [#​558](https://togithub.com/pre-commit/pre-commit-hooks/issues/558) PR by [@​AdityaKhursale](https://togithub.com/AdityaKhursale). - [#​554](https://togithub.com/pre-commit/pre-commit-hooks/issues/554) issue by [@​adamchainz](https://togithub.com/adamchainz). - `no-commit-to-branch`: add `main` to default blocked branches. - [#​565](https://togithub.com/pre-commit/pre-commit-hooks/issues/565) PR by [@​ndevenish](https://togithub.com/ndevenish). - `check-case-conflict`: check conflicts in directory names as well. - [#​575](https://togithub.com/pre-commit/pre-commit-hooks/issues/575) PR by [@​slsyy](https://togithub.com/slsyy). - [#​70](https://togithub.com/pre-commit/pre-commit-hooks/issues/70) issue by [@​andyjack](https://togithub.com/andyjack). - `check-vcs-permalinks`: forbid other branch names. - [#​582](https://togithub.com/pre-commit/pre-commit-hooks/issues/582) PR by [@​jack1142](https://togithub.com/jack1142). - [#​581](https://togithub.com/pre-commit/pre-commit-hooks/issues/581) issue by [@​jack1142](https://togithub.com/jack1142). - `check-shebang-scripts-are-executable`: new hook which ensures shebang'd scripts are executable. - [#​545](https://togithub.com/pre-commit/pre-commit-hooks/issues/545) PR by [@​scop](https://togithub.com/scop). ##### Fixes - `check-executables-have-shebangs`: Short circuit shebang lookup on windows. - [#​544](https://togithub.com/pre-commit/pre-commit-hooks/issues/544) PR by [@​scop](https://togithub.com/scop). - `requirements-txt-fixer`: Fix comments which have indentation - [#​549](https://togithub.com/pre-commit/pre-commit-hooks/issues/549) PR by [@​greshilov](https://togithub.com/greshilov). - [#​548](https://togithub.com/pre-commit/pre-commit-hooks/issues/548) issue by [@​greshilov](https://togithub.com/greshilov). - `pretty-format-json`: write to stdout using UTF-8 encoding. - [#​571](https://togithub.com/pre-commit/pre-commit-hooks/issues/571) PR by [@​jack1142](https://togithub.com/jack1142). - [#​570](https://togithub.com/pre-commit/pre-commit-hooks/issues/570) issue by [@​jack1142](https://togithub.com/jack1142). - Use more inclusive language. - [#​599](https://togithub.com/pre-commit/pre-commit-hooks/issues/599) PR by [@​asottile](https://togithub.com/asottile). ##### Breaking changes - Remove deprecated hooks: `flake8`, `pyflakes`, `autopep8-wrapper`. - [#​597](https://togithub.com/pre-commit/pre-commit-hooks/issues/597) PR by [@​asottile](https://togithub.com/asottile).
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻️ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-storage). --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2a87c6d4d..41a6222b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.0.1 hooks: - id: trailing-whitespace - id: end-of-file-fixer From 9d7f59488ce05acb0202d8a865c647a0f3da11c9 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Wed, 19 May 2021 16:13:09 -0700 Subject: [PATCH 6/6] skip testcase stdlib ConnectionError on Python2 --- tests/unit/test_retry.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 600896b7b..3111584cb 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -19,6 +19,14 @@ import mock +try: + ConnectionError +except NameError: + _HAS_STDLIB_CONNECTION_ERROR = False +else: + _HAS_STDLIB_CONNECTION_ERROR = True + + class Test_should_retry(unittest.TestCase): def _call_fut(self, exc): from google.cloud.storage import retry @@ -65,15 +73,12 @@ def test_miss_w_stdlib_error(self): exc = ValueError("testing") self.assertFalse(self._call_fut(exc)) + @unittest.skipUnless( + _HAS_STDLIB_CONNECTION_ERROR, "No builtin 'ConnectionError' in Python 2", + ) def test_w_stdlib_connection_error(self): - from google.cloud.storage import retry - - try: - exc = ConnectionError() - self.assertTrue(self._call_fut(exc)) - self.assertTrue(ConnectionError in retry._RETRYABLE_TYPES) - except NameError: - pass + exc = ConnectionError() + self.assertTrue(self._call_fut(exc)) class TestConditionalRetryPolicy(unittest.TestCase):