From 7fe560751815fcd88d9565ac416abf6ae47edf45 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 11:30:51 +0100 Subject: [PATCH 01/13] Require pyopenssl Ensures the package will not install on python versions without SSL --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 5295692..8a5dd22 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ install_requires=[ 'Requests>=2.2.0', 'six>=1.4.0', + 'pyopenssl>=16.0.0' ], test_suite='tests', tests_require=[ From 79356d5812307cc92ee030b68688a5bb5b5e7a91 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 11:54:40 +0100 Subject: [PATCH 02/13] Bump version and changelog --- Changes.txt | 3 +++ setup.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Changes.txt b/Changes.txt index 5f1309b..c201e71 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,3 +1,6 @@ +v1.2.1 Wed 6 May 2020 + Ensure OpenSSL is available on installation + v1.2 Sun 9 Jun 2019 Use https as default Handle 401 and 403 exceptions diff --git a/setup.py b/setup.py index 8a5dd22..6d4bb22 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( name="opencage", - version="1.2", + version="1.2.1", description="Simple wrapper module for the OpenCage Geocoder API", long_description=long_description, long_description_content_type='text/markdown', @@ -56,7 +56,7 @@ install_requires=[ 'Requests>=2.2.0', 'six>=1.4.0', - 'pyopenssl>=16.0.0' + 'pyopenssl>=0.15.1' ], test_suite='tests', tests_require=[ From ed7a2fa4c4f0df17e73f3494e4d7120cf438df57 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 15:39:37 +0100 Subject: [PATCH 03/13] Add exponential backoff with jitter to unknown errors --- Changes.txt | 1 + opencage/__init__.py | 2 +- opencage/geocoder.py | 39 ++++++++++++++++++++++----------------- setup.py | 3 ++- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Changes.txt b/Changes.txt index c201e71..3eb443c 100644 --- a/Changes.txt +++ b/Changes.txt @@ -1,5 +1,6 @@ v1.2.1 Wed 6 May 2020 Ensure OpenSSL is available on installation + Add exponential backoff to HTTP requests v1.2 Sun 9 Jun 2019 Use https as default diff --git a/opencage/__init__.py b/opencage/__init__.py index d4043d5..8d050a0 100644 --- a/opencage/__init__.py +++ b/opencage/__init__.py @@ -2,4 +2,4 @@ __author__ = "OpenCage Data" __email__ = 'info@opencagedata.com' -__version__ = '1.0.0' +__version__ = '1.2.1' diff --git a/opencage/geocoder.py b/opencage/geocoder.py index aea7ebc..4b164f9 100644 --- a/opencage/geocoder.py +++ b/opencage/geocoder.py @@ -6,6 +6,7 @@ import six import requests +import backoff class OpenCageGeocodeError(Exception): @@ -142,7 +143,26 @@ def geocode(self, query, **kwargs): # Add user parameters data.update(kwargs) - response = requests.get(self.url, params=data) + response = self._opencage_request(data) + + return floatify_latlng(response['results']) + + def reverse_geocode(self, lat, lng, **kwargs): + """ + Given a latitude & longitude, return an address for that point from OpenCage's Geocoder. + + :param lat: Latitude + :param lng: Longitude + :return: Results from OpenCageData + :rtype: dict + :raises RateLimitExceededError: if you have exceeded the number of queries you can make. Exception says when you can try again + :raises UnknownError: if something goes wrong with the OpenCage API + """ + return self.geocode(_query_for_reverse_geocoding(lat, lng), **kwargs) + + @backoff.on_exception(backoff.expo, UnknownError, max_tries=8, max_time=120) + def _opencage_request(self, params): + response = requests.get(self.url, params=params) if (response.status_code == 401): raise NotAuthorizedError() @@ -166,22 +186,7 @@ def geocode(self, query, **kwargs): if 'results' not in response_json: raise UnknownError("JSON from API doesn't have a 'results' key") - - return floatify_latlng(response_json['results']) - - def reverse_geocode(self, lat, lng, **kwargs): - """ - Given a latitude & longitude, return an address for that point from OpenCage's Geocoder. - - :param lat: Latitude - :param lng: Longitude - :return: Results from OpenCageData - :rtype: dict - :raises RateLimitExceededError: if you have exceeded the number of queries you can make. Exception says when you can try again - :raises UnknownError: if something goes wrong with the OpenCage API - """ - return self.geocode(_query_for_reverse_geocoding(lat, lng), **kwargs) - + return response_json def _query_for_reverse_geocoding(lat, lng): """ diff --git a/setup.py b/setup.py index 6d4bb22..948aa72 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,8 @@ install_requires=[ 'Requests>=2.2.0', 'six>=1.4.0', - 'pyopenssl>=0.15.1' + 'pyopenssl>=0.15.1', + 'backoff>=1.10.0' ], test_suite='tests', tests_require=[ From b90eb28dfa18145ab93a011ea7c4ef183e9055d8 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 16:05:37 +0100 Subject: [PATCH 04/13] Reduce backoff max_time to 1s in tests --- opencage/geocoder.py | 9 ++++++++- tests.py | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/opencage/geocoder.py b/opencage/geocoder.py index 4b164f9..db62c13 100644 --- a/opencage/geocoder.py +++ b/opencage/geocoder.py @@ -4,10 +4,14 @@ from decimal import Decimal import collections +import os import six import requests import backoff +def backoff_max_time(): + return int(os.environ.get('BACKOFF_MAX_TIME', '120')) + class OpenCageGeocodeError(Exception): """Base class for all errors/exceptions that can happen when geocoding.""" @@ -160,7 +164,10 @@ def reverse_geocode(self, lat, lng, **kwargs): """ return self.geocode(_query_for_reverse_geocoding(lat, lng), **kwargs) - @backoff.on_exception(backoff.expo, UnknownError, max_tries=8, max_time=120) + @backoff.on_exception( + backoff.expo, + (UnknownError), + max_tries=5, max_time=backoff_max_time) def _opencage_request(self, params): response = requests.get(self.url, params=params) diff --git a/tests.py b/tests.py index e8c1d4c..c2c2550 100644 --- a/tests.py +++ b/tests.py @@ -6,6 +6,7 @@ import unittest +import os import six import httpretty @@ -13,6 +14,8 @@ from opencage.geocoder import InvalidInputError, RateLimitExceededError, UnknownError, ForbiddenError, NotAuthorizedError from opencage.geocoder import floatify_latlng, _query_for_reverse_geocoding +# reduce maximum backoff retry time from 120s to 1s +os.environ['BACKOFF_MAX_TIME'] = '1' class OpenCageGeocodeTestCase(unittest.TestCase): def setUp(self): From 2423d11ec5d1485414c369e9cf4df443236c700b Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 16:06:02 +0100 Subject: [PATCH 05/13] Add generic RequestException to retry exceptions --- opencage/geocoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opencage/geocoder.py b/opencage/geocoder.py index db62c13..0465056 100644 --- a/opencage/geocoder.py +++ b/opencage/geocoder.py @@ -166,7 +166,7 @@ def reverse_geocode(self, lat, lng, **kwargs): @backoff.on_exception( backoff.expo, - (UnknownError), + (UnknownError, requests.exceptions.RequestException), max_tries=5, max_time=backoff_max_time) def _opencage_request(self, params): response = requests.get(self.url, params=params) From 15c6d50a49e8221216ac20f4f324a53f3e84bbb4 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:03:53 +0100 Subject: [PATCH 06/13] Try pypy3 in CI --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 5caf7dd..4864307 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py36, pypy +envlist = py27, py33, py36, pypy3 [testenv] commands = {envpython} setup.py test From 19e42158f90318bbc91c574c2f42262eef970df0 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:11:14 +0100 Subject: [PATCH 07/13] Change travis yml to update pypy CI version to pypy3 Reverted mistaken tox.ini change from previous commit --- .travis.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72420ba..a89d57a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - "2.7" - "3.5" - "3.6" - - "pypy" + - "pypy3" matrix: include: diff --git a/tox.ini b/tox.ini index 4864307..5caf7dd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py33, py36, pypy3 +envlist = py27, py33, py36, pypy [testenv] commands = {envpython} setup.py test From 39e628cdabfabc166d5b9fefef8637b6010ac49c Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:16:42 +0100 Subject: [PATCH 08/13] Update CI to use newer Ubuntu LTS --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index a89d57a..2ee9d45 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: bionic git: depth: false language: python @@ -8,11 +8,11 @@ python: - "3.6" - "pypy3" -matrix: - include: - - python: 3.7 - dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) - sudo: required # required for Python 3.7 (travis-ci/travis-ci#9069) +# matrix: +# include: +# - python: 3.7 +# dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) +# sudo: required # required for Python 3.7 (travis-ci/travis-ci#9069) install: - pip install tox-travis From c5a5df1904cda9844c5c0e9259fb9028daa645d9 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:21:32 +0100 Subject: [PATCH 09/13] Include python 3.7 in CI --- .travis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ee9d45..840167f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: bionic +dist: trusty git: depth: false language: python @@ -6,14 +6,9 @@ python: - "2.7" - "3.5" - "3.6" + - "3.7" - "pypy3" -# matrix: -# include: -# - python: 3.7 -# dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) -# sudo: required # required for Python 3.7 (travis-ci/travis-ci#9069) - install: - pip install tox-travis script: From 917f55882790d6f878e8af972309816d867c8b43 Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:24:23 +0100 Subject: [PATCH 10/13] Revert to xenial for python 3.7 --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 840167f..a89d57a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,9 +6,14 @@ python: - "2.7" - "3.5" - "3.6" - - "3.7" - "pypy3" +matrix: + include: + - python: 3.7 + dist: xenial # required for Python 3.7 (travis-ci/travis-ci#9069) + sudo: required # required for Python 3.7 (travis-ci/travis-ci#9069) + install: - pip install tox-travis script: From ee878e15a7ab7fc034f67f72a5602a5686b19d8d Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:26:54 +0100 Subject: [PATCH 11/13] Change main tests back to bionic --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a89d57a..efe8a0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: trusty +dist: bionic git: depth: false language: python From 45469f539765ae5906a41ac4ce5ebf015e539dec Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 17:55:36 +0100 Subject: [PATCH 12/13] Revert travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index efe8a0b..a89d57a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -dist: bionic +dist: trusty git: depth: false language: python From 1f797b613d92c5ae73a14b85d29ed6356110a22a Mon Sep 17 00:00:00 2001 From: Samuel Scully Date: Wed, 6 May 2020 18:02:56 +0100 Subject: [PATCH 13/13] Comment out pypy in CI for now --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a89d57a..5dcebb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - "2.7" - "3.5" - "3.6" - - "pypy3" + # - "pypy3" # out of date OpenSSL error matrix: include: