diff --git a/.travis.yml b/.travis.yml index 39e9445..f7c19f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,6 @@ install: # command to run tests script: - - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 # for testing + - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 httpretty==0.9.6 # for testing - LOG_LEVEL=DEBUG python -m pytest --junit-xml=junit.xml -vv app/tests - LOG_LEVEL=DEBUG python app/demo.py diff --git a/deepomatic/api/__init__.py b/deepomatic/api/__init__.py index 97cf7a6..1850c0f 100644 --- a/deepomatic/api/__init__.py +++ b/deepomatic/api/__init__.py @@ -1 +1,3 @@ from deepomatic.api.version import __version__ + +__all__ = ["__version__"] diff --git a/deepomatic/api/client.py b/deepomatic/api/client.py index b3bc902..f41a20b 100644 --- a/deepomatic/api/client.py +++ b/deepomatic/api/client.py @@ -23,10 +23,11 @@ """ from deepomatic.api.http_helper import HTTPHelper +from deepomatic.api.resources.account import Account from deepomatic.api.resources.network import Network -from deepomatic.api.resources.recognition import RecognitionSpec, RecognitionVersion +from deepomatic.api.resources.recognition import (RecognitionSpec, + RecognitionVersion) from deepomatic.api.resources.task import Task -from deepomatic.api.resources.account import Account class Client(object): @@ -44,7 +45,7 @@ def __init__(self, *args, **kwargs): If it fails raise a `DeepomaticException`. :type api_key: string :param verify_ssl (optional): whether to ask `requests` to verify the TLS/SSL certificates. - Defaults to `None`. + Defaults to `None`. If `None` try to get it from the `DEEPOMATIC_API_VERIFY_TLS` environment variable (`0`: False, `1`: True). If not found it is set to True. :type verify_ssl: bool @@ -60,6 +61,14 @@ def __init__(self, *args, **kwargs): :param pool_maxsize (optional): Set `requests.adapters.HTTPAdapter.pool_maxsize` for concurrent calls. Defaults to 20. :type pool_maxsize: int + :param requests_timeout: timeout of each request. + Defaults to `http_helper.RequestsTimeout.FAST`. + More details in the `requests` documentation: https://2.python-requests.org/en/master/user/advanced/#timeouts + :type requests_timeout: float or tuple(float, float) + :param http_retry (optional): Customize the retry of http errors. + Defaults to `HTTPRetry()`. Check out `http_retry.HTTPRetry` documentation for more information about the parameters and default values. + If `None`, no retry will be done on errors. + :type http_retry: http_retry.HTTPRetry :return: :class:`Client` object :rtype: deepomatic.api.client.Client diff --git a/deepomatic/api/exceptions.py b/deepomatic/api/exceptions.py index 1546061..7587ee9 100644 --- a/deepomatic/api/exceptions.py +++ b/deepomatic/api/exceptions.py @@ -23,6 +23,7 @@ """ import json +from tenacity import RetryError ############################################################################### @@ -83,11 +84,23 @@ def get_task_id(self): ############################################################################### class TaskTimeout(DeepomaticException): - def __init__(self, task): + def __init__(self, task, retry_error=None): self.task = task + self.retry_error = retry_error def __str__(self): return "Timeout on task:\n{}".format(json.dumps(self.task)) def get_task_id(self): return self.task['id'] + + +############################################################################### + + +class HTTPRetryError(RetryError): + pass + + +class TaskRetryError(RetryError): + pass diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 8aa755c..3dadbd9 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -22,16 +22,18 @@ THE SOFTWARE. """ -import os +import functools import json -import requests -import sys +import os import platform -from requests.structures import CaseInsensitiveDict -from six import string_types +import sys -from deepomatic.api.exceptions import DeepomaticException, BadStatus +import requests +from deepomatic.api.exceptions import BadStatus, DeepomaticException +from deepomatic.api.http_retry import HTTPRetry from deepomatic.api.version import __title__, __version__ +from requests.structures import CaseInsensitiveDict +from six import string_types API_HOST = 'https://api.deepomatic.com' API_VERSION = 0.7 @@ -39,11 +41,31 @@ ############################################################################### +class RequestsTimeout(object): + FAST = (3.05, 10.) + MEDIUM = (3.05, 60.) + SLOW = (3.05, 600.) + + class HTTPHelper(object): - def __init__(self, app_id=None, api_key=None, verify_ssl=None, host=None, version=API_VERSION, check_query_parameters=True, user_agent_prefix='', user_agent_suffix='', pool_maxsize=20): + def __init__(self, app_id=None, api_key=None, verify_ssl=None, + host=None, version=API_VERSION, check_query_parameters=True, + user_agent_prefix='', user_agent_suffix='', pool_maxsize=20, + requests_timeout=RequestsTimeout.FAST, **kwargs): """ - Init the HTTP helper with API key and secret + Init the HTTP helper with API key and secret. + Check out the `client.Client` documentation for more details about the parameters. """ + + # `http_retry` is retrieved from `kwargs` because a default parameter `http_retry=HTTPRetry()` is dangerous + # If the rest of the code mutates `self.http_retry`, it would change the default parameter for all other `Client` instances + self.http_retry = kwargs.pop('http_retry', HTTPRetry()) + + if len(kwargs) > 0: + raise TypeError("Too many parameters. HTTPRetry does not handle kwargs: {}".format(kwargs)) + + self.requests_timeout = requests_timeout + if host is None: host = os.getenv('DEEPOMATIC_API_URL', API_HOST) if verify_ssl is None: @@ -157,7 +179,25 @@ def recursive_json_dump(prefix, obj, data_dict, omit_dot=False): return new_dict - def make_request(self, func, resource, params=None, data=None, content_type='application/json', files=None, stream=False, *args, **kwargs): + def send_request(self, requests_callable, *args, **kwargs): + # requests_callable must be a method from the requests module + + # this is the timeout of requests module + requests_timeout = kwargs.pop('timeout', self.requests_timeout) + http_retry = kwargs.pop('http_retry', self.http_retry) + + functor = functools.partial(requests_callable, *args, + verify=self.verify, + timeout=requests_timeout, **kwargs) + + if http_retry is not None: + return http_retry.retry(functor) + + return functor() + + def make_request(self, func, resource, params=None, data=None, + content_type='application/json', files=None, + stream=False, *args, **kwargs): if content_type is not None: if content_type.strip() == 'application/json': @@ -204,7 +244,11 @@ def make_request(self, func, resource, params=None, data=None, content_type='app if not resource.startswith('http'): resource = self.resource_prefix + resource - response = func(resource, params=params, data=data, files=files, headers=headers, verify=self.verify, stream=stream, *args, **kwargs) + + response = self.send_request(func, resource, *args, + params=params, data=data, + files=files, headers=headers, + stream=stream, **kwargs) # Close opened files for file in opened_files: diff --git a/deepomatic/api/http_retry.py b/deepomatic/api/http_retry.py new file mode 100644 index 0000000..a9c7ded --- /dev/null +++ b/deepomatic/api/http_retry.py @@ -0,0 +1,87 @@ +import functools + +from deepomatic.api import utils +from deepomatic.api.exceptions import HTTPRetryError +from requests.exceptions import (ProxyError, RequestException, + TooManyRedirects, URLRequired) +from tenacity import (retry_if_exception, retry_if_result, stop_after_delay, + wait_chain, wait_fixed, wait_random_exponential) + + +class retry_if_exception_type(retry_if_exception): + # Taken from https://github.com/jd/tenacity/blob/2775f13b34b3ec67a774061a77fcd4e1e9b4157c/tenacity/retry.py#L72 + # Extented to support blacklist types + def __predicate(self, e): + return (isinstance(e, self.exception_types) and + not isinstance(e, self.exception_types_blacklist)) + + def __init__(self, exception_types=Exception, + exception_types_blacklist=()): + self.exception_types = exception_types + self.exception_types_blacklist = exception_types_blacklist + super(retry_if_exception_type, self).__init__(self.__predicate) + + +class HTTPRetry(object): + + """ + :param retry_if (optional): predicate to retry on requests errors. + More details directly in tenacity source code: + + - https://github.com/jd/tenacity/blob/5.1.1/tenacity/__init__.py#L179 + - https://github.com/jd/tenacity/blob/5.1.1/tenacity/retry.py + If not provided, the default behavior is: + - Retry on status code from Default.RETRY_STATUS_CODES + - Retry on exceptions from Default.RETRY_EXCEPTION_TYPES excluding those from Default.RETRY_EXCEPTION_TYPES_BLACKLIST + :type retry_if: tenacity.retry_base + :param wait (optional): how to wait between retry + More details directly in tenacity source code https://github.com/jd/tenacity/blob/5.1.1/tenacity/wait.py + + if not provided, the default behavior is: + ``` + random_wait = wait_random_exponential(multiplier=Default.RETRY_EXP_MULTIPLIER, + max=Default.RETRY_EXP_MAX) + wait_chain(wait_fixed(0.05), + wait_fixed(0.1), + wait_fixed(0.1) + random_wait) + ``` + :type wait: tenacity.wait_base + :param stop (optional). Tell when to stop retrying. By default it stops retrying after a delay of 60 seconds. A last retry can be done just before this delay is reached, thus the total amount of elapsed time might be a bit higher. More details in tenacity source code https://github.com/jd/tenacity/blob/5.1.1/tenacity/stop.py + Raises tenacity.RetryError when timeout is reached. + :type timeout: tenacity.stop_base + """ + + class Default(object): + RETRY_EXP_MAX = 10. + RETRY_EXP_MULTIPLIER = 0.5 + RETRY_STATUS_CODES = [500, 502, 503, 504] + RETRY_EXCEPTION_TYPES = (RequestException, ) + RETRY_EXCEPTION_TYPES_BLACKLIST = (ValueError, ProxyError, TooManyRedirects, URLRequired) + + def __init__(self, retry_if=None, wait=None, stop=None): + self.retry_status_code = {} + self.retry_if = retry_if + self.wait = wait + self.stop = stop + + if self.stop is None: + self.stop = stop_after_delay(60) + + if self.retry_if is None: + self.retry_status_code = set(HTTPRetry.Default.RETRY_STATUS_CODES) + self.retry_if = (retry_if_result(self.retry_if_status_code) | + retry_if_exception_type(HTTPRetry.Default.RETRY_EXCEPTION_TYPES, + HTTPRetry.Default.RETRY_EXCEPTION_TYPES_BLACKLIST)) + + if self.wait is None: + random_wait = wait_random_exponential(multiplier=HTTPRetry.Default.RETRY_EXP_MULTIPLIER, + max=HTTPRetry.Default.RETRY_EXP_MAX) + self.wait = wait_chain(wait_fixed(0.05), + wait_fixed(0.1), + wait_fixed(0.1) + random_wait) + + def retry(self, functor): + return utils.retry(functor, self.retry_if, self.wait, self.stop, retry_error_cls=HTTPRetryError) + + def retry_if_status_code(self, response): + return response.status_code in self.retry_status_code diff --git a/deepomatic/api/inference.py b/deepomatic/api/inference.py new file mode 100644 index 0000000..1c94f01 --- /dev/null +++ b/deepomatic/api/inference.py @@ -0,0 +1,23 @@ +from deepomatic.api.exceptions import DeepomaticException +from deepomatic.api.resources.task import Task +from deepomatic.api.inputs import format_inputs + + +class InferenceResource(object): + def inference(self, return_task=False, wait_task=True, **kwargs): + assert(self._pk is not None) + + inputs = kwargs.pop('inputs', None) + if inputs is None: + raise DeepomaticException("Missing keyword argument: inputs") + content_type, data, files = format_inputs(inputs, kwargs) + result = self._helper.post(self._uri(pk=self._pk, suffix='/inference'), content_type=content_type, data=data, files=files) + task_id = result['task_id'] + task = Task(self._helper, pk=task_id) + if wait_task: + task.wait() + + if return_task: + return task + else: + return task['data'] diff --git a/deepomatic/api/mixins.py b/deepomatic/api/mixins.py index 0ca100b..a6857e2 100644 --- a/deepomatic/api/mixins.py +++ b/deepomatic/api/mixins.py @@ -86,6 +86,16 @@ def delete(self): class CreateableResource(object): def create(self, content_type='application/json', files=None, **kwargs): + post_kwargs = {} + # TODO: this is a hack, kwargs shouldn't be the data to post + # it should be the requests kwargs + # the fix will be a breaking change, so for now we just pop them + for key in ['http_retry', 'timeout']: + try: + post_kwargs[key] = kwargs.pop(key) + except KeyError: + pass + if self._helper.check_query_parameters: for arg_name in kwargs: if arg_name not in self.object_template or self.object_template[arg_name]._shoud_be_present_when_adding is False: @@ -96,7 +106,10 @@ def create(self, content_type='application/json', files=None, **kwargs): if files is not None: content_type = 'multipart/mixed' - data = self._helper.post(self._uri(), data=kwargs, content_type=content_type, files=files) + + data = self._helper.post(self._uri(), data=kwargs, + content_type=content_type, + files=files, **post_kwargs) return self.__class__(self._helper, pk=data['id'], data=data) @@ -108,5 +121,3 @@ def list(self, offset=0, limit=100, **kwargs): ############################################################################### - - diff --git a/deepomatic/api/resources/network.py b/deepomatic/api/resources/network.py index 28dae55..10c2be9 100644 --- a/deepomatic/api/resources/network.py +++ b/deepomatic/api/resources/network.py @@ -22,17 +22,19 @@ THE SOFTWARE. """ -from six import string_types import numpy as np - +from deepomatic.api.http_helper import RequestsTimeout +from deepomatic.api.inference import InferenceResource +from deepomatic.api.mixins import (CreateableResource, DeletableResource, + ImmutableArg, ListableResource, + OptionnalArg, RequiredArg, + UpdatableResource) from deepomatic.api.resource import Resource -from deepomatic.api.utils import InferenceResource -from deepomatic.api.mixins import CreateableResource, ListableResource, UpdatableResource, DeletableResource -from deepomatic.api.mixins import RequiredArg, OptionnalArg, ImmutableArg - +from six import string_types ############################################################################### + class Network(ListableResource, CreateableResource, UpdatableResource, @@ -66,6 +68,12 @@ def inference(self, convert_to_numpy=True, return_task=False, **kwargs): else: return result + def create(self, *args, **kwargs): + # No retry on Network.create() errors by default as this is a large request + kwargs['http_retry'] = kwargs.get('http_retry', None) + kwargs['timeout'] = kwargs.get('timeout', RequestsTimeout.SLOW) + return super(Network, self).create(*args, **kwargs) + @staticmethod def _convert_result_to_numpy(result): new_result = {} diff --git a/deepomatic/api/resources/recognition.py b/deepomatic/api/resources/recognition.py index b8e2974..d46b212 100644 --- a/deepomatic/api/resources/recognition.py +++ b/deepomatic/api/resources/recognition.py @@ -25,7 +25,7 @@ from six import string_types from deepomatic.api.resource import Resource, ResourceList -from deepomatic.api.utils import InferenceResource +from deepomatic.api.inference import InferenceResource from deepomatic.api.mixins import CreateableResource, ListableResource, UpdatableResource, DeletableResource from deepomatic.api.mixins import RequiredArg, OptionnalArg, ImmutableArg, UpdateOnlyArg @@ -76,4 +76,3 @@ class RecognitionVersion(CreateableResource, 'network_id': RequiredArg(), 'post_processings': RequiredArg(), } - diff --git a/deepomatic/api/resources/task.py b/deepomatic/api/resources/task.py index d6ba23d..2112794 100644 --- a/deepomatic/api/resources/task.py +++ b/deepomatic/api/resources/task.py @@ -22,13 +22,17 @@ THE SOFTWARE. """ -from tenacity import Retrying, wait_random_exponential, stop_after_delay, retry_if_result, before_log, after_log, RetryError -from deepomatic.api.resource import Resource -from deepomatic.api.mixins import ListableResource -from deepomatic.api.exceptions import TaskError, TaskTimeout +import functools import logging +from deepomatic.api.exceptions import TaskError, TaskTimeout, HTTPRetryError, TaskRetryError +from deepomatic.api.mixins import ListableResource +from deepomatic.api.resource import Resource +from deepomatic.api.utils import retry, warn_on_http_retry_error +from tenacity import (RetryError, retry_if_exception_type, retry_if_result, + stop_after_delay, wait_chain, wait_fixed, stop_never, + wait_random_exponential) logger = logging.getLogger(__name__) @@ -52,6 +56,19 @@ def has_pending_tasks(pending_tasks): return len(pending_tasks) > 0 +def retry_get_tasks(apply_func, retry_if, timeout=60, + wait_exp_multiplier=0.05, wait_exp_max=1.0): + if timeout is None: + stop = stop_never + else: + stop = stop_after_delay(timeout) + + wait = wait_chain(wait_fixed(0.05), + wait_fixed(0.1) + wait_random_exponential(multiplier=wait_exp_multiplier, + max=min(timeout, wait_exp_max))) + return retry(apply_func, retry_if, wait, stop, retry_error_cls=TaskRetryError) + + class Task(ListableResource, Resource): base_uri = '/tasks/' @@ -62,34 +79,34 @@ def list(self, task_ids): assert(isinstance(task_ids, list)) return super(Task, self).list(task_ids=task_ids) - def wait(self, timeout=60, wait_exp_multiplier=0.05, wait_exp_max=1.0): + def wait(self, **retry_kwargs): """ Wait until task is completed. Expires after 'timeout' seconds. """ try: - retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max), - stop=stop_after_delay(timeout), - retry=retry_if_result(is_pending_status), - before=before_log(logger, logging.DEBUG), - after=after_log(logger, logging.DEBUG)) - retryer(self._refresh_status) - except RetryError: - raise TaskTimeout(self.data()) + retry_get_tasks(self._refresh_status, + retry_if_exception_type(HTTPRetryError) | + retry_if_result(is_pending_status), + **retry_kwargs) + except TaskRetryError as retry_error: + raise TaskTimeout(self._data, retry_error) - if is_error_status(self['status']): - raise TaskError(self.data()) + if is_error_status(self._data['status']): + raise TaskError(self._data) return self def _refresh_status(self): logger.debug("Refreshing Task {}".format(self)) - self.refresh() - return self['status'] + warn_on_http_retry_error(self.refresh, suffix="Retrying until Task.wait timeouts.", reraise=True) + return self._data['status'] def _refresh_tasks_status(self, pending_tasks, success_tasks, error_tasks, positions): logger.debug("Refreshing batch of Task {}".format(pending_tasks)) task_ids = [task.pk for idx, task in pending_tasks] - refreshed_tasks = self.list(task_ids=task_ids) + functor = functools.partial(self.list, task_ids=task_ids) + refreshed_tasks = warn_on_http_retry_error(functor, suffix="Retrying until Task.batch_wait timeouts.", reraise=True) + pending_tasks[:] = [] # clear the list (we have to keep the reference) for task in refreshed_tasks: status = task['status'] @@ -106,7 +123,7 @@ def _refresh_tasks_status(self, pending_tasks, success_tasks, error_tasks, posit return pending_tasks - def batch_wait(self, tasks, timeout=300, wait_exp_multiplier=0.05, wait_exp_max=1.0): + def batch_wait(self, tasks, **retry_kwargs): """ Wait until a list of task are completed. Expires after 'timeout' seconds. @@ -114,6 +131,7 @@ def batch_wait(self, tasks, timeout=300, wait_exp_multiplier=0.05, wait_exp_max= Each list contains a couple (original_position, task) sorted by original_position asc original_position gives the original index in the input tasks list parameter. This helps to keep the order. """ + retry_kwargs['timeout'] = retry_kwargs.get('timeout', 300) try: positions = {} pending_tasks = [] @@ -122,12 +140,11 @@ def batch_wait(self, tasks, timeout=300, wait_exp_multiplier=0.05, wait_exp_max= pending_tasks.append((pos, task)) success_tasks = [] error_tasks = [] - retryer = Retrying(wait=wait_random_exponential(multiplier=wait_exp_multiplier, max=wait_exp_max), - stop=stop_after_delay(timeout), - retry=retry_if_result(has_pending_tasks), - before=before_log(logger, logging.DEBUG), - after=after_log(logger, logging.DEBUG)) - retryer(self._refresh_tasks_status, pending_tasks, success_tasks, error_tasks, positions) + + functor = functools.partial(self._refresh_tasks_status, pending_tasks, + success_tasks, error_tasks, positions) + retry_get_tasks(functor, retry_if_result(has_pending_tasks), **retry_kwargs) + except RetryError: pass diff --git a/deepomatic/api/utils.py b/deepomatic/api/utils.py index 2b94b48..1a53a28 100644 --- a/deepomatic/api/utils.py +++ b/deepomatic/api/utils.py @@ -22,31 +22,41 @@ THE SOFTWARE. """ -from deepomatic.api.exceptions import DeepomaticException -from deepomatic.api.resources.task import Task -from deepomatic.api.inputs import format_inputs - - -############################################################################### - -class InferenceResource(object): - def inference(self, return_task=False, wait_task=True, **kwargs): - assert(self._pk is not None) - - inputs = kwargs.pop('inputs', None) - if inputs is None: - raise DeepomaticException("Missing keyword argument: inputs") - content_type, data, files = format_inputs(inputs, kwargs) - result = self._helper.post(self._uri(pk=self._pk, suffix='/inference'), content_type=content_type, data=data, files=files) - task_id = result['task_id'] - task = Task(self._helper, pk=task_id) - if wait_task: - task.wait() - - if return_task: - return task +import logging + +from deepomatic.api.exceptions import HTTPRetryError +from tenacity import (Retrying, after_log, before_log) + +logger = logging.getLogger(__name__) + + +def retry(apply_func, retry_if, wait, stop, **kwargs): + retryer = Retrying(retry=retry_if, + wait=wait, + stop=stop, + before=before_log(logger, logging.DEBUG), + after=after_log(logger, logging.DEBUG), + **kwargs) + return retryer(apply_func) + + +def warn_on_http_retry_error(http_func, suffix='', reraise=True): + # http helper can raise a HTTPRetryError + try: + # this should be an http_helper call + return http_func() + except HTTPRetryError as e: + last_attempt = e.last_attempt + last_exception = last_attempt.exception(timeout=0) + msg = "HTTPHelper failed to refresh task status. In the last attempt, " + if last_exception is None: + last_response = last_attempt.result() + msg += 'the status code was {}.'.format(last_response.status_code) else: - return task['data'] - - -############################################################################### + msg += 'an exception occured: {}.'.format(last_exception) + if suffix: + msg += ' ' + suffix + logger.warning(msg) + if reraise: + raise + return None diff --git a/deploy/install.sh b/deploy/install.sh index 2c05442..494bdbc 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -4,4 +4,7 @@ set -e apt-get update && apt-get install -y build-essential pip install -r requirements.txt -pip install pytest==4.0.2 pytest-cov==2.6.1 pytest-voluptuous==1.1.0 # for testing +pip install pytest==4.0.2 \ + pytest-cov==2.6.1 \ + pytest-voluptuous==1.1.0 \ + httpretty==0.9.6 # for testing diff --git a/requirements.txt b/requirements.txt index 6b5ec8c..9e16833 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ numpy>=1.10.0,<2 promise>=2.1,<3 six>=1.10.0,<2 requests>=2.19.0,<3 # will not work below in python3 -tenacity>=4.12.0,<5 +tenacity>=5.1,<6 diff --git a/setup.py b/setup.py index bed6697..58addf5 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,11 @@ import io from setuptools import find_packages, setup -try: # for pip >= 10 +try: + # for pip >= 10 from pip._internal.req import parse_requirements -except ImportError: # for pip <= 9.0.3 +except ImportError: + # for pip <= 9.0.3 from pip.req import parse_requirements diff --git a/tests/test_client.py b/tests/test_client.py index d861330..e58e24d 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,25 +1,36 @@ -import os import base64 -import tarfile -import pytest -import tempfile +import functools import hashlib +import logging +import os +import re import shutil -import requests +import tempfile +import time import zipfile -from deepomatic.api.version import __title__, __version__ + +import httpretty +import pytest +import requests +import six from deepomatic.api.client import Client +from deepomatic.api.exceptions import BadStatus, TaskTimeout, HTTPRetryError, TaskRetryError +from deepomatic.api.http_retry import HTTPRetry from deepomatic.api.inputs import ImageInput +from deepomatic.api.version import __title__, __version__ +from requests.exceptions import ConnectionError, MissingSchema +from tenacity import RetryError, stop_after_delay + from pytest_voluptuous import S -from voluptuous.validators import All, Length, Any -import six +from voluptuous.validators import All, Any, Length -import logging logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO')) logger = logging.getLogger(__name__) DEMO_URL = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" +USER_AGENT_PREFIX = '{}-tests/{}'.format(__title__, __version__) + def ExactLen(nb): return Length(min=nb, max=nb) @@ -39,12 +50,13 @@ def download_file(url): return filename +def get_client(*args, **kwargs): + return Client(*args, user_agent_prefix=USER_AGENT_PREFIX, **kwargs) + + @pytest.fixture(scope='session') def client(): - api_host = os.getenv('DEEPOMATIC_API_URL') - app_id = os.environ['DEEPOMATIC_APP_ID'] - api_key = os.environ['DEEPOMATIC_API_KEY'] - yield Client(app_id, api_key, host=api_host, user_agent_prefix='{}-tests/{}'.format(__title__, __version__)) + yield get_client() @pytest.fixture(scope='session') @@ -286,3 +298,103 @@ def test_batch_wait(self, client): for pos, success in success_tasks: assert(tasks[pos].pk == success.pk) assert inference_schema(2, 0, 'golden retriever', 0.8) == success['data'] + + +class TestClientRetry(object): + DEFAULT_TIMEOUT = 2 + DEFAULT_MIN_ATTEMPT_NUMBER = 3 + + def get_client_with_retry(self): + http_retry = HTTPRetry(stop=stop_after_delay(self.DEFAULT_TIMEOUT)) + return get_client(http_retry=http_retry) + + def send_request_and_expect_retry(self, client, timeout, min_attempt_number): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v3') # doesn't make any http call + start_time = time.time() + with pytest.raises(RetryError) as exc: + print(spec.data()) # does make a http call + + diff = time.time() - start_time + assert diff > timeout and diff < timeout + HTTPRetry.Default.RETRY_EXP_MAX + last_attempt = exc.value.last_attempt + assert last_attempt.attempt_number >= min_attempt_number + return last_attempt + + def test_retry_network_failure(self): + http_retry = HTTPRetry(stop=stop_after_delay(self.DEFAULT_TIMEOUT)) + client = get_client(host='http://unknown-domain.com', + http_retry=http_retry) + last_attempt = self.send_request_and_expect_retry(client, self.DEFAULT_TIMEOUT, + self.DEFAULT_MIN_ATTEMPT_NUMBER) + exc = last_attempt.exception(timeout=0) + assert isinstance(exc, ConnectionError) + assert 'Name or service not known' in str(exc) + + def register_uri(self, methods, status): + for method in methods: + httpretty.register_uri( + method, + re.compile(r'https?://.*'), + status=502, + content_type="application/json" + ) + + @httpretty.activate + def test_retry_bad_http_status(self): + self.register_uri([httpretty.GET, httpretty.POST], 502) + client = self.get_client_with_retry() + last_attempt = self.send_request_and_expect_retry(client, self.DEFAULT_TIMEOUT, + self.DEFAULT_MIN_ATTEMPT_NUMBER) + assert last_attempt.exception(timeout=0) is None # no exception raised during retry + last_response = last_attempt.result() + assert 502 == last_response.status_code + + @httpretty.activate + def test_no_retry_create_network(self): + self.register_uri([httpretty.GET, httpretty.POST], 502) + client = self.get_client_with_retry() + # Creating network doesn't retry, we directly get a 502 + t = time.time() + with pytest.raises(BadStatus) as exc: + client.Network.create(name="My first network", + framework='tensorflow-1.x', + preprocessing=["useless"], + files=["useless"]) + assert 502 == exc.status_code + assert time.time() - t < 0.3 + + def test_retry_task_with_http_errors(self): + # We create two clients on purpose because of a bug in httpretty + # https://github.com/gabrielfalcao/HTTPretty/issues/381 + # Also this allow us to test a simple requests with no retryer + + client = get_client(http_retry=None) + spec = client.RecognitionSpec.retrieve('imagenet-inception-v3') + task = spec.inference(inputs=[ImageInput(DEMO_URL)], return_task=True, wait_task=False) + + client = self.get_client_with_retry() + with httpretty.enabled(): + task = client.Task.retrieve(task.pk) + httpretty.register_uri( + httpretty.GET, + re.compile(r'https?://.*?/tasks/\d+/?'), + status=502, + ) + + with pytest.raises(TaskTimeout) as task_timeout: + task.wait(timeout=5) + + # Test nested retry errors + # TaskRetryError has been raised because of too many HTTPRetryError + # (couldn't refresh the status once) + retry_error = task_timeout.value.retry_error + assert isinstance(retry_error, TaskRetryError) + last_exception = retry_error.last_attempt.exception(timeout=0) + assert isinstance(last_exception, HTTPRetryError) + assert 502 == last_exception.last_attempt.result().status_code + + def test_no_retry_blacklist_exception(self): + client = self.get_client_with_retry() + # check that there is no retry on exceptions from DEFAULT_RETRY_EXCEPTION_TYPES_BLACKLIST + with pytest.raises(MissingSchema): + client.http_helper.http_retry.retry(functools.partial(requests.get, ''))