diff --git a/.dockerignore b/.dockerignore index 9b4052e..1c9530c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,8 @@ Jenkinsfile build/ dist/ +**/*coverage.xml +**/*junit.xml **/*.pyc **/*~ **/.#* diff --git a/.gitignore b/.gitignore index 63c4c7d..156939b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .dmake/ DMakefile +*coverage.xml +*junit.xml *.pyc *~ .#* diff --git a/.travis.yml b/.travis.yml index f5f10a3..39e9445 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,11 @@ python: install: - python setup.py bdist_wheel - pip install dist/deepomatic_api-*.whl - - mkdir samples - - cp demo.py samples/demo.py + - mkdir app + - cp -r tests demo.py app/ # command to run tests script: - - python samples/demo.py + - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 # for testing + - LOG_LEVEL=DEBUG python -m pytest --junit-xml=junit.xml -vv app/tests + - LOG_LEVEL=DEBUG python app/demo.py diff --git a/deepomatic/api/client.py b/deepomatic/api/client.py index ec7d89d..725266b 100644 --- a/deepomatic/api/client.py +++ b/deepomatic/api/client.py @@ -31,17 +31,12 @@ ############################################################################### API_VERSION = 0.7 -API_HOST = 'https://api.deepomatic.com' - ############################################################################### class Client(object): def __init__(self, app_id=None, api_key=None, verify_ssl=True, check_query_parameters=True, host=None, version=API_VERSION, user_agent_suffix='', pool_maxsize=20): - if host is None: - host = API_HOST - self.http_helper = HTTPHelper(app_id, api_key, verify_ssl, host, version, check_query_parameters, user_agent_suffix, pool_maxsize) # /accounts diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 9049c2f..b91a3f3 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -33,6 +33,8 @@ from deepomatic.api.exceptions import DeepomaticException, BadStatus from deepomatic.api.version import __version__ +API_HOST = 'https://api.deepomatic.com' + ############################################################################### @@ -41,6 +43,8 @@ def __init__(self, app_id, api_key, verify, host, version, check_query_parameter """ Init the HTTP helper with API key and secret """ + if host is None: + host = os.getenv('DEEPOMATIC_API_URL', API_HOST) if app_id is None: app_id = os.getenv('DEEPOMATIC_APP_ID') if api_key is None: @@ -65,7 +69,7 @@ def __init__(self, app_id, api_key, verify, host, version, check_query_parameter 'platform': platform.platform() } - self.user_agent = 'deepomatic-client-python/{package_version} requests/{requests_version} python/{python_version} platform/{platform}\ + self.user_agent = 'deepomatic-api/{package_version} requests/{requests_version} python/{python_version} platform/{platform}\ '.format(**user_agent_params) if user_agent_suffix: self.user_agent += ' ' + user_agent_suffix diff --git a/demo.py b/demo.py index bc77144..9b8f609 100644 --- a/demo.py +++ b/demo.py @@ -1,30 +1,24 @@ import os -import sys import json import base64 import tarfile +import logging +import tempfile +import shutil +import hashlib +import requests from deepomatic.api.client import Client from deepomatic.api.inputs import ImageInput -if sys.version_info >= (3, 0): - from urllib.request import urlretrieve -else: - from urllib import urlretrieve - -if len(sys.argv) < 2: - api_host = None -else: - api_host = sys.argv[1] - -app_id = os.getenv('DEEPOMATIC_APP_ID') -api_key = os.getenv('DEEPOMATIC_API_KEY') -client = Client(app_id, api_key, host=api_host) - demo_url = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" +logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'), + format='[%(levelname)s %(name)s %(asctime)s %(process)d %(thread)d %(filename)s:%(lineno)s] %(message)s') +logger = logging.getLogger(__name__) -def demo(): + +def demo(client=None): """ Our REST client works by exposing resources. A resource usually has the following synchronous methods: @@ -38,39 +32,35 @@ def demo(): - delete(): allow to delete the object. """ + ######### + # Setup # + ######### + + # You can create a client in two ways: + # i) explicitly: you pass your APP_ID and API_KEY by calling `client = Client(app_id, api_key)` + # ii) implicitly: you define environment variables `DEEPOMATIC_APP_ID` and `DEEPOMATIC_API_KEY` + # and just call `client = Client()` + # + # Here we actually use a mix of those two methods to illustrate: + if client is None: + app_id = os.getenv('DEEPOMATIC_APP_ID') + api_key = os.getenv('DEEPOMATIC_API_KEY') + client = Client(app_id, api_key) # this would be equivalent to using `Client()` in this case. + ################### # Public networks # ################### - print_header("Listing public networks") - """ - You can access the list of public networks with: 'client.Network.list(public=True)' - Here, public networks are read only so you can only call '.list()'. - The '.list()' method returns a paginated list of objects, i.e. an API call may not return all objects. - By default, it returns 100 objects and gives your the URI at which you will find the next page. - It takes two optionnal arguments: - - 'offset': the index at which we should start iterating (defaut: 0) - - 'limit': the number of element per page (default: 100) - """ - for network in client.Network.list(public=True): - print_comment("{network_id}: {name}".format(network_id=network['id'], name=network['name'])) - - """ - You may also query the list of object with '.data()' but it will only return the JSON associated with - the current page, unlike the iterator version above that will loop trough all the data. - """ - result = client.Network.list(public=True).data() - pretty_print_json(result) - print_header("Getting network") """ + Let's start by getting some public neural network. You can get an object resource using the client with the '.retrieve(id)' method. It will return an object resource which may have '.update(...)' and '.delete()' methods. They respectively modifiy it or delete the object. You may also invoke special actions like '.inference()' (see below). Here, we retrieve a public network named 'imagenet-inception-v1' """ network = client.Network.retrieve('imagenet-inception-v1') - print(network) + logger.info(network) ############################# # Public recognition models # @@ -82,10 +72,23 @@ def demo(): This is the role of a recognition specification: precisely describing some expected output. Those specifications will then be matched to a network via "specification versions". Lets first see the list of public recognition models with 'client.RecognitionSpec.list(public=True)' + Here, public recognition models are read only so you can only call '.list()'. + The '.list()' method returns a paginated list of objects, i.e. an API call may not return all objects. + By default, it returns 100 objects and gives your the URI at which you will find the next page. + It takes two optionnal arguments: + - 'offset': the index at which we should start iterating (defaut: 0) + - 'limit': the number of element per page (default: 100) """ for spec in client.RecognitionSpec.list(public=True): print_comment("- {spec_id}: {name}".format(spec_id=spec['id'], name=spec['name'])) + """ + You may also query the list of object with '.data()' but it will only return the JSON associated with + the current page, unlike the iterator version above that will loop trough all the data. + """ + result = client.RecognitionSpec.list(public=True).data() + pretty_print_json(result) + print_header("Getting spec") """ Let's now focus on what we can do with a recognition models. @@ -110,7 +113,7 @@ def demo(): '.inference()' also support when you pass a single input instead of a list of inputs. Here, it takes a file pointer as input. """ - file = open(download(demo_url, '/tmp/img.jpg'), 'rb') + file = open(download_file(demo_url), 'rb') result = spec.inference(inputs=[ImageInput(file)], show_discarded=True, max_predictions=3) pretty_print_json(result) @@ -150,13 +153,13 @@ def demo(): - snapshot_caffemodel: the file that describe the learned parameters of the model - mean_file: the file that stores the mean image that needs to be substracted from the input """ - deploy_prototxt = download('https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt', '/tmp/deploy.prototxt') - snapshot_caffemodel = download('http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel', '/tmp/snapshot.caffemodel') - mean_file = '/tmp/imagenet_mean.binaryproto' + deploy_prototxt = download_file('https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt') + snapshot_caffemodel = download_file('http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel') + mean_file = os.path.join(tempfile.gettempdir(), 'imagenet_mean.binaryproto') if not os.path.exists(mean_file): - archive = download('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz', '/tmp/caffe_ilsvrc12.tar.gz') + archive = download_file('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz') tar = tarfile.open(archive, "r:gz") - tar.extractall(path='/tmp/') + tar.extractall(path=tempfile.gettempdir()) tar.close() else: print_comment("Skipping download of mean file: {}".format(mean_file)) @@ -272,16 +275,16 @@ def demo(): And this current version can be used to run inference for the spec directly """ result = spec.inference(inputs=[ImageInput(demo_url)], show_discarded=True, max_predictions=3) - print(result) + logger.info(result) print_header("Run inference on specific version with a bounding box") result = version.inference(inputs=[ImageInput(demo_url, bbox={"xmin": 0.1, "ymin": 0.1, "xmax": 0.9, "ymax": 0.9})], show_discarded=True, max_predictions=3) - print(result) + logger.info(result) """ Show all versions of a spec """ - print(spec.versions()) + logger.info(spec.versions()) """ Test the possibility of getting multiple tasks at the same time @@ -289,7 +292,7 @@ def demo(): task = spec.inference(inputs=[ImageInput(demo_url)], return_task=True, wait_task=False) task_id = task.pk tasks = client.Task.list(task_ids=[task_id]) - print(tasks) + logger.info(tasks) print_header("Delete networks and recognition models") """ @@ -297,29 +300,30 @@ def demo(): """ network.delete() + ######################### + # Batched wait on tasks # + ######################### -def demo_batch_tasks(): - """ - Wait tasks per batch - """ print_header("Run multiple inferences and wait for them per batch") spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') tasks = [] timeout = 30 nb_inference = 20 - print("Pushing %d inferences" % nb_inference) + + logger.info("Pushing %d inferences" % nb_inference) for i in range(nb_inference): task = spec.inference(inputs=[ImageInput(demo_url)], return_task=True, wait_task=False) tasks.append(task) - print("Waiting for the results") + + logger.info("Waiting for the results") pending_tasks, success_tasks, error_tasks = client.Task.batch_wait(tasks=tasks, timeout=timeout) if pending_tasks: - print("Warning: %d tasks are still pending after %s seconds" % (len(pending_tasks), timeout)) + logger.warning("%d tasks are still pending after %s seconds" % (len(pending_tasks), timeout)) if error_tasks: - print("Warning: %d tasks are in error" % len(error_tasks)) - print(pending_tasks) - print(error_tasks) - print(success_tasks) + logger.warning("%d tasks are in error" % len(error_tasks)) + logger.info(pending_tasks) + logger.info(error_tasks) + logger.info(success_tasks) # pending_tasks, error_tasks and success_tasks contains the original offset of the input parameter tasks for pos, pending in pending_tasks: @@ -329,36 +333,35 @@ def demo_batch_tasks(): for pos, success in success_tasks: assert(tasks[pos].pk == success.pk) - - - ########### # Helpers # ########### -def download(url, local_path): - if not os.path.isfile(local_path): - print("Downloading {} to {}".format(url, local_path)) - urlretrieve(url, local_path) - if url.endswith('.tar.gz'): - tar = tarfile.open(local_path, "r:gz") - tar.extractall(path='/tmp/') - tar.close() - else: - print("Skipping download of {} to {}: file already exist".format(url, local_path)) - return local_path +def download_file(url): + _, ext = os.path.splitext(url) + filename = os.path.join(tempfile.gettempdir(), + hashlib.sha1(url.encode()).hexdigest() + ext) + if os.path.exists(filename): # avoid redownloading + logger.info("Skipping download of {}: file already exist in ".format(url, filename)) + return filename + r = requests.get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + return filename def print_header(text): - print("\n{}".format(text)) + logger.info("**** {} ****".format(text)) def print_comment(text): - print("--> " + text) + logger.info("--> " + text) def pretty_print_json(data): - print(json.dumps(data, indent=4, separators=(',', ': '))) + logger.info(json.dumps(data, indent=4, separators=(',', ': '))) def display_inference_tensor(result): @@ -368,4 +371,3 @@ def display_inference_tensor(result): if __name__ == '__main__': demo() - demo_batch_tasks() diff --git a/Dockerfile b/deploy/Dockerfile similarity index 72% rename from Dockerfile rename to deploy/Dockerfile index 91c2c14..0aba4b5 100644 --- a/Dockerfile +++ b/deploy/Dockerfile @@ -9,8 +9,11 @@ RUN python setup.py bdist_wheel FROM ${BASE_IMAGE} as runtime +WORKDIR /app + # copy egg COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ -COPY --from=builder /app/demo.py /samples/ +COPY --from=builder /app/demo.py /app/ +COPY --from=builder /app/tests /app/tests RUN pip install /tmp/deepomatic_api-*.whl diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100755 index 0000000..2c05442 --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +apt-get update && apt-get install -y build-essential +pip install -r requirements.txt +pip install pytest==4.0.2 pytest-cov==2.6.1 pytest-voluptuous==1.1.0 # for testing diff --git a/dmake.yml b/dmake.yml index 44a4980..f874d12 100644 --- a/dmake.yml +++ b/dmake.yml @@ -6,6 +6,7 @@ env: variables: DEEPOMATIC_APP_ID: ${TEST_CLIENTS_APP_ID} DEEPOMATIC_API_KEY: ${TEST_CLIENTS_API_KEY} + DEEPOMATIC_API_URL: ${DEEPOMATIC_API_URL:-https://api.deepomatic.com} branches: master: source: ${DEEPOMATIC_CONFIG_DIR}/prod.sh @@ -20,6 +21,10 @@ docker: name: deepomatic-client-python variant: 2.7 root_image: python:2.7 + copy_files: + - requirements.txt + install_scripts: + - deploy/install.sh # Nothing changes comparing to above, except 'variant' and 'root_image' - <<: *base_image variant: 3.4 @@ -31,17 +36,22 @@ docker: variant: 3.6 root_image: python:3.6 + services: - service_name: client config: docker_image: - build: . + build: + context: . + dockerfile: deploy/Dockerfile base_image_variant: - 2.7 - 3.4 - 3.5 - 3.6 - tests: commands: - - python /samples/demo.py + - LOG_LEVEL=DEBUG pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /app/tests + - LOG_LEVEL=DEBUG python /app/demo.py + junit_report: junit.xml + cobertura_report: coverage.xml diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..5ae30e8 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,273 @@ +import os +import base64 +import tarfile +import pytest +import tempfile +import hashlib +import shutil +import requests +from deepomatic.api.client import Client +from deepomatic.api.inputs import ImageInput +from pytest_voluptuous import S +from voluptuous.validators import All, Length, Any +import six + +import logging +logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO')) +logger = logging.getLogger(__name__) + +DEMO_URL = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" + + +def ExactLen(nb): + return Length(min=nb, max=nb) + + +def download_file(url): + _, ext = os.path.splitext(url) + filename = os.path.join(tempfile.gettempdir(), + hashlib.sha1(url.encode()).hexdigest() + ext) + if os.path.exists(filename): # avoid redownloading + return filename + r = requests.get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + return filename + + +@pytest.fixture(scope='session') +def client(): + api_host = os.getenv('DEEPOMATIC_API_URL') + app_id = os.environ['DEEPOMATIC_APP_ID'] + api_key = os.environ['DEEPOMATIC_API_KEY'] + yield Client(app_id, api_key, host=api_host) + + +@pytest.fixture(scope='session') +def custom_network(client): + deploy_prototxt = download_file('https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt') + snapshot_caffemodel = download_file('http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel') + extract_dir = tempfile.gettempdir() + mean_file = os.path.join(extract_dir, 'imagenet_mean.binaryproto') + if not os.path.exists(mean_file): + archive = download_file('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz') + tar = tarfile.open(archive, "r:gz") + tar.extractall(path=extract_dir) + tar.close() + + preprocessing = { + "inputs": [ + { + "tensor_name": "data", + "image": { + "color_channels": "BGR", + "target_size": "224x224", + "resize_type": "SQUASH", + "mean_file": "mean.binaryproto", + "dimension_order": "NCHW", + "pixel_scaling": 255.0, + "data_type": "FLOAT32" + } + } + ], + "batched_output": True + } + + files = { + 'deploy.prototxt': deploy_prototxt, + 'snapshot.caffemodel': snapshot_caffemodel, + 'mean.binaryproto': mean_file + } + + network = client.Network.create(name="My first network", + framework='nv-caffe-0.x-mod', + preprocessing=preprocessing, + files=files) + assert network['id'] + data = network.data() + assert network['name'] == 'My first network' + assert 'description' in data + assert 'create_date' in data + assert 'update_date' in data + + yield network + network.delete() + + +def check_first_prediction(first_label_name, first_score_range): + def check(predicted): + assert predicted[0]['label_name'] == first_label_name + assert predicted[0]['score'] > first_score_range + return predicted + return check + + +def check_score_threshold(is_predicted): + def check(predicted): + for pred in predicted: + if is_predicted: + assert pred['score'] >= pred['threshold'] + else: + assert pred['score'] < pred['threshold'] + return predicted + return check + + +def prediction_schema(exact_len, *args): + return All([{ + 'threshold': float, + 'label_id': int, + 'score': float, + 'label_name': Any(*six.string_types), + }], ExactLen(exact_len), *args) + + +def inference_schema(predicted_len, discarded_len, first_label, first_score): + return S({ + 'outputs': All([{ + 'labels': { + 'predicted': prediction_schema(predicted_len, check_first_prediction(first_label, first_score), + check_score_threshold(is_predicted=True)), + 'discarded': prediction_schema(discarded_len, check_score_threshold(is_predicted=False)) + } + }], ExactLen(1)), + }) + + +class TestClient(object): + + def test_headers(self, client): + http_helper = client.http_helper + session_headers = http_helper.session.headers + assert session_headers['User-Agent'].startswith('deepomatic-api/') + assert 'platform/' in session_headers['User-Agent'] + assert 'python/' in session_headers['User-Agent'] + assert session_headers['X-APP-ID'] + assert session_headers['X-API-KEY'] + + headers = http_helper.setup_headers(headers={'Hello': 'World'}, + content_type='application/json') + assert headers == { + 'Hello': 'World', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + def test_list_specs(self, client): + specs = client.RecognitionSpec.list(public=True) + assert specs.count() > 0 + for spec in specs: + assert spec['id'] + data = spec.data() + assert 'name' in data + assert 'description' in data + assert 'update_date' in data + + result = client.RecognitionSpec.list(public=True).data() + assert len(result['results']) > 0 + assert result['count'] > 0 + + def test_retrieve_spec(self, client): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + assert spec['id'] + data = spec.data() + assert 'name' in data + assert 'description' in data + assert 'update_date' in data + + def test_inference_spec(self, client): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + + first_result = spec.inference(inputs=[ImageInput(DEMO_URL)], show_discarded=True, max_predictions=3) + + assert inference_schema(1, 2, 'golden retriever', 0.9) == first_result + + f = open(download_file(DEMO_URL), 'rb') + result = spec.inference(inputs=[ImageInput(f)], show_discarded=True, max_predictions=3) + assert result == first_result + + f.seek(0) + binary_data = f.read() + result = spec.inference(inputs=[ImageInput(binary_data, encoding="binary")], show_discarded=True, max_predictions=3) + assert result == first_result + + b64 = base64.b64encode(binary_data) + result = spec.inference(inputs=[ImageInput(b64, encoding="base64")], show_discarded=True, max_predictions=3) + assert result == first_result + + def test_create_custom_reco_and_infer(self, client, custom_network): + + # test query by id + network = client.Network.retrieve(custom_network['id']) + assert network['name'] + + custom_network.update(description="I had forgotten the description") + + outputs = client.RecognitionSpec.retrieve('imagenet-inception-v1')['outputs'] + + spec = client.RecognitionSpec.create(name="My recognition model", outputs=outputs) + + version = client.RecognitionVersion.create(network_id=custom_network['id'], spec_id=spec['id'], post_processings=[ + { + "classification": { + "output_tensor": "prob", + } + } + + ]) + assert version['id'] + data = version.data() + assert data['network_id'] == custom_network['id'] + assert 'post_processings' in data + + client.Task.retrieve(custom_network['task_id']).wait() + + result = spec.inference(inputs=[ImageInput(DEMO_URL)], show_discarded=False, max_predictions=3) + assert inference_schema(1, 0, 'golden retriever', 0.9) == result + + result = version.inference(inputs=[ImageInput(DEMO_URL, bbox={"xmin": 0.1, "ymin": 0.1, "xmax": 0.9, "ymax": 0.9})], show_discarded=True, max_predictions=3) + assert inference_schema(1, 2, 'golden retriever', 0.9) == result + + versions = spec.versions() + assert versions.count() > 0 + data = versions.data() + assert len(data['results']) > 0 + assert data['count'] > 0 + + task = spec.inference(inputs=[ImageInput(DEMO_URL)], return_task=True, wait_task=False) + task_id = task.pk + tasks = client.Task.list(task_ids=[task_id]) + tasks = list(tasks) # convert iterables to list + assert len(tasks) == 1 + task = tasks[0] + assert task['status'] in ['pending', 'success'] + assert task['error'] is None + task.wait() + assert task['status'] == 'success' + assert inference_schema(1, 0, 'golden retriever', 0.9) == task['data'] + + def test_batch_wait(self, client): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + tasks = [] + timeout = 30 + nb_inference = 20 + + for i in range(nb_inference): + task = spec.inference(inputs=[ImageInput(DEMO_URL)], return_task=True, wait_task=False) + tasks.append(task) + + pending_tasks, success_tasks, error_tasks = client.Task.batch_wait(tasks=tasks, timeout=timeout) + assert len(pending_tasks) == 0 + assert len(error_tasks) == 0 + assert len(success_tasks) == len(tasks) + + # pending_tasks, error_tasks and success_tasks contains the original offset of the input parameter tasks + for pos, pending in pending_tasks: + assert(tasks[pos].pk == pending.pk) + for pos, err in error_tasks: + assert(tasks[pos].pk == err.pk) + for pos, success in success_tasks: + assert(tasks[pos].pk == success.pk) + assert inference_schema(1, 0, 'golden retriever', 0.9) == success['data']