From 4eb9a326280fddff090c8f802ef0ff2765831d5c Mon Sep 17 00:00:00 2001 From: Vincent Delaitre Date: Tue, 29 Jan 2019 07:18:47 +0100 Subject: [PATCH 01/17] Get host from env --- deepomatic/api/client.py | 5 ----- deepomatic/api/http_helper.py | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/deepomatic/api/client.py b/deepomatic/api/client.py index ec7d89d..725266b 100644 --- a/deepomatic/api/client.py +++ b/deepomatic/api/client.py @@ -31,17 +31,12 @@ ############################################################################### API_VERSION = 0.7 -API_HOST = 'https://api.deepomatic.com' - ############################################################################### class Client(object): def __init__(self, app_id=None, api_key=None, verify_ssl=True, check_query_parameters=True, host=None, version=API_VERSION, user_agent_suffix='', pool_maxsize=20): - if host is None: - host = API_HOST - self.http_helper = HTTPHelper(app_id, api_key, verify_ssl, host, version, check_query_parameters, user_agent_suffix, pool_maxsize) # /accounts diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 9049c2f..2a63809 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -33,6 +33,8 @@ from deepomatic.api.exceptions import DeepomaticException, BadStatus from deepomatic.api.version import __version__ +API_HOST = 'https://api.deepomatic.com' + ############################################################################### @@ -41,6 +43,8 @@ def __init__(self, app_id, api_key, verify, host, version, check_query_parameter """ Init the HTTP helper with API key and secret """ + if host is None: + host = os.getenv('DEEPOMATIC_API_URL', API_HOST) if app_id is None: app_id = os.getenv('DEEPOMATIC_APP_ID') if api_key is None: From 3f3abc831f11e3f8e3b5796a4d4677a041c8e373 Mon Sep 17 00:00:00 2001 From: Vincent Delaitre Date: Tue, 29 Jan 2019 07:26:30 +0100 Subject: [PATCH 02/17] Fix demo --- demo.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/demo.py b/demo.py index bc77144..ed63b6e 100644 --- a/demo.py +++ b/demo.py @@ -42,28 +42,9 @@ def demo(): # Public networks # ################### - print_header("Listing public networks") - """ - You can access the list of public networks with: 'client.Network.list(public=True)' - Here, public networks are read only so you can only call '.list()'. - The '.list()' method returns a paginated list of objects, i.e. an API call may not return all objects. - By default, it returns 100 objects and gives your the URI at which you will find the next page. - It takes two optionnal arguments: - - 'offset': the index at which we should start iterating (defaut: 0) - - 'limit': the number of element per page (default: 100) - """ - for network in client.Network.list(public=True): - print_comment("{network_id}: {name}".format(network_id=network['id'], name=network['name'])) - - """ - You may also query the list of object with '.data()' but it will only return the JSON associated with - the current page, unlike the iterator version above that will loop trough all the data. - """ - result = client.Network.list(public=True).data() - pretty_print_json(result) - print_header("Getting network") """ + Let's start by getting some public neural network. You can get an object resource using the client with the '.retrieve(id)' method. It will return an object resource which may have '.update(...)' and '.delete()' methods. They respectively modifiy it or delete the object. You may also invoke special actions like '.inference()' @@ -82,10 +63,23 @@ def demo(): This is the role of a recognition specification: precisely describing some expected output. Those specifications will then be matched to a network via "specification versions". Lets first see the list of public recognition models with 'client.RecognitionSpec.list(public=True)' + Here, public recognition models are read only so you can only call '.list()'. + The '.list()' method returns a paginated list of objects, i.e. an API call may not return all objects. + By default, it returns 100 objects and gives your the URI at which you will find the next page. + It takes two optionnal arguments: + - 'offset': the index at which we should start iterating (defaut: 0) + - 'limit': the number of element per page (default: 100) """ for spec in client.RecognitionSpec.list(public=True): print_comment("- {spec_id}: {name}".format(spec_id=spec['id'], name=spec['name'])) + """ + You may also query the list of object with '.data()' but it will only return the JSON associated with + the current page, unlike the iterator version above that will loop trough all the data. + """ + result = client.RecognitionSpec.list(public=True).data() + pretty_print_json(result) + print_header("Getting spec") """ Let's now focus on what we can do with a recognition models. From 096b31533dde8da1f855ccf72d4545e82d05fef3 Mon Sep 17 00:00:00 2001 From: Vincent Delaitre Date: Tue, 29 Jan 2019 07:35:53 +0100 Subject: [PATCH 03/17] Move demo.py to package --- .travis.yml | 4 +--- Dockerfile | 1 - demo.py => deepomatic/api/demo.py | 0 dmake.yml | 2 +- 4 files changed, 2 insertions(+), 5 deletions(-) rename demo.py => deepomatic/api/demo.py (100%) diff --git a/.travis.yml b/.travis.yml index f5f10a3..2f00aef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,7 @@ python: install: - python setup.py bdist_wheel - pip install dist/deepomatic_api-*.whl - - mkdir samples - - cp demo.py samples/demo.py # command to run tests script: - - python samples/demo.py + - python -c "from deepomatic.api.demo import demo; demo()" diff --git a/Dockerfile b/Dockerfile index 91c2c14..aafb05b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,5 @@ FROM ${BASE_IMAGE} as runtime # copy egg COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ -COPY --from=builder /app/demo.py /samples/ RUN pip install /tmp/deepomatic_api-*.whl diff --git a/demo.py b/deepomatic/api/demo.py similarity index 100% rename from demo.py rename to deepomatic/api/demo.py diff --git a/dmake.yml b/dmake.yml index 44a4980..a37bf0b 100644 --- a/dmake.yml +++ b/dmake.yml @@ -44,4 +44,4 @@ services: tests: commands: - - python /samples/demo.py + - python -c "from deepomatic.api.demo import demo; demo()" From 9abb89ca8c4d397e47ea516a8283a2c2e26d9fb9 Mon Sep 17 00:00:00 2001 From: Vincent Delaitre Date: Tue, 29 Jan 2019 07:44:42 +0100 Subject: [PATCH 04/17] Late client setup --- deepomatic/api/demo.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/deepomatic/api/demo.py b/deepomatic/api/demo.py index ed63b6e..c85793c 100644 --- a/deepomatic/api/demo.py +++ b/deepomatic/api/demo.py @@ -12,15 +12,6 @@ else: from urllib import urlretrieve -if len(sys.argv) < 2: - api_host = None -else: - api_host = sys.argv[1] - -app_id = os.getenv('DEEPOMATIC_APP_ID') -api_key = os.getenv('DEEPOMATIC_API_KEY') -client = Client(app_id, api_key, host=api_host) - demo_url = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" @@ -38,6 +29,20 @@ def demo(): - delete(): allow to delete the object. """ + ######### + # Setup # + ######### + + # You can create a client in two ways: + # i) explicitly: you pass your APP_ID and API_KEY by calling `client = Client(app_id, api_key)` + # ii) implicitly: you define environment variables `DEEPOMATIC_APP_ID` and `DEEPOMATIC_API_KEY` + # and just call `client = Client()` + # + # Here we actually use a mix of those two methods to illustrate: + app_id = os.getenv('DEEPOMATIC_APP_ID') + api_key = os.getenv('DEEPOMATIC_API_KEY') + client = Client(app_id, api_key) # this would be equivalent to using `Client()` in this case. + ################### # Public networks # ################### @@ -291,20 +296,21 @@ def demo(): """ network.delete() + ######################### + # Batched wait on tasks # + ######################### -def demo_batch_tasks(): - """ - Wait tasks per batch - """ print_header("Run multiple inferences and wait for them per batch") spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') tasks = [] timeout = 30 nb_inference = 20 + print("Pushing %d inferences" % nb_inference) for i in range(nb_inference): task = spec.inference(inputs=[ImageInput(demo_url)], return_task=True, wait_task=False) tasks.append(task) + print("Waiting for the results") pending_tasks, success_tasks, error_tasks = client.Task.batch_wait(tasks=tasks, timeout=timeout) if pending_tasks: @@ -323,9 +329,6 @@ def demo_batch_tasks(): for pos, success in success_tasks: assert(tasks[pos].pk == success.pk) - - - ########### # Helpers # ########### @@ -362,4 +365,3 @@ def display_inference_tensor(result): if __name__ == '__main__': demo() - demo_batch_tasks() From b2196750d98c01ff3acc095bb95e1bf65dfceb81 Mon Sep 17 00:00:00 2001 From: Vincent Delaitre Date: Tue, 29 Jan 2019 07:55:26 +0100 Subject: [PATCH 05/17] Return the client --- deepomatic/api/demo.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deepomatic/api/demo.py b/deepomatic/api/demo.py index c85793c..3a3fc82 100644 --- a/deepomatic/api/demo.py +++ b/deepomatic/api/demo.py @@ -329,6 +329,9 @@ def demo(): for pos, success in success_tasks: assert(tasks[pos].pk == success.pk) + # Return the client to perform assertions on e2e tests + return client + ########### # Helpers # ########### From b8060975918a9da1aa6e76a28cb273c70695137c Mon Sep 17 00:00:00 2001 From: Vincent Delaitre Date: Tue, 29 Jan 2019 08:02:23 +0100 Subject: [PATCH 06/17] Pass client to demo --- deepomatic/api/demo.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/deepomatic/api/demo.py b/deepomatic/api/demo.py index 3a3fc82..bd4a184 100644 --- a/deepomatic/api/demo.py +++ b/deepomatic/api/demo.py @@ -15,7 +15,7 @@ demo_url = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" -def demo(): +def demo(client=None): """ Our REST client works by exposing resources. A resource usually has the following synchronous methods: @@ -39,9 +39,10 @@ def demo(): # and just call `client = Client()` # # Here we actually use a mix of those two methods to illustrate: - app_id = os.getenv('DEEPOMATIC_APP_ID') - api_key = os.getenv('DEEPOMATIC_API_KEY') - client = Client(app_id, api_key) # this would be equivalent to using `Client()` in this case. + if client is None: + app_id = os.getenv('DEEPOMATIC_APP_ID') + api_key = os.getenv('DEEPOMATIC_API_KEY') + client = Client(app_id, api_key) # this would be equivalent to using `Client()` in this case. ################### # Public networks # @@ -329,9 +330,6 @@ def demo(): for pos, success in success_tasks: assert(tasks[pos].pk == success.pk) - # Return the client to perform assertions on e2e tests - return client - ########### # Helpers # ########### From a0da4c28700f165f6223f364e57f3e7116df6e89 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Fri, 1 Feb 2019 14:55:27 +0100 Subject: [PATCH 07/17] Put back demo in root + added pytests tests --- deepomatic/api/demo.py => demo.py | 0 deploy/Dockerfile | 15 ++ deploy/install.sh | 7 + dmake.yml | 10 +- tests/test_client.py | 255 ++++++++++++++++++++++++++++++ 5 files changed, 285 insertions(+), 2 deletions(-) rename deepomatic/api/demo.py => demo.py (100%) create mode 100644 deploy/Dockerfile create mode 100755 deploy/install.sh create mode 100644 tests/test_client.py diff --git a/deepomatic/api/demo.py b/demo.py similarity index 100% rename from deepomatic/api/demo.py rename to demo.py diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..aafb05b --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,15 @@ +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} as builder + +WORKDIR /app +COPY . . +RUN python setup.py bdist_wheel + + +FROM ${BASE_IMAGE} as runtime + +# copy egg +COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ + +RUN pip install /tmp/deepomatic_api-*.whl diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100755 index 0000000..d91e954 --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +apt-get update && apt-get install -y build-essential +pip install -r requirements.txt +pip install pytest==4.0.2 # for testing diff --git a/dmake.yml b/dmake.yml index a37bf0b..498feef 100644 --- a/dmake.yml +++ b/dmake.yml @@ -6,6 +6,7 @@ env: variables: DEEPOMATIC_APP_ID: ${TEST_CLIENTS_APP_ID} DEEPOMATIC_API_KEY: ${TEST_CLIENTS_API_KEY} + DEEPOMATIC_API_URL: ${DEEPOMATIC_API_URL:-https://api.deepomatic.com} branches: master: source: ${DEEPOMATIC_CONFIG_DIR}/prod.sh @@ -20,6 +21,10 @@ docker: name: deepomatic-client-python variant: 2.7 root_image: python:2.7 + copy_files: + - requirements.txt + install_scripts: + - deploy/install.sh # Nothing changes comparing to above, except 'variant' and 'root_image' - <<: *base_image variant: 3.4 @@ -31,6 +36,7 @@ docker: variant: 3.6 root_image: python:3.6 + services: - service_name: client config: @@ -41,7 +47,7 @@ services: - 3.4 - 3.5 - 3.6 - tests: commands: - - python -c "from deepomatic.api.demo import demo; demo()" + - python -m pytest --junit-xml=junit.xml -vv + junit_report: junit.xml diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..67a1f49 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,255 @@ +import os +import base64 +import tarfile +import pytest +import tempfile +import hashlib +import shutil +import requests +from deepomatic.api.client import Client +from deepomatic.api.inputs import ImageInput +from pytest_voluptuous import S +from voluptuous.validators import All, Length, Range, Any +import six + +import logging +logging.basicConfig(level='INFO') +logger = logging.getLogger(__name__) + +DEMO_URL = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" + + +def ExactLen(nb): + return Length(min=nb, max=nb) + + +def download_file(url): + _, ext = os.path.splitext(url) + filename = os.path.join(tempfile.gettempdir(), + hashlib.sha1(url.encode()).hexdigest() + ext) + if os.path.exists(filename): # avoid redownloading + return filename + r = requests.get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + return filename + + +@pytest.fixture(scope='session') +def client(): + api_host = os.environ['DEEPOMATIC_API_URL'] + app_id = os.environ['DEEPOMATIC_APP_ID'] + api_key = os.environ['DEEPOMATIC_API_KEY'] + yield Client(app_id, api_key, host=api_host) + + +@pytest.fixture(scope='session') +def custom_network(client): + deploy_prototxt = download_file('https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt') + snapshot_caffemodel = download_file('http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel') + extract_dir = tempfile.gettempdir() + mean_file = os.path.join(extract_dir, 'imagenet_mean.binaryproto') + if not os.path.exists(mean_file): + archive = download_file('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz') + tar = tarfile.open(archive, "r:gz") + tar.extractall(path=extract_dir) + tar.close() + + preprocessing = { + "inputs": [ + { + "tensor_name": "data", + "image": { + "color_channels": "BGR", + "target_size": "224x224", + "resize_type": "SQUASH", + "mean_file": "mean.binaryproto", + "dimension_order": "NCHW", + "pixel_scaling": 255.0, + "data_type": "FLOAT32" + } + } + ], + "batched_output": True + } + + files = { + 'deploy.prototxt': deploy_prototxt, + 'snapshot.caffemodel': snapshot_caffemodel, + 'mean.binaryproto': mean_file + } + + network = client.Network.create(name="My first network", + framework='nv-caffe-0.x-mod', + preprocessing=preprocessing, + files=files) + assert network['id'] + assert network['name'] == 'My first network' + data = network.data() + assert 'update_date' in data + yield network + network.delete() + + +def check_first_prediction(first_label_name, first_score_range): + def check(predicted): + assert predicted[0]['label_name'] == first_label_name + assert predicted[0]['score'] > first_score_range + return predicted + return check + + +def check_score_threshold(is_predicted): + def check(predicted): + for pred in predicted: + if is_predicted: + assert pred['score'] >= pred['threshold'] + else: + assert pred['score'] < pred['threshold'] + return predicted + return check + + +def prediction_schema(exact_len, *args): + return All([{ + 'threshold': float, + 'label_id': int, + 'score': float, + 'label_name': All(Any(*six.string_types)), + }], ExactLen(exact_len), *args) + + +def inference_schema(predicted_len, discarded_len, first_label, first_score): + return S({ + 'outputs': All([{ + 'labels': { + 'predicted': prediction_schema(predicted_len, check_first_prediction(first_label, first_score), + check_score_threshold(is_predicted=True)), + 'discarded': prediction_schema(discarded_len, check_score_threshold(is_predicted=False)) + } + }], ExactLen(1)), + }) + + +class TestClient(object): + def test_retrieve_network(self, client): + network = client.Network.retrieve('imagenet-inception-v1') + assert network['id'] + data = network.data() + assert 'name' in data + assert 'description' in data + assert 'create_date' in data + assert 'update_date' in data + + def test_list_specs(self, client): + specs = client.RecognitionSpec.list(public=True) + assert specs.count() > 0 + for spec in specs: + assert spec['id'] + data = spec.data() + assert 'name' in data + assert 'description' in data + assert 'update_date' in data + + result = client.RecognitionSpec.list(public=True).data() + assert len(result['results']) > 0 + assert result['count'] > 0 + + def test_retrieve_spec(self, client): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + assert spec['id'] + data = spec.data() + assert 'name' in data + assert 'description' in data + assert 'update_date' in data + + def test_inference_spec(self, client): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + first_result = spec.inference(inputs=[ImageInput(DEMO_URL)], show_discarded=True, max_predictions=3) + + assert inference_schema(1, 2, 'golden retriever', 0.9) == first_result + + f = open(download_file(DEMO_URL), 'rb') + result = spec.inference(inputs=[ImageInput(f)], show_discarded=True, max_predictions=3) + assert result == first_result + + f.seek(0) + binary_data = f.read() + result = spec.inference(inputs=[ImageInput(binary_data, encoding="binary")], show_discarded=True, max_predictions=3) + assert result == first_result + + b64 = base64.b64encode(binary_data) + result = spec.inference(inputs=[ImageInput(b64, encoding="base64")], show_discarded=True, max_predictions=3) + assert result == first_result + + def test_create_custom_reco_and_infer(self, client, custom_network): + custom_network.update(description="I had forgotten the description") + + outputs = client.RecognitionSpec.retrieve('imagenet-inception-v1')['outputs'] + + spec = client.RecognitionSpec.create(name="My recognition model", outputs=outputs) + + version = client.RecognitionVersion.create(network_id=custom_network['id'], spec_id=spec['id'], post_processings=[ + { + "classification": { + "output_tensor": "prob", + } + } + + ]) + assert version['id'] + data = version.data() + assert data['network_id'] == custom_network['id'] + assert 'post_processings' in data + + client.Task.retrieve(custom_network['task_id']).wait() + + result = spec.inference(inputs=[ImageInput(DEMO_URL)], show_discarded=False, max_predictions=3) + assert inference_schema(1, 0, 'golden retriever', 0.9) == result + + result = version.inference(inputs=[ImageInput(DEMO_URL, bbox={"xmin": 0.1, "ymin": 0.1, "xmax": 0.9, "ymax": 0.9})], show_discarded=True, max_predictions=3) + assert inference_schema(1, 2, 'golden retriever', 0.9) == result + + versions = spec.versions() + assert versions.count() > 0 + data = versions.data() + assert len(data['results']) > 0 + assert data['count'] > 0 + + task = spec.inference(inputs=[ImageInput(DEMO_URL)], return_task=True, wait_task=False) + task_id = task.pk + tasks = client.Task.list(task_ids=[task_id]) + tasks = list(tasks) # convert iterables to list + assert len(tasks) == 1 + task = tasks[0] + assert task['status'] in ['pending', 'success'] + assert task['error'] is None + task.wait() + assert task['status'] == 'success' + assert inference_schema(1, 0, 'golden retriever', 0.9) == task['data'] + + def test_batch_wait(self, client): + spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + tasks = [] + timeout = 30 + nb_inference = 20 + + for i in range(nb_inference): + task = spec.inference(inputs=[ImageInput(DEMO_URL)], return_task=True, wait_task=False) + tasks.append(task) + + pending_tasks, success_tasks, error_tasks = client.Task.batch_wait(tasks=tasks, timeout=timeout) + assert len(pending_tasks) == 0 + assert len(error_tasks) == 0 + assert len(success_tasks) == len(tasks) + + # pending_tasks, error_tasks and success_tasks contains the original offset of the input parameter tasks + for pos, pending in pending_tasks: + assert(tasks[pos].pk == pending.pk) + for pos, err in error_tasks: + assert(tasks[pos].pk == err.pk) + for pos, success in success_tasks: + assert(tasks[pos].pk == success.pk) + assert inference_schema(1, 0, 'golden retriever', 0.9) == success['data'] From d7a2b37507d8a4108bc72903f2548db762fc433d Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Fri, 1 Feb 2019 15:51:27 +0100 Subject: [PATCH 08/17] Running demo and tests --- .travis.yml | 5 ++++- deploy/Dockerfile | 1 + dmake.yml | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f00aef..4f3ba01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,10 @@ python: install: - python setup.py bdist_wheel - pip install dist/deepomatic_api-*.whl + - mkdir samples + - cp -r tests demo.py samples/ # command to run tests script: - - python -c "from deepomatic.api.demo import demo; demo()" + - python -m pytest --junit-xml=junit.xml -vv samples/tests + - python samples/demo.py diff --git a/deploy/Dockerfile b/deploy/Dockerfile index aafb05b..e5805cb 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -11,5 +11,6 @@ FROM ${BASE_IMAGE} as runtime # copy egg COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ +COPY --from=builder /app/demo.py /app/tests /samples/ RUN pip install /tmp/deepomatic_api-*.whl diff --git a/dmake.yml b/dmake.yml index 498feef..41250b8 100644 --- a/dmake.yml +++ b/dmake.yml @@ -49,5 +49,6 @@ services: - 3.6 tests: commands: - - python -m pytest --junit-xml=junit.xml -vv + - python -m /samples/pytest --junit-xml=junit.xml -vv + - python /samples/demo.py junit_report: junit.xml From e9afd121eae8eee58992616e0f8d626c267482d6 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Fri, 1 Feb 2019 16:15:24 +0100 Subject: [PATCH 09/17] coverage --- .travis.yml | 1 + Dockerfile | 15 --------------- deploy/Dockerfile | 3 ++- deploy/install.sh | 2 +- dmake.yml | 9 ++++++--- 5 files changed, 10 insertions(+), 20 deletions(-) delete mode 100644 Dockerfile diff --git a/.travis.yml b/.travis.yml index 4f3ba01..c2566a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,6 @@ install: # command to run tests script: + - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 # for testing - python -m pytest --junit-xml=junit.xml -vv samples/tests - python samples/demo.py diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index aafb05b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG BASE_IMAGE - -FROM ${BASE_IMAGE} as builder - -WORKDIR /app -COPY . . -RUN python setup.py bdist_wheel - - -FROM ${BASE_IMAGE} as runtime - -# copy egg -COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ - -RUN pip install /tmp/deepomatic_api-*.whl diff --git a/deploy/Dockerfile b/deploy/Dockerfile index e5805cb..906e441 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -11,6 +11,7 @@ FROM ${BASE_IMAGE} as runtime # copy egg COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ -COPY --from=builder /app/demo.py /app/tests /samples/ +COPY --from=builder /app/demo.py /samples/ +COPY --from=builder /app/tests /samples/tests RUN pip install /tmp/deepomatic_api-*.whl diff --git a/deploy/install.sh b/deploy/install.sh index d91e954..2c05442 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -4,4 +4,4 @@ set -e apt-get update && apt-get install -y build-essential pip install -r requirements.txt -pip install pytest==4.0.2 # for testing +pip install pytest==4.0.2 pytest-cov==2.6.1 pytest-voluptuous==1.1.0 # for testing diff --git a/dmake.yml b/dmake.yml index 41250b8..0a5528f 100644 --- a/dmake.yml +++ b/dmake.yml @@ -41,7 +41,9 @@ services: - service_name: client config: docker_image: - build: . + build: + context: . + dockerfile: deploy/Dockerfile base_image_variant: - 2.7 - 3.4 @@ -49,6 +51,7 @@ services: - 3.6 tests: commands: - - python -m /samples/pytest --junit-xml=junit.xml -vv + - pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /samples/tests - python /samples/demo.py - junit_report: junit.xml + # junit_report: junit.xml + # cobertura_report: coverage.xml From bb6b1178a4246d9076f2ef96385228277b5ac73f Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Fri, 1 Feb 2019 17:02:34 +0100 Subject: [PATCH 10/17] Better demo --- demo.py | 81 +++++++++++++++++++++++--------------------- tests/test_client.py | 4 +-- 2 files changed, 45 insertions(+), 40 deletions(-) diff --git a/demo.py b/demo.py index bd4a184..764b818 100644 --- a/demo.py +++ b/demo.py @@ -1,19 +1,22 @@ import os -import sys import json import base64 import tarfile +import logging +import tempfile +import shutil +import hashlib +import requests from deepomatic.api.client import Client from deepomatic.api.inputs import ImageInput -if sys.version_info >= (3, 0): - from urllib.request import urlretrieve -else: - from urllib import urlretrieve - demo_url = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" +logging.basicConfig(level='INFO', + format='[%(levelname)s %(name)s %(asctime)s %(process)d %(thread)d %(filename)s:%(lineno)s] %(message)s') +logger = logging.getLogger(__name__) + def demo(client=None): """ @@ -57,7 +60,7 @@ def demo(client=None): (see below). Here, we retrieve a public network named 'imagenet-inception-v1' """ network = client.Network.retrieve('imagenet-inception-v1') - print(network) + logger.info(network) ############################# # Public recognition models # @@ -110,7 +113,7 @@ def demo(client=None): '.inference()' also support when you pass a single input instead of a list of inputs. Here, it takes a file pointer as input. """ - file = open(download(demo_url, '/tmp/img.jpg'), 'rb') + file = open(download_file(demo_url), 'rb') result = spec.inference(inputs=[ImageInput(file)], show_discarded=True, max_predictions=3) pretty_print_json(result) @@ -150,13 +153,13 @@ def demo(client=None): - snapshot_caffemodel: the file that describe the learned parameters of the model - mean_file: the file that stores the mean image that needs to be substracted from the input """ - deploy_prototxt = download('https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt', '/tmp/deploy.prototxt') - snapshot_caffemodel = download('http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel', '/tmp/snapshot.caffemodel') - mean_file = '/tmp/imagenet_mean.binaryproto' + deploy_prototxt = download_file('https://raw.githubusercontent.com/BVLC/caffe/master/models/bvlc_googlenet/deploy.prototxt') + snapshot_caffemodel = download_file('http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel') + mean_file = os.path.join(tempfile.gettempdir(), 'imagenet_mean.binaryproto') if not os.path.exists(mean_file): - archive = download('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz', '/tmp/caffe_ilsvrc12.tar.gz') + archive = download_file('http://dl.caffe.berkeleyvision.org/caffe_ilsvrc12.tar.gz') tar = tarfile.open(archive, "r:gz") - tar.extractall(path='/tmp/') + tar.extractall(path=tempfile.gettempdir()) tar.close() else: print_comment("Skipping download of mean file: {}".format(mean_file)) @@ -272,16 +275,16 @@ def demo(client=None): And this current version can be used to run inference for the spec directly """ result = spec.inference(inputs=[ImageInput(demo_url)], show_discarded=True, max_predictions=3) - print(result) + logger.info(result) print_header("Run inference on specific version with a bounding box") result = version.inference(inputs=[ImageInput(demo_url, bbox={"xmin": 0.1, "ymin": 0.1, "xmax": 0.9, "ymax": 0.9})], show_discarded=True, max_predictions=3) - print(result) + logger.info(result) """ Show all versions of a spec """ - print(spec.versions()) + logger.info(spec.versions()) """ Test the possibility of getting multiple tasks at the same time @@ -289,7 +292,7 @@ def demo(client=None): task = spec.inference(inputs=[ImageInput(demo_url)], return_task=True, wait_task=False) task_id = task.pk tasks = client.Task.list(task_ids=[task_id]) - print(tasks) + logger.info(tasks) print_header("Delete networks and recognition models") """ @@ -307,20 +310,20 @@ def demo(client=None): timeout = 30 nb_inference = 20 - print("Pushing %d inferences" % nb_inference) + logger.info("Pushing %d inferences" % nb_inference) for i in range(nb_inference): task = spec.inference(inputs=[ImageInput(demo_url)], return_task=True, wait_task=False) tasks.append(task) - print("Waiting for the results") + logger.info("Waiting for the results") pending_tasks, success_tasks, error_tasks = client.Task.batch_wait(tasks=tasks, timeout=timeout) if pending_tasks: - print("Warning: %d tasks are still pending after %s seconds" % (len(pending_tasks), timeout)) + logger.warning("%d tasks are still pending after %s seconds" % (len(pending_tasks), timeout)) if error_tasks: - print("Warning: %d tasks are in error" % len(error_tasks)) - print(pending_tasks) - print(error_tasks) - print(success_tasks) + logger.warning("%d tasks are in error" % len(error_tasks)) + logger.info(pending_tasks) + logger.info(error_tasks) + logger.info(success_tasks) # pending_tasks, error_tasks and success_tasks contains the original offset of the input parameter tasks for pos, pending in pending_tasks: @@ -334,29 +337,31 @@ def demo(client=None): # Helpers # ########### -def download(url, local_path): - if not os.path.isfile(local_path): - print("Downloading {} to {}".format(url, local_path)) - urlretrieve(url, local_path) - if url.endswith('.tar.gz'): - tar = tarfile.open(local_path, "r:gz") - tar.extractall(path='/tmp/') - tar.close() - else: - print("Skipping download of {} to {}: file already exist".format(url, local_path)) - return local_path +def download_file(url): + _, ext = os.path.splitext(url) + filename = os.path.join(tempfile.gettempdir(), + hashlib.sha1(url.encode()).hexdigest() + ext) + if os.path.exists(filename): # avoid redownloading + logger.info("Skipping download of {}: file already exist in ".format(url, filename)) + return filename + r = requests.get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as f: + r.raw.decode_content = True + shutil.copyfileobj(r.raw, f) + return filename def print_header(text): - print("\n{}".format(text)) + logger.info("\n{}".format(text)) def print_comment(text): - print("--> " + text) + logger.info("--> " + text) def pretty_print_json(data): - print(json.dumps(data, indent=4, separators=(',', ': '))) + logger.info(json.dumps(data, indent=4, separators=(',', ': '))) def display_inference_tensor(result): diff --git a/tests/test_client.py b/tests/test_client.py index 67a1f49..9af251c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -9,7 +9,7 @@ from deepomatic.api.client import Client from deepomatic.api.inputs import ImageInput from pytest_voluptuous import S -from voluptuous.validators import All, Length, Range, Any +from voluptuous.validators import All, Length, Any import six import logging @@ -39,7 +39,7 @@ def download_file(url): @pytest.fixture(scope='session') def client(): - api_host = os.environ['DEEPOMATIC_API_URL'] + api_host = os.getenv('DEEPOMATIC_API_URL') app_id = os.environ['DEEPOMATIC_APP_ID'] api_key = os.environ['DEEPOMATIC_API_KEY'] yield Client(app_id, api_key, host=api_host) From e0fe08c878eac224e2064fc51ce4b1beb350610e Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Fri, 1 Feb 2019 17:05:59 +0100 Subject: [PATCH 11/17] formatting --- demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demo.py b/demo.py index 764b818..7045d04 100644 --- a/demo.py +++ b/demo.py @@ -353,7 +353,7 @@ def download_file(url): return filename def print_header(text): - logger.info("\n{}".format(text)) + logger.info("**** {} ****".format(text)) def print_comment(text): From 0f7bad7b022e0fcdf213bbca5b2f24c91511861e Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Fri, 1 Feb 2019 22:19:05 +0100 Subject: [PATCH 12/17] removed public net test --- tests/test_client.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 9af251c..6bf1773 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -86,9 +86,12 @@ def custom_network(client): preprocessing=preprocessing, files=files) assert network['id'] - assert network['name'] == 'My first network' data = network.data() + assert network['name'] == 'My first network' + assert 'description' in data + assert 'create_date' in data assert 'update_date' in data + yield network network.delete() @@ -134,15 +137,6 @@ def inference_schema(predicted_len, discarded_len, first_label, first_score): class TestClient(object): - def test_retrieve_network(self, client): - network = client.Network.retrieve('imagenet-inception-v1') - assert network['id'] - data = network.data() - assert 'name' in data - assert 'description' in data - assert 'create_date' in data - assert 'update_date' in data - def test_list_specs(self, client): specs = client.RecognitionSpec.list(public=True) assert specs.count() > 0 @@ -167,6 +161,7 @@ def test_retrieve_spec(self, client): def test_inference_spec(self, client): spec = client.RecognitionSpec.retrieve('imagenet-inception-v1') + first_result = spec.inference(inputs=[ImageInput(DEMO_URL)], show_discarded=True, max_predictions=3) assert inference_schema(1, 2, 'golden retriever', 0.9) == first_result @@ -185,6 +180,11 @@ def test_inference_spec(self, client): assert result == first_result def test_create_custom_reco_and_infer(self, client, custom_network): + + # test query by id + network = client.Network.retrieve(custom_network['id']) + assert network['name'] + custom_network.update(description="I had forgotten the description") outputs = client.RecognitionSpec.retrieve('imagenet-inception-v1')['outputs'] From e65d4de8c35a10d4cf979f2f551ede846db0a832 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Mon, 4 Feb 2019 11:41:57 +0100 Subject: [PATCH 13/17] add test headers --- deepomatic/api/http_helper.py | 2 +- tests/test_client.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 2a63809..b91a3f3 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -69,7 +69,7 @@ def __init__(self, app_id, api_key, verify, host, version, check_query_parameter 'platform': platform.platform() } - self.user_agent = 'deepomatic-client-python/{package_version} requests/{requests_version} python/{python_version} platform/{platform}\ + self.user_agent = 'deepomatic-api/{package_version} requests/{requests_version} python/{python_version} platform/{platform}\ '.format(**user_agent_params) if user_agent_suffix: self.user_agent += ' ' + user_agent_suffix diff --git a/tests/test_client.py b/tests/test_client.py index 6bf1773..7b86a76 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -137,6 +137,25 @@ def inference_schema(predicted_len, discarded_len, first_label, first_score): class TestClient(object): + + def test_headers(self, client): + http_helper = client.http_helper + session_headers = http_helper.session.headers + assert session_headers['User-Agent'].startswith('deepomatic-api/') + assert 'platform/' in session_headers['User-Agent'] + assert 'python/' in session_headers['User-Agent'] + assert session_headers['X-APP-ID'] + assert session_headers['X-API-KEY'] + + headers = http_helper.setup_headers(headers={'Hello': 'World'}, + content_type='application/json') + assert headers == { + 'Hello': 'World', + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + def test_list_specs(self, client): specs = client.RecognitionSpec.list(public=True) assert specs.count() > 0 From 097948621131f81516ce67366c6c80facf537c58 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Mon, 4 Feb 2019 12:02:51 +0100 Subject: [PATCH 14/17] Fixed workdir --- .dockerignore | 2 ++ .gitignore | 2 ++ .travis.yml | 8 ++++---- deploy/Dockerfile | 6 ++++-- dmake.yml | 8 ++++---- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9b4052e..1c9530c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,8 @@ Jenkinsfile build/ dist/ +**/*coverage.xml +**/*junit.xml **/*.pyc **/*~ **/.#* diff --git a/.gitignore b/.gitignore index 63c4c7d..156939b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .dmake/ DMakefile +*coverage.xml +*junit.xml *.pyc *~ .#* diff --git a/.travis.yml b/.travis.yml index c2566a6..81aa8f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,11 +9,11 @@ python: install: - python setup.py bdist_wheel - pip install dist/deepomatic_api-*.whl - - mkdir samples - - cp -r tests demo.py samples/ + - mkdir app + - cp -r tests demo.py app/ # command to run tests script: - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 # for testing - - python -m pytest --junit-xml=junit.xml -vv samples/tests - - python samples/demo.py + - python -m pytest --junit-xml=junit.xml -vv app/tests + - python app/demo.py diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 906e441..0aba4b5 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -9,9 +9,11 @@ RUN python setup.py bdist_wheel FROM ${BASE_IMAGE} as runtime +WORKDIR /app + # copy egg COPY --from=builder /app/dist/deepomatic_api-*.whl /tmp/ -COPY --from=builder /app/demo.py /samples/ -COPY --from=builder /app/tests /samples/tests +COPY --from=builder /app/demo.py /app/ +COPY --from=builder /app/tests /app/tests RUN pip install /tmp/deepomatic_api-*.whl diff --git a/dmake.yml b/dmake.yml index 0a5528f..f9f6039 100644 --- a/dmake.yml +++ b/dmake.yml @@ -51,7 +51,7 @@ services: - 3.6 tests: commands: - - pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /samples/tests - - python /samples/demo.py - # junit_report: junit.xml - # cobertura_report: coverage.xml + - pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /app/tests + - python /app/demo.py + junit_report: junit.xml + cobertura_report: coverage.xml From ee7f03a985be8871c167bbb2808564e925e58ab3 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Mon, 4 Feb 2019 12:04:17 +0100 Subject: [PATCH 15/17] cleanup --- tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index 7b86a76..8c76db5 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -120,7 +120,7 @@ def prediction_schema(exact_len, *args): 'threshold': float, 'label_id': int, 'score': float, - 'label_name': All(Any(*six.string_types)), + 'label_name': Any(*six.string_types), }], ExactLen(exact_len), *args) From 41166590e431b30e9968f5535a858f605bf4282d Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Mon, 4 Feb 2019 14:38:56 +0100 Subject: [PATCH 16/17] Customizable log level --- .travis.yml | 4 ++-- demo.py | 2 +- dmake.yml | 4 ++-- tests/test_client.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 81aa8f2..39e9445 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,5 +15,5 @@ install: # command to run tests script: - pip install pytest==4.0.2 pytest-voluptuous==1.1.0 # for testing - - python -m pytest --junit-xml=junit.xml -vv app/tests - - python app/demo.py + - LOG_LEVEL=DEBUG python -m pytest --junit-xml=junit.xml -vv app/tests + - LOG_LEVEL=DEBUG python app/demo.py diff --git a/demo.py b/demo.py index 7045d04..9b8f609 100644 --- a/demo.py +++ b/demo.py @@ -13,7 +13,7 @@ demo_url = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" -logging.basicConfig(level='INFO', +logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO'), format='[%(levelname)s %(name)s %(asctime)s %(process)d %(thread)d %(filename)s:%(lineno)s] %(message)s') logger = logging.getLogger(__name__) diff --git a/dmake.yml b/dmake.yml index f9f6039..f874d12 100644 --- a/dmake.yml +++ b/dmake.yml @@ -51,7 +51,7 @@ services: - 3.6 tests: commands: - - pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /app/tests - - python /app/demo.py + - LOG_LEVEL=DEBUG pytest --junit-xml=junit.xml --cov=. --cov-report=xml:coverage.xml --cov-report html:cover -vv /app/tests + - LOG_LEVEL=DEBUG python /app/demo.py junit_report: junit.xml cobertura_report: coverage.xml diff --git a/tests/test_client.py b/tests/test_client.py index 8c76db5..d66aafc 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -13,7 +13,7 @@ import six import logging -logging.basicConfig(level='INFO') +logging.basicConfig(level=os.getenv('LOG_LEVEL', 'INFO')) logger = logging.getLogger(__name__) DEMO_URL = "https://static.deepomatic.com/resources/demos/api-clients/dog1.jpg" From a5ff9e707a5b8553f724fbce074ec153f5a36de5 Mon Sep 17 00:00:00 2001 From: Hugo Maingonnat Date: Mon, 4 Feb 2019 15:06:44 +0100 Subject: [PATCH 17/17] pepe --- tests/test_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index d66aafc..5ae30e8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -155,7 +155,6 @@ def test_headers(self, client): 'Accept': 'application/json' } - def test_list_specs(self, client): specs = client.RecognitionSpec.list(public=True) assert specs.count() > 0