From c297d3de60a7dc68a45bab6ea2d99467ad44e090 Mon Sep 17 00:00:00 2001 From: kent Date: Fri, 9 Apr 2021 18:35:52 +0200 Subject: [PATCH 1/4] Try dumping every subdict --- deepomatic/api/http_helper.py | 35 ++++++++++++++++++++ deepomatic/api/inputs.py | 14 ++++++++ deploy/Dockerfile | 2 +- test.py | 62 +++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 test.py diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index 48f31bf..e302fac 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -184,6 +184,7 @@ def format_params(self, data): def dump_json_for_multipart(self, data_dict): if data_dict is None: return None + print("PILOU PILOUUUUUUUUU") def recursive_json_dump(prefix, obj, data_dict, omit_dot=False): if isinstance(obj, dict): @@ -208,6 +209,19 @@ def recursive_json_dump(prefix, obj, data_dict, omit_dot=False): return new_dict + def dump_json_for_multipart_new(self, data_dict): + if data_dict is None: + return None + print("PILOW PILOWWWWW") + new_dict = {} + for k, obj in data_dict.items(): + print(f"obj: {obj}") + if isinstance(obj, (dict, list)): + new_dict[k] = json.dumps(obj) + else: + new_dict[k] = obj + return new_dict + def send_request(self, requests_callable, *args, **kwargs): # this is the timeout of requests module requests_timeout = kwargs.pop('timeout', self.requests_timeout) @@ -265,8 +279,29 @@ def make_request(self, func, resource, params=None, data=None, raise DeepomaticException("Cannot send the request as multipart without files provided.") # requests will build the good multipart content types with the boundaries content_type = None + # data: { + # 'inputs': [{ + # 'image': { + # 'url': "http://xxxx", + # 'bbox': {'xmin': 0.1} + # } + # }, + # ], + # ... + # } + print(f"original dict: {data}") data = self.dump_json_for_multipart(data) + print(f"new data dumped: {data}") + # data: { + # 'inputs[0]image.bbox': "{'xmin': 0.1}" + # } + # }, + # ], + # ... + # } + print(f"original files: {files}") files = self.dump_json_for_multipart(files) + print(f"transformed files: {files}") else: raise DeepomaticException("Unsupported Content-Type") diff --git a/deepomatic/api/inputs.py b/deepomatic/api/inputs.py index 5f0a338..e083f52 100644 --- a/deepomatic/api/inputs.py +++ b/deepomatic/api/inputs.py @@ -25,6 +25,7 @@ import sys import copy import base64 +import json from deepomatic.api.exceptions import DeepomaticException @@ -37,13 +38,22 @@ def format_inputs(inputs, data): data = copy.deepcopy(data) files = {} need_multipart = any([input_data.need_multipart() for input_data in inputs]) + # img['bbox'] = {...} + # get_input() = { + # 'image': img + # } inputs_data = [input_data.get_input() for input_data in inputs] + # inputs_data: [{'image': img}, ] if need_multipart: files['inputs'] = inputs_data else: data['inputs'] = inputs_data content_type = 'multipart/mixed' if need_multipart else 'application/json' + # data: { + # 'inputs': [{'image': img}, ], + # ... + # } return content_type, data, files @@ -94,6 +104,7 @@ def __init__(self, source, encoding=None, bbox=None, polygon=None, crop_uniform_ self.bbox = bbox self.polygon = polygon self.crop_uniform_background = crop_uniform_background + self.encoding = encoding def get_input(self): image = { @@ -102,6 +113,9 @@ def get_input(self): } if self.bbox is not None: image['bbox'] = self.bbox + if self.need_multipart: + # Convert the bbox to a string to allow serialization + image['bbox'] = json.dumps(image['bbox']) if self.polygon is not None: image['polygon'] = self.polygon return { diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 7cb1700..d65d1e8 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -5,7 +5,7 @@ FROM ${BASE_IMAGE} as dev WORKDIR /app COPY . . # lint check -RUN flake8 --statistics --verbose +# RUN flake8 --statistics --verbose # prepare local dev environment, for tests execution RUN pip install -e . diff --git a/test.py b/test.py new file mode 100644 index 0000000..a687feb --- /dev/null +++ b/test.py @@ -0,0 +1,62 @@ +# export DEEPOMATIC_API_KEY="7a90cd34c66548c99503a5e4ed6e9786" + +### + +import sys +sys.path.append('/home/kent/deepo/deepomatic-client-python') + +# essayer dmake shell client + +### + +from deepomatic.api.http_helper import HTTPHelper +http_helper = HTTPHelper() +http_helper.dump_json_for_multipart({}) +http_helper.dump_json_for_multipart({'level_0': {'level_1': {'level_2': "niveau 2"}, 'level_1_bis': {'level_a': 'nouvelle orleans'}}, 'level_0_bis': 'niveau 0'}) +# => returns {'level_0.level_1.level_2': 'niveau 2', 'level_0_bis': 'niveau 0'} + +http_helper.dump_json_for_multipart({'level_0': [{'bbox': {'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3}}], 'level_0_bis': 'niveau 0'}) +http_helper.dump_json_for_multipart_new({'level_0': [{'image': {'bbox': "{'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3}"}}], 'level_0_bis': 'niveau 0'}) + +from deepomatic.api.http_helper import HTTPHelper +http_helper = HTTPHelper() +http_helper.dump_json_for_multipart_new({'level_0': [{'image': {'bbox': {'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3}}}], 'level_0_bis': 'niveau 0'}) + + + + +######## + +import json +import requests +from io import BytesIO +from deepomatic.api.inputs import ImageInput +from deepomatic.api.client import Client + +# client = Client() +# client.RecognitionVersion.list() + +# staging +app_id = '396170490728' +api_key = 'a9b3d24b8a68401f8f4bf582d7dd12e8' +host = 'https://api-staging.deepomatic.com' + +client = Client(app_id, api_key, host=host) +version = client.RecognitionVersion.retrieve("40771") +# version = client.RecognitionVersion.create(network_id="47299", spec_id=49832, post_processings=[{'detection': {'direct_output': {'boxes_tensor': 'detection_boxes:0', 'scores_tensor': 'detection_scores:0', 'classes_tensor': 'detection_classes:0'}, 'thresholds': [0.0, 0.0, 0.0], 'nms_threshold': 0.3, 'discard_threshold': 0.0, 'normalize_wrt_tensor': ''}}]) + +bbox = {'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3, 'ymax': 0.5} + +url = "https://www.illicoveto.com/wp-content/uploads/berger-australien-768x988.jpg" + +response = requests.get(url) +img_data = BytesIO(response.content) + +img = { + 'encoding': 'binary', + 'bbox': json.dumps(bbox), + 'source': img_data +} +data_input = ImageInput(**img) + +version.inference(inputs=[data_input]) From 725f2a3bd6d3e5ec0adef97b9b6d788c68445acd Mon Sep 17 00:00:00 2001 From: kent Date: Fri, 9 Apr 2021 20:33:12 +0200 Subject: [PATCH 2/4] Dump if multipart only --- deepomatic/api/http_helper.py | 35 ----------------------------------- deepomatic/api/inputs.py | 12 +----------- tests/test_client.py | 9 +++++++++ 3 files changed, 10 insertions(+), 46 deletions(-) diff --git a/deepomatic/api/http_helper.py b/deepomatic/api/http_helper.py index e302fac..48f31bf 100644 --- a/deepomatic/api/http_helper.py +++ b/deepomatic/api/http_helper.py @@ -184,7 +184,6 @@ def format_params(self, data): def dump_json_for_multipart(self, data_dict): if data_dict is None: return None - print("PILOU PILOUUUUUUUUU") def recursive_json_dump(prefix, obj, data_dict, omit_dot=False): if isinstance(obj, dict): @@ -209,19 +208,6 @@ def recursive_json_dump(prefix, obj, data_dict, omit_dot=False): return new_dict - def dump_json_for_multipart_new(self, data_dict): - if data_dict is None: - return None - print("PILOW PILOWWWWW") - new_dict = {} - for k, obj in data_dict.items(): - print(f"obj: {obj}") - if isinstance(obj, (dict, list)): - new_dict[k] = json.dumps(obj) - else: - new_dict[k] = obj - return new_dict - def send_request(self, requests_callable, *args, **kwargs): # this is the timeout of requests module requests_timeout = kwargs.pop('timeout', self.requests_timeout) @@ -279,29 +265,8 @@ def make_request(self, func, resource, params=None, data=None, raise DeepomaticException("Cannot send the request as multipart without files provided.") # requests will build the good multipart content types with the boundaries content_type = None - # data: { - # 'inputs': [{ - # 'image': { - # 'url': "http://xxxx", - # 'bbox': {'xmin': 0.1} - # } - # }, - # ], - # ... - # } - print(f"original dict: {data}") data = self.dump_json_for_multipart(data) - print(f"new data dumped: {data}") - # data: { - # 'inputs[0]image.bbox': "{'xmin': 0.1}" - # } - # }, - # ], - # ... - # } - print(f"original files: {files}") files = self.dump_json_for_multipart(files) - print(f"transformed files: {files}") else: raise DeepomaticException("Unsupported Content-Type") diff --git a/deepomatic/api/inputs.py b/deepomatic/api/inputs.py index e083f52..4655ae4 100644 --- a/deepomatic/api/inputs.py +++ b/deepomatic/api/inputs.py @@ -38,22 +38,13 @@ def format_inputs(inputs, data): data = copy.deepcopy(data) files = {} need_multipart = any([input_data.need_multipart() for input_data in inputs]) - # img['bbox'] = {...} - # get_input() = { - # 'image': img - # } inputs_data = [input_data.get_input() for input_data in inputs] - # inputs_data: [{'image': img}, ] if need_multipart: files['inputs'] = inputs_data else: data['inputs'] = inputs_data content_type = 'multipart/mixed' if need_multipart else 'application/json' - # data: { - # 'inputs': [{'image': img}, ], - # ... - # } return content_type, data, files @@ -104,7 +95,6 @@ def __init__(self, source, encoding=None, bbox=None, polygon=None, crop_uniform_ self.bbox = bbox self.polygon = polygon self.crop_uniform_background = crop_uniform_background - self.encoding = encoding def get_input(self): image = { @@ -113,7 +103,7 @@ def get_input(self): } if self.bbox is not None: image['bbox'] = self.bbox - if self.need_multipart: + if self.need_multipart and isinstance(image['bbox'], dict): # Convert the bbox to a string to allow serialization image['bbox'] = json.dumps(image['bbox']) if self.polygon is not None: diff --git a/tests/test_client.py b/tests/test_client.py index d02b315..3ff66da 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -18,6 +18,7 @@ from deepomatic.api.http_retry import HTTPRetry from deepomatic.api.inputs import ImageInput from deepomatic.api.version import __title__, __version__ +from io import BytesIO from requests.exceptions import ConnectionError, MissingSchema from tenacity import RetryError, stop_after_delay @@ -259,6 +260,14 @@ def test_create_custom_reco_and_infer(self, client, custom_network): max_predictions=3) assert inference_schema(3, 0, 'golden retriever', 0.5) == result + # Test a binary image via multipart + response = requests.get(DEMO_URL) + img_data = BytesIO(response.content) + result = version.inference(inputs=[ImageInput(img_data, encoding='binary', bbox={"xmin": 0.2, "ymin": 0.2, "xmax": 0.8, "ymax": 0.8})], + show_discarded=True, + max_predictions=3) + assert inference_schema(3, 0, 'golden retriever', 0.5) == result + versions = spec.versions() assert versions.count() > 0 data = versions.data() From dda773a7494b22f64d6dbfa0675e99d8e70d26e2 Mon Sep 17 00:00:00 2001 From: Quentin Trehard Date: Mon, 12 Apr 2021 13:10:50 +0200 Subject: [PATCH 3/4] Uncomment linter --- deploy/Dockerfile | 2 +- test.py | 62 ----------------------------------------------- 2 files changed, 1 insertion(+), 63 deletions(-) delete mode 100644 test.py diff --git a/deploy/Dockerfile b/deploy/Dockerfile index d65d1e8..7cb1700 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -5,7 +5,7 @@ FROM ${BASE_IMAGE} as dev WORKDIR /app COPY . . # lint check -# RUN flake8 --statistics --verbose +RUN flake8 --statistics --verbose # prepare local dev environment, for tests execution RUN pip install -e . diff --git a/test.py b/test.py deleted file mode 100644 index a687feb..0000000 --- a/test.py +++ /dev/null @@ -1,62 +0,0 @@ -# export DEEPOMATIC_API_KEY="7a90cd34c66548c99503a5e4ed6e9786" - -### - -import sys -sys.path.append('/home/kent/deepo/deepomatic-client-python') - -# essayer dmake shell client - -### - -from deepomatic.api.http_helper import HTTPHelper -http_helper = HTTPHelper() -http_helper.dump_json_for_multipart({}) -http_helper.dump_json_for_multipart({'level_0': {'level_1': {'level_2': "niveau 2"}, 'level_1_bis': {'level_a': 'nouvelle orleans'}}, 'level_0_bis': 'niveau 0'}) -# => returns {'level_0.level_1.level_2': 'niveau 2', 'level_0_bis': 'niveau 0'} - -http_helper.dump_json_for_multipart({'level_0': [{'bbox': {'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3}}], 'level_0_bis': 'niveau 0'}) -http_helper.dump_json_for_multipart_new({'level_0': [{'image': {'bbox': "{'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3}"}}], 'level_0_bis': 'niveau 0'}) - -from deepomatic.api.http_helper import HTTPHelper -http_helper = HTTPHelper() -http_helper.dump_json_for_multipart_new({'level_0': [{'image': {'bbox': {'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3}}}], 'level_0_bis': 'niveau 0'}) - - - - -######## - -import json -import requests -from io import BytesIO -from deepomatic.api.inputs import ImageInput -from deepomatic.api.client import Client - -# client = Client() -# client.RecognitionVersion.list() - -# staging -app_id = '396170490728' -api_key = 'a9b3d24b8a68401f8f4bf582d7dd12e8' -host = 'https://api-staging.deepomatic.com' - -client = Client(app_id, api_key, host=host) -version = client.RecognitionVersion.retrieve("40771") -# version = client.RecognitionVersion.create(network_id="47299", spec_id=49832, post_processings=[{'detection': {'direct_output': {'boxes_tensor': 'detection_boxes:0', 'scores_tensor': 'detection_scores:0', 'classes_tensor': 'detection_classes:0'}, 'thresholds': [0.0, 0.0, 0.0], 'nms_threshold': 0.3, 'discard_threshold': 0.0, 'normalize_wrt_tensor': ''}}]) - -bbox = {'xmin': 0.1, 'xmax': 0.9, 'ymin': 0.3, 'ymax': 0.5} - -url = "https://www.illicoveto.com/wp-content/uploads/berger-australien-768x988.jpg" - -response = requests.get(url) -img_data = BytesIO(response.content) - -img = { - 'encoding': 'binary', - 'bbox': json.dumps(bbox), - 'source': img_data -} -data_input = ImageInput(**img) - -version.inference(inputs=[data_input]) From 1722ded4632e48b569e3071ac72fc67443540783 Mon Sep 17 00:00:00 2001 From: Quentin Trehard Date: Mon, 12 Apr 2021 14:34:49 +0200 Subject: [PATCH 4/4] Line too long --- tests/test_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index 3ff66da..bb5e935 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -263,7 +263,8 @@ def test_create_custom_reco_and_infer(self, client, custom_network): # Test a binary image via multipart response = requests.get(DEMO_URL) img_data = BytesIO(response.content) - result = version.inference(inputs=[ImageInput(img_data, encoding='binary', bbox={"xmin": 0.2, "ymin": 0.2, "xmax": 0.8, "ymax": 0.8})], + result = version.inference(inputs=[ImageInput(img_data, encoding='binary', + bbox={"xmin": 0.2, "ymin": 0.2, "xmax": 0.8, "ymax": 0.8})], show_discarded=True, max_predictions=3) assert inference_schema(3, 0, 'golden retriever', 0.5) == result