From 20fa3373c58c4c9aec5a1937bcecf367c93c8cc4 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 15 Sep 2023 14:19:26 -0700 Subject: [PATCH 1/5] Working on implementing patience time and wait time --- src/groundlight/client.py | 52 +++++++++++++++++++++++++++++++++++++++ src/groundlight/todo.txt | 10 ++++++++ 2 files changed, 62 insertions(+) create mode 100644 src/groundlight/todo.txt diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 65e0b97b..c34f65f1 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -43,6 +43,7 @@ class Groundlight: """ DEFAULT_WAIT: float = 30.0 + DEFAULT_PATIENCE: float = 60.0 POLLING_INITIAL_DELAY = 0.25 POLLING_EXPONENTIAL_BACKOFF = 1.3 # This still has the nice backoff property that the max number of requests @@ -223,6 +224,57 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) return self._fixup_image_query(image_query) + def ask_confident(self): + pass + + def ask_ml(self): + pass + + def ask_async( + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + patience_time: float = DEFAULT_PATIENCE, + human_review: Optional[str] = None, + ) -> ImageQuery: + """Sends an image to Groundlight without waiting for a response. + :param detector: the Detector object, or string id of a detector like `det_12345` + :param image: The image, in several possible formats: + - filename (string) of a jpeg file + - byte array or BytesIO or BufferedReader with jpeg bytes + - numpy array with values 0-255 and dimensions (H,W,3) in BGR order + (Note OpenCV uses BGR not RGB. `img[:, :, ::-1]` will reverse the channels) + - PIL Image + Any binary format must be JPEG-encoded already. Any pixel format will get + converted to JPEG at high quality before sending to service. + :param patience_time: How long Groundlight will work to answer the query. + :param human_review: If `None` or `DEFAULT`, send the image query for human review + only if the ML prediction is not confident. + If set to `ALWAYS`, always send the image query for human review. + If set to `NEVER`, never send the image query for human review. + """ + + detector_id = detector.id if isinstance(detector, Detector) else detector + + image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) + + params = {"detector_id": detector_id, "body": image_bytesio} + if patience_time == 0: + params["patience_time"] = self.DEFAULT_PATIENCE + else: + params["patience_time"] = patience_time + + if human_review is not None: + params["human_review"] = human_review + + # If no inspection_id is provided, we submit the image query using image_queries_api (autogenerated via OpenAPI) + # However, our autogenerated code does not currently support inspection_id, so if an inspection_id was + # provided, we use the private API client instead. + + raw_image_query = self.image_queries_api.submit_image_query(**params) + image_query = ImageQuery.parse_obj(raw_image_query.to_dict()) + return self._fixup_image_query(image_query) + def wait_for_confident_result( self, image_query: Union[ImageQuery, str], diff --git a/src/groundlight/todo.txt b/src/groundlight/todo.txt new file mode 100644 index 00000000..8aeeda58 --- /dev/null +++ b/src/groundlight/todo.txt @@ -0,0 +1,10 @@ +out of scope: + changing the behavior of submit_image_query - this is a breaking change that will require a lot of love and planning. I see submit_image_query as the heavy duty utility function, being a superset of the three primary functions we're introducing +TODO: + write + aks_ml + aks confident + add test cases + ask_async + aks_ml + aks confident From 397ec9eea7cbe12fbfad6d6e75ce27411b492ad9 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 15 Sep 2023 15:09:49 -0700 Subject: [PATCH 2/5] First pass at implementing different use cases --- src/groundlight/client.py | 38 ++++++++++++++++++---------- src/groundlight/todo.txt | 10 -------- test/integration/test_groundlight.py | 21 +++++++++++++++ 3 files changed, 46 insertions(+), 23 deletions(-) delete mode 100644 src/groundlight/todo.txt diff --git a/src/groundlight/client.py b/src/groundlight/client.py index c34f65f1..29fd41ed 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -43,7 +43,7 @@ class Groundlight: """ DEFAULT_WAIT: float = 30.0 - DEFAULT_PATIENCE: float = 60.0 + DEFAULT_PATIENCE: float = 45.0 POLLING_INITIAL_DELAY = 0.25 POLLING_EXPONENTIAL_BACKOFF = 1.3 # This still has the nice backoff property that the max number of requests @@ -171,6 +171,7 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], wait: Optional[float] = None, + patience_time: Optional[float] = None, human_review: Optional[str] = None, inspection_id: Optional[str] = None, ) -> ImageQuery: @@ -194,16 +195,18 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments """ if wait is None: wait = self.DEFAULT_WAIT + if patience_time is None: + patience_time = self.DEFAULT_PATIENCE detector_id = detector.id if isinstance(detector, Detector) else detector image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) params = {"detector_id": detector_id, "body": image_bytesio} - if wait == 0: - params["patience_time"] = self.DEFAULT_WAIT + if patience_time == 0: + params["patience_time"] = self.DEFAULT_PATIENCE else: - params["patience_time"] = wait + params["patience_time"] = patience_time if human_review is not None: params["human_review"] = human_review @@ -224,18 +227,27 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) return self._fixup_image_query(image_query) - def ask_confident(self): - pass + def ask_confident( + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + patience_time: float = None, + ) -> ImageQuery: + self.submit_image_query(detector, image, wait=patience_time, patience_time=patience_time) - def ask_ml(self): - pass + def ask_ml( + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + wait: Optional[float] = None, + ) -> ImageQuery: + self.submit_image_query(detector, image, wait=wait) def ask_async( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], - patience_time: float = DEFAULT_PATIENCE, - human_review: Optional[str] = None, + patience_time: float = None, ) -> ImageQuery: """Sends an image to Groundlight without waiting for a response. :param detector: the Detector object, or string id of a detector like `det_12345` @@ -254,6 +266,9 @@ def ask_async( If set to `NEVER`, never send the image query for human review. """ + if patience_time is None: + patience_time = self.DEFAULT_PATIENCE + detector_id = detector.id if isinstance(detector, Detector) else detector image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) @@ -264,9 +279,6 @@ def ask_async( else: params["patience_time"] = patience_time - if human_review is not None: - params["human_review"] = human_review - # If no inspection_id is provided, we submit the image query using image_queries_api (autogenerated via OpenAPI) # However, our autogenerated code does not currently support inspection_id, so if an inspection_id was # provided, we use the private API client instead. diff --git a/src/groundlight/todo.txt b/src/groundlight/todo.txt deleted file mode 100644 index 8aeeda58..00000000 --- a/src/groundlight/todo.txt +++ /dev/null @@ -1,10 +0,0 @@ -out of scope: - changing the behavior of submit_image_query - this is a breaking change that will require a lot of love and planning. I see submit_image_query as the heavy duty utility function, being a superset of the three primary functions we're introducing -TODO: - write - aks_ml - aks confident - add test cases - ask_async - aks_ml - aks confident diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 5c836714..5374dc3b 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -236,6 +236,27 @@ def test_submit_image_query_pil(gl: Groundlight, detector: Detector): _image_query = gl.submit_image_query(detector=detector.id, image=black) +def test_ask_confident(gl: Groundlight, detector: Detector): + _iq = gl.ask_confident(detector=detector.id, image="test/assets/cat.jpeg") + _iq = gl.ask_confident(detector=detector.id, image="test/assets/cat.jpeg", patience_time=10) + # TODO: Check that we get a proper error if 0 < patience time < minimum backend patience time + + +def test_ask_ml(gl: Groundlight, detector: Detector): + _iq = gl.ask_ml(detector=detector.id, image="test/assets/cat.jpeg") + _iq = gl.ask_ml(detector=detector.id, image="test/assets/cat.jpeg", wait=10) + _iq = gl.ask_ml( + detector=detector.id, image="test/assets/cat.jpeg", wait=1 + ) # should be legal as wait isn't subject to minimum backend patience time + + +def test_ask_async(gl: Groundlight, detector: Detector): + _iq = gl.ask_async(detector=detector.id, image="test/assets/cat.jpeg") + _iq = gl.ask_async(detector=detector.id, image="test/assets/cat.jpeg", patience_time=10) + _iq = gl.ask_async(detector=detector.id, image="test/assets/cat.jpeg", patience_time=600) + # TODO: Check that we get a proper error if 0 < patience time < minimum backend patience time + + def test_list_image_queries(gl: Groundlight): image_queries = gl.list_image_queries() assert str(image_queries) From 1c545915bd31ba62fe8332adde1a1919bca31f52 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 15 Sep 2023 15:24:25 -0700 Subject: [PATCH 3/5] fix defaults values, linting --- src/groundlight/client.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 29fd41ed..5bdfa5ce 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -43,7 +43,7 @@ class Groundlight: """ DEFAULT_WAIT: float = 30.0 - DEFAULT_PATIENCE: float = 45.0 + DEFAULT_PATIENCE: float = 30.0 POLLING_INITIAL_DELAY = 0.25 POLLING_EXPONENTIAL_BACKOFF = 1.3 # This still has the nice backoff property that the max number of requests @@ -197,6 +197,8 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments wait = self.DEFAULT_WAIT if patience_time is None: patience_time = self.DEFAULT_PATIENCE + if wait > patience_time: + patience_time = wait detector_id = detector.id if isinstance(detector, Detector) else detector @@ -231,7 +233,7 @@ def ask_confident( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], - patience_time: float = None, + patience_time: Optional[float] = None, ) -> ImageQuery: self.submit_image_query(detector, image, wait=patience_time, patience_time=patience_time) @@ -247,7 +249,7 @@ def ask_async( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], - patience_time: float = None, + patience_time: Optional[float] = None, ) -> ImageQuery: """Sends an image to Groundlight without waiting for a response. :param detector: the Detector object, or string id of a detector like `det_12345` From 614fd5405b584c81063898eacaabeada56b86425 Mon Sep 17 00:00:00 2001 From: brandon Date: Tue, 19 Sep 2023 08:14:57 -0700 Subject: [PATCH 4/5] Draft version of what true async function might look like and what ask_ml might look like --- src/groundlight/client.py | 114 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 108 insertions(+), 6 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 5bdfa5ce..9a05d881 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -2,7 +2,8 @@ import os import time from io import BufferedReader, BytesIO -from typing import Optional, Union +from typing import Optional, Union, List +import asyncio from model import Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList from openapi_client import Configuration @@ -235,6 +236,7 @@ def ask_confident( image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], patience_time: Optional[float] = None, ) -> ImageQuery: + # Not yet differentiated from submit_image_query other than simplified parameter set self.submit_image_query(detector, image, wait=patience_time, patience_time=patience_time) def ask_ml( @@ -242,8 +244,35 @@ def ask_ml( detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], wait: Optional[float] = None, + human_review: Optional[str] = None, ) -> ImageQuery: - self.submit_image_query(detector, image, wait=wait) + if wait is None: + wait = self.DEFAULT_WAIT + if patience_time is None: + patience_time = self.DEFAULT_PATIENCE + if wait > patience_time: + patience_time = wait + + detector_id = detector.id if isinstance(detector, Detector) else detector + + image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) + + params = {"detector_id": detector_id, "body": image_bytesio} + if patience_time == 0: + params["patience_time"] = self.DEFAULT_PATIENCE + else: + params["patience_time"] = patience_time + + # still available to even within ask_ml + if human_review is not None: + params["human_review"] = human_review + + raw_image_query = self.image_queries_api.submit_image_query(**params) + image_query = ImageQuery.parse_obj(raw_image_query.to_dict()) + + if wait: + image_query = self.wait_for_fast_ml_result(image_query, timeout_sec=wait) + return self._fixup_image_query(image_query) def ask_async( self, @@ -276,19 +305,92 @@ def ask_async( image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) params = {"detector_id": detector_id, "body": image_bytesio} + ### This would require a corresponding backend change, but could save up to a couple seconds of time + ### waiting for the server response + ### alternatively, we could use the asyncio + params["async"] = True if patience_time == 0: params["patience_time"] = self.DEFAULT_PATIENCE else: params["patience_time"] = patience_time - # If no inspection_id is provided, we submit the image query using image_queries_api (autogenerated via OpenAPI) - # However, our autogenerated code does not currently support inspection_id, so if an inspection_id was - # provided, we use the private API client instead. + raw_image_query = self.image_queries_api.submit_image_query(**params) # best api call we have, still has delay + image_query = ImageQuery.parse_obj(raw_image_query.to_dict()) + return self._fixup_image_query(image_query) - raw_image_query = self.image_queries_api.submit_image_query(**params) + async def ask_async_alternate( + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + patience_time: Optional[float] = None, + ) -> ImageQuery: + if patience_time is None: + patience_time = self.DEFAULT_PATIENCE + + detector_id = detector.id if isinstance(detector, Detector) else detector + + image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) + + params = {"detector_id": detector_id, "body": image_bytesio} + if patience_time == 0: + params["patience_time"] = self.DEFAULT_PATIENCE + else: + params["patience_time"] = patience_time + ### This would still benefit from a backend change, but uses true async + # params["async"] = True + raw_image_query = await self.image_queries_api.submit_image_query( + **params + ) # best api call we have, still has delay image_query = ImageQuery.parse_obj(raw_image_query.to_dict()) return self._fixup_image_query(image_query) + def ask_async_alternate_wrapper( + self, + detector: Union[Detector, str], + image_set: List[Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray]], + patience_time: Optional[float] = None, + ) -> List[ImageQuery]: + async def wrapper(): + tasks = [ + asyncio.create_task(self.ask_async_alternate(detector, image, patience_time)) for image in image_set + ] + for task in tasks: + await task + # alternatively use asyncio.gather + # await asyncio.gather(*(self.ask_async_alternate(i) for i in image_set)) + + asyncio.run(wrapper()) + + def wait_for_fast_ml_result( + self, + image_query: Union[ImageQuery, str], + timeout_sec: float = 30.0, + ) -> ImageQuery: + # Convert from image_query_id to ImageQuery if needed. + if isinstance(image_query, str): + image_query = self.get_image_query(image_query) + + start_time = time.time() + next_delay = self.POLLING_INITIAL_DELAY + target_delay = 0.0 + image_query = self._fixup_image_query(image_query) + while True: + patience_so_far = time.time() - start_time + if iq_has_answer(image_query): # Primary difference from wait_for_confident_result + logger.debug(f"Confident answer for {image_query} after {patience_so_far:.1f}s") + break + if patience_so_far >= timeout_sec: + logger.debug(f"Timeout after {timeout_sec:.0f}s waiting for {image_query}") + break + target_delay = min(patience_so_far + next_delay, timeout_sec) + sleep_time = max(target_delay - patience_so_far, 0) + + time.sleep(sleep_time) + next_delay *= self.POLLING_EXPONENTIAL_BACKOFF + image_query = self.get_image_query(image_query.id) + image_query = self._fixup_image_query(image_query) + return image_query + def wait_for_confident_result( self, image_query: Union[ImageQuery, str], From 8f46cdfff84180d7efad02dba7ac84b15b145104 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Tue, 19 Sep 2023 15:16:24 +0000 Subject: [PATCH 5/5] Automatically reformatting code --- src/groundlight/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 9a05d881..8c9237db 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -1,9 +1,9 @@ +import asyncio import logging import os import time from io import BufferedReader, BytesIO -from typing import Optional, Union, List -import asyncio +from typing import List, Optional, Union from model import Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList from openapi_client import Configuration