From 8aea7adea607c915fe75f110942c41048c1f9cec Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 4 Oct 2023 15:06:22 -0700 Subject: [PATCH 01/17] Adding ask_confident and ask_fast --- src/groundlight/client.py | 72 ++++++++++++++++++++++++++-- test/integration/test_groundlight.py | 46 ++++++++++++++++++ 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index a7d7948c..af7723d7 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -21,6 +21,7 @@ ) from groundlight.optional_imports import Image, np + logger = logging.getLogger("groundlight.sdk") @@ -43,6 +44,7 @@ class Groundlight: """ DEFAULT_WAIT: float = 30.0 + DEFAULT_PATIENCE: float = 30.0 POLLING_INITIAL_DELAY = 0.25 POLLING_EXPONENTIAL_BACKOFF = 1.3 # This still has the nice backoff property that the max number of requests @@ -172,11 +174,62 @@ def list_image_queries(self, page: int = 1, page_size: int = 10) -> PaginatedIma image_queries.results = [self._fixup_image_query(iq) for iq in image_queries.results] return image_queries + def ask_confident( + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + wait: Optional[float] = None, + ) -> ImageQuery: + """Evaluates an image with Groundlight, waiting until an answer above the confidence threshold of the detector is reached or the wait period has passed. + :param detector: the Detector object, or string id of a detector like `det_12345` + :param image: The image, in several possible formats: + - filename (string) of a jpeg file + - byte array or BytesIO or BufferedReader with jpeg bytes + - numpy array with values 0-255 and dimensions (H,W,3) in BGR order + (Note OpenCV uses BGR not RGB. `img[:, :, ::-1]` will reverse the channels) + - PIL Image + Any binary format must be JPEG-encoded already. Any pixel format will get + converted to JPEG at high quality before sending to service. + :param wait: How long to wait (in seconds) for a confident answer. + """ + return self.submit_image_query( + detector, + image, + wait=wait, + ) + + def ask_fast( + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + wait: Optional[float] = None, + ) -> ImageQuery: + """Evaluates an image with Groundlight, getting the first answer Groundlight can provide. + :param detector: the Detector object, or string id of a detector like `det_12345` + :param image: The image, in several possible formats: + - filename (string) of a jpeg file + - byte array or BytesIO or BufferedReader with jpeg bytes + - numpy array with values 0-255 and dimensions (H,W,3) in BGR order + (Note OpenCV uses BGR not RGB. `img[:, :, ::-1]` will reverse the channels) + - PIL Image + Any binary format must be JPEG-encoded already. Any pixel format will get + converted to JPEG at high quality before sending to service. + :param wait: How long to wait (in seconds) for a confident answer. + """ + return self.submit_image_query( + detector, + image, + wait=wait, + confidence_threshold=0.5, + ) + def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], wait: Optional[float] = None, + patience_time: Optional[float] = None, + confidence_threshold: Optional[float] = None, human_review: Optional[str] = None, inspection_id: Optional[str] = None, ) -> ImageQuery: @@ -191,6 +244,8 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. :param wait: How long to wait (in seconds) for a confident answer. + :param patience_time: How long Groundlight should work to generate a confident answer, even working beyond when wait time + :param confidence_threshold: If set, override the detector's confidence threshold for this query. :param human_review: If `None` or `DEFAULT`, send the image query for human review only if the ML prediction is not confident. If set to `ALWAYS`, always send the image query for human review. @@ -200,16 +255,15 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments """ if wait is None: wait = self.DEFAULT_WAIT + if patience_time is None: + patience_time = self.DEFAULT_PATIENCE detector_id = detector.id if isinstance(detector, Detector) else detector image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) params = {"detector_id": detector_id, "body": image_bytesio} - if wait == 0: - params["patience_time"] = self.DEFAULT_WAIT - else: - params["patience_time"] = wait + params["patience_time"] = patience_time if human_review is not None: params["human_review"] = human_review @@ -226,7 +280,10 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments image_query = self.get_image_query(iq_id) if wait: - threshold = self.get_detector(detector).confidence_threshold + if confidence_threshold is None: + threshold = self.get_detector(detector).confidence_threshold + else: + threshold = confidence_threshold image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) return self._fixup_image_query(image_query) @@ -270,6 +327,11 @@ def wait_for_confident_result( image_query = self._fixup_image_query(image_query) return image_query + # def wait_for_first_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: + # """Waits for the first + # . Wait, this is wait for confident with very low confidence threshold, assuming we don't get the placeholder result back + # """ + def add_label(self, image_query: Union[ImageQuery, str], label: Union[Label, str]): """Add a new label to an image query. This answers the detector's question. :param image_query: Either an ImageQuery object (returned from `submit_image_query`) or diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 5c836714..277cbc4f 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -162,6 +162,41 @@ def test_get_detector_by_name(gl: Groundlight, detector: Detector): gl.get_detector_by_name(name="not a real name") +def test_ask_confident(gl: Groundlight, detector: Detector): + _image_query = gl.ask_confident(detector=detector.id, image="test/assets/dog.jpeg", wait=10) + assert str(_image_query) + assert isinstance(_image_query, ImageQuery) + assert is_valid_display_result(_image_query.result) + + +def test_ask_fast(gl: Groundlight, detector: Detector): + _image_query = gl.ask_fast(detector=detector.id, image="test/assets/dog.jpeg", wait=10) + assert str(_image_query) + assert isinstance(_image_query, ImageQuery) + assert is_valid_display_result(_image_query.result) + + +def test_submit_image_query(gl: Groundlight, detector: Detector): + def validate_image_query(_image_query: ImageQuery): + assert str(_image_query) + assert isinstance(_image_query, ImageQuery) + assert is_valid_display_result(_image_query.result) + + _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=10) + validate_image_query(_image_query) + _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=3) + validate_image_query(_image_query) + _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=10, patience_time=20) + validate_image_query(_image_query) + _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", human_review="NEVER") + validate_image_query(_image_query) + _image_query = gl.submit_image_query( + detector=detector.id, image="test/assets/dog.jpeg", wait=180, confidence_threshold=0.75 + ) + validate_image_query(_image_query) + assert _image_query.result.confidence >= 0.75 + + def test_submit_image_query_blocking(gl: Groundlight, detector: Detector): _image_query = gl.submit_image_query(detector=detector.id, image="test/assets/dog.jpeg", wait=10) assert str(_image_query) @@ -421,6 +456,17 @@ def submit_noisy_image(image, label=None): assert False, "The detector performance has not improved after two minutes" +def test_ask_method_quality(gl: Groundlight): + # asks for some level of quality on how fast ask_fast is and that we will get a confident result from ask_confident + name = f"Test {datetime.utcnow()}" # Need a unique name + query = "Is there a dog?" + detector = gl.create_detector(name=name, query=query, confidence_threshold=0.8) + fast_iq = gl.ask_fast(detector=detector.id, image="test/assets/dog.jpeg", wait=0) + assert fast_iq.result.confidence > 0.5 + confident_iq = gl.ask_confident(detector=detector.id, image="test/assets/dog.jpeg", wait=180) + assert confident_iq.result.confidence > 0.8 + + def test_start_inspection(gl: Groundlight): inspection_id = gl.start_inspection() From 37a25f53aba3c8db8e796c3ebe224d78786729d1 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Wed, 4 Oct 2023 22:07:23 +0000 Subject: [PATCH 02/17] Automatically reformatting code --- src/groundlight/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index af7723d7..0f915a86 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -21,7 +21,6 @@ ) from groundlight.optional_imports import Image, np - logger = logging.getLogger("groundlight.sdk") From 603beb490eb217df554e7cea7ea4e90fd53fed53 Mon Sep 17 00:00:00 2001 From: brandon Date: Tue, 10 Oct 2023 13:57:29 -0700 Subject: [PATCH 03/17] Fixing ask_ml behavior --- src/groundlight/client.py | 66 +++++++++++++++++++++++++--------- src/groundlight/internalapi.py | 10 ++++++ 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 0f915a86..c29aca32 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -2,7 +2,7 @@ import os import time from io import BufferedReader, BytesIO -from typing import Optional, Union +from typing import Optional, Union, Callable from model import Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList from openapi_client import Configuration @@ -17,6 +17,7 @@ GroundlightApiClient, NotFoundError, iq_is_confident, + iq_is_answered, sanitize_endpoint_url, ) from groundlight.optional_imports import Image, np @@ -43,7 +44,6 @@ class Groundlight: """ DEFAULT_WAIT: float = 30.0 - DEFAULT_PATIENCE: float = 30.0 POLLING_INITIAL_DELAY = 0.25 POLLING_EXPONENTIAL_BACKOFF = 1.3 # This still has the nice backoff property that the max number of requests @@ -177,6 +177,7 @@ def ask_confident( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + confidence_threshold: Optional[float] = None, wait: Optional[float] = None, ) -> ImageQuery: """Evaluates an image with Groundlight, waiting until an answer above the confidence threshold of the detector is reached or the wait period has passed. @@ -189,15 +190,18 @@ def ask_confident( - PIL Image Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. + :param confidence_threshold: The confidence threshold to wait for. If not set, use the detector's confidence threshold. :param wait: How long to wait (in seconds) for a confident answer. + """ return self.submit_image_query( detector, image, + confidence_threshold=confidence_threshold, wait=wait, ) - def ask_fast( + def ask_ml( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], @@ -213,14 +217,17 @@ def ask_fast( - PIL Image Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. - :param wait: How long to wait (in seconds) for a confident answer. + :param wait: How long to wait (in seconds) for any answer. """ - return self.submit_image_query( + iq = self.submit_image_query( detector, image, - wait=wait, - confidence_threshold=0.5, + wait=0, ) + if iq_is_answered(iq): + return iq + else: + return self._wait_for_ml_result(iq, timeout_sec=wait) def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments self, @@ -243,7 +250,7 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. :param wait: How long to wait (in seconds) for a confident answer. - :param patience_time: How long Groundlight should work to generate a confident answer, even working beyond when wait time + :param patience_time: How long Groundlight should work to generate a confident answer, even working beyond the specified wait time :param confidence_threshold: If set, override the detector's confidence threshold for this query. :param human_review: If `None` or `DEFAULT`, send the image query for human review only if the ML prediction is not confident. @@ -254,15 +261,14 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments """ if wait is None: wait = self.DEFAULT_WAIT - if patience_time is None: - patience_time = self.DEFAULT_PATIENCE detector_id = detector.id if isinstance(detector, Detector) else detector image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) params = {"detector_id": detector_id, "body": image_bytesio} - params["patience_time"] = patience_time + if patience_time is not None: + params["patience_time"] = patience_time if human_review is not None: params["human_review"] = human_review @@ -283,10 +289,10 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments threshold = self.get_detector(detector).confidence_threshold else: threshold = confidence_threshold - image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) + image_query = self._wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) return self._fixup_image_query(image_query) - def wait_for_confident_result( + def _wait_for_confident_result( self, image_query: Union[ImageQuery, str], confidence_threshold: float, @@ -326,10 +332,36 @@ def wait_for_confident_result( image_query = self._fixup_image_query(image_query) return image_query - # def wait_for_first_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: - # """Waits for the first - # . Wait, this is wait for confident with very low confidence threshold, assuming we don't get the placeholder result back - # """ + def _wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: + """Waits for the first ml result to be returned. + Currently this is done by polling with an exponential back-off. + :param image_query: An ImageQuery object to poll + :param confidence_threshold: The minimum confidence level required to return before the timeout. + :param timeout_sec: The maximum number of seconds to wait. + """ + if isinstance(image_query, str): + image_query = self.get_image_query(image_query) + + start_time = time.time() + next_delay = self.POLLING_INITIAL_DELAY + target_delay = 0.0 + image_query = self._fixup_image_query(image_query) + while True: + patience_so_far = time.time() - start_time + if iq_is_answered(image_query): + logger.debug(f"ML answer for {image_query} after {patience_so_far:.1f}s") + break + if patience_so_far >= timeout_sec: + logger.debug(f"Timeout after {timeout_sec:.0f}s waiting for {image_query}") + break + target_delay = min(patience_so_far + next_delay, timeout_sec) + sleep_time = max(target_delay - patience_so_far, 0) + logger.debug(f"Polling ({target_delay:.1f}/{timeout_sec:.0f}s) {image_query} until ML result is available") + time.sleep(sleep_time) + next_delay *= self.POLLING_EXPONENTIAL_BACKOFF + image_query = self.get_image_query(image_query.id) + image_query = self._fixup_image_query(image_query) + return image_query def add_label(self, image_query: Union[ImageQuery, str], label: Union[Label, str]): """Add a new label to an image query. This answers the detector's question. diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index 95243cd9..a327a028 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -71,6 +71,16 @@ def iq_is_confident(iq: ImageQuery, confidence_threshold: float) -> bool: return iq.result.confidence >= confidence_threshold +def iq_is_answered(iq: ImageQuery) -> bool: + """Returns True if the image query has a ML or human label. + Placeholder and special labels (out of domain) have confidences exactly 0.5 + """ + if iq.result.confidence is None: + # Human label + return True + return iq.result.label >= 0.5 + + class InternalApiError(ApiException, RuntimeError): # TODO: We should really avoid this double inheritance since # both `ApiException` and `RuntimeError` are subclasses of From 241915487ebd2bb62743ea4240838ca6f40a8ac1 Mon Sep 17 00:00:00 2001 From: brandon Date: Tue, 10 Oct 2023 14:07:45 -0700 Subject: [PATCH 04/17] Adding to test --- test/integration/test_groundlight.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 213063c5..3546cd7a 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -463,8 +463,10 @@ def submit_noisy_image(image, label=None): assert False, "The detector performance has not improved after two minutes" -def test_ask_method_quality(gl: Groundlight): - # asks for some level of quality on how fast ask_fast is and that we will get a confident result from ask_confident +def test_ask_method_quality(gl: Groundlight, detector: Detector): + # asks for some level of quality on how fast ask_ml is and that we will get a confident result from ask_confident + fast_always_yes_iq = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=0) + assert fast_always_yes_iq.result.confidence > 0.5 name = f"Test {datetime.utcnow()}" # Need a unique name query = "Is there a dog?" detector = gl.create_detector(name=name, query=query, confidence_threshold=0.8) From 1ddf226a7ebb8a7eeca6a9279d3ddfc452c990f7 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Tue, 10 Oct 2023 21:08:52 +0000 Subject: [PATCH 05/17] Automatically reformatting code --- src/groundlight/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index c29aca32..4ed7980b 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -2,7 +2,7 @@ import os import time from io import BufferedReader, BytesIO -from typing import Optional, Union, Callable +from typing import Optional, Union from model import Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList from openapi_client import Configuration @@ -16,8 +16,8 @@ from groundlight.internalapi import ( GroundlightApiClient, NotFoundError, - iq_is_confident, iq_is_answered, + iq_is_confident, sanitize_endpoint_url, ) from groundlight.optional_imports import Image, np From bff74591c514350453b3d4a100efb0da335ba00f Mon Sep 17 00:00:00 2001 From: brandon Date: Tue, 10 Oct 2023 14:28:46 -0700 Subject: [PATCH 06/17] set default wait for ask_ml --- src/groundlight/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index c29aca32..59662d07 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -205,7 +205,7 @@ def ask_ml( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], - wait: Optional[float] = None, + wait: Optional[float] = 10, ) -> ImageQuery: """Evaluates an image with Groundlight, getting the first answer Groundlight can provide. :param detector: the Detector object, or string id of a detector like `det_12345` From d982578f3dde7519d07565a3c75fdb5c9a3fc346 Mon Sep 17 00:00:00 2001 From: brandon Date: Wed, 11 Oct 2023 22:00:59 -0700 Subject: [PATCH 07/17] Unhide wait functions, merging logic, fixed iq_is_answered logic --- src/groundlight/client.py | 57 +++++++++++++--------------------- src/groundlight/internalapi.py | 2 +- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 831aedb0..9c05e81b 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -2,7 +2,7 @@ import os import time from io import BufferedReader, BytesIO -from typing import Optional, Union +from typing import Optional, Union, Callable from model import Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList from openapi_client import Configuration @@ -227,7 +227,7 @@ def ask_ml( if iq_is_answered(iq): return iq else: - return self._wait_for_ml_result(iq, timeout_sec=wait) + return self.wait_for_ml_result(iq, timeout_sec=wait) def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments self, @@ -250,7 +250,7 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. :param wait: How long to wait (in seconds) for a confident answer. - :param patience_time: How long Groundlight should work to generate a confident answer, even working beyond the specified wait time + :param patience_time: If set, how long Groundlight should work to generate a confident answer, even working beyond the specified wait time :param confidence_threshold: If set, override the detector's confidence threshold for this query. :param human_review: If `None` or `DEFAULT`, send the image query for human review only if the ML prediction is not confident. @@ -289,10 +289,10 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments threshold = self.get_detector(detector).confidence_threshold else: threshold = confidence_threshold - image_query = self._wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) + image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) return self._fixup_image_query(image_query) - def _wait_for_confident_result( + def wait_for_confident_result( self, image_query: Union[ImageQuery, str], confidence_threshold: float, @@ -304,41 +304,26 @@ def _wait_for_confident_result( :param confidence_threshold: The minimum confidence level required to return before the timeout. :param timeout_sec: The maximum number of seconds to wait. """ - # Convert from image_query_id to ImageQuery if needed. - if isinstance(image_query, str): - image_query = self.get_image_query(image_query) - - start_time = time.time() - next_delay = self.POLLING_INITIAL_DELAY - target_delay = 0.0 - image_query = self._fixup_image_query(image_query) - while True: - patience_so_far = time.time() - start_time - if iq_is_confident(image_query, confidence_threshold): - logger.debug(f"Confident answer for {image_query} after {patience_so_far:.1f}s") - break - if patience_so_far >= timeout_sec: - logger.debug(f"Timeout after {timeout_sec:.0f}s waiting for {image_query}") - break - target_delay = min(patience_so_far + next_delay, timeout_sec) - sleep_time = max(target_delay - patience_so_far, 0) - logger.debug( - f"Polling ({target_delay:.1f}/{timeout_sec:.0f}s) {image_query} until" - f" confidence>={confidence_threshold:.3f}" - ) - time.sleep(sleep_time) - next_delay *= self.POLLING_EXPONENTIAL_BACKOFF - image_query = self.get_image_query(image_query.id) - image_query = self._fixup_image_query(image_query) - return image_query + confidence_above_thresh = lambda iq: iq_is_confident(iq, confidence_threshold=confidence_threshold) + return self._wait_for_result(image_query, condition=confidence_above_thresh, timeout_sec=timeout_sec) - def _wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: + def wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: """Waits for the first ml result to be returned. Currently this is done by polling with an exponential back-off. :param image_query: An ImageQuery object to poll :param confidence_threshold: The minimum confidence level required to return before the timeout. :param timeout_sec: The maximum number of seconds to wait. """ + return self._wait_for_result(image_query, condition=iq_is_answered, timeout_sec=timeout_sec) + + def _wait_for_result( + self, image_query: Union[ImageQuery, str], condition: Callable, timeout_sec: float = 30.0 + ) -> ImageQuery: + """Performs polling with exponential back-off until the condition is met for the image query. + :param image_query: An ImageQuery object to poll + :param condition: A callable that takes an ImageQuery and returns True or False whether to keep waiting for a better result. + :param timeout_sec: The maximum number of seconds to wait. + """ if isinstance(image_query, str): image_query = self.get_image_query(image_query) @@ -348,15 +333,15 @@ def _wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: image_query = self._fixup_image_query(image_query) while True: patience_so_far = time.time() - start_time - if iq_is_answered(image_query): - logger.debug(f"ML answer for {image_query} after {patience_so_far:.1f}s") + if condition(image_query): + logger.debug(f"Answer for {image_query} after {patience_so_far:.1f}s") break if patience_so_far >= timeout_sec: logger.debug(f"Timeout after {timeout_sec:.0f}s waiting for {image_query}") break target_delay = min(patience_so_far + next_delay, timeout_sec) sleep_time = max(target_delay - patience_so_far, 0) - logger.debug(f"Polling ({target_delay:.1f}/{timeout_sec:.0f}s) {image_query} until ML result is available") + logger.debug(f"Polling ({target_delay:.1f}/{timeout_sec:.0f}s) {image_query} until result is available") time.sleep(sleep_time) next_delay *= self.POLLING_EXPONENTIAL_BACKOFF image_query = self.get_image_query(image_query.id) diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index a327a028..fa8844ab 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -78,7 +78,7 @@ def iq_is_answered(iq: ImageQuery) -> bool: if iq.result.confidence is None: # Human label return True - return iq.result.label >= 0.5 + return iq.result.label > 0.5 class InternalApiError(ApiException, RuntimeError): From 3de4e432a77b1b8e7ce535256322fc6e7dce3103 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Thu, 12 Oct 2023 05:02:22 +0000 Subject: [PATCH 08/17] Automatically reformatting code --- src/groundlight/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 9c05e81b..bf7b52b8 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -2,7 +2,7 @@ import os import time from io import BufferedReader, BytesIO -from typing import Optional, Union, Callable +from typing import Callable, Optional, Union from model import Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList from openapi_client import Configuration @@ -304,7 +304,10 @@ def wait_for_confident_result( :param confidence_threshold: The minimum confidence level required to return before the timeout. :param timeout_sec: The maximum number of seconds to wait. """ - confidence_above_thresh = lambda iq: iq_is_confident(iq, confidence_threshold=confidence_threshold) + + def confidence_above_thresh(iq): + return iq_is_confident(iq, confidence_threshold=confidence_threshold) + return self._wait_for_result(image_query, condition=confidence_above_thresh, timeout_sec=timeout_sec) def wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: float = 30.0) -> ImageQuery: From 47a938eb516e3addc747998d30a8e26a179e5e1f Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 10:35:09 -0700 Subject: [PATCH 09/17] Rewriting doc strings in Sphinx style --- src/groundlight/client.py | 66 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 03764099..47ed4106 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -289,6 +289,8 @@ def ask_confident( ) -> ImageQuery: """Evaluates an image with Groundlight, waiting until an answer above the confidence threshold of the detector is reached or the wait period has passed. :param detector: the Detector object, or string id of a detector like `det_12345` + :type detector: Detector or str + :param image: The image, in several possible formats: - filename (string) of a jpeg file - byte array or BytesIO or BufferedReader with jpeg bytes @@ -297,9 +299,16 @@ def ask_confident( - PIL Image Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. + :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray + :param confidence_threshold: The confidence threshold to wait for. If not set, use the detector's confidence threshold. + :type confidence_threshold: float + :param wait: How long to wait (in seconds) for a confident answer. + :type wait: float + :return ImageQuery + :rtype ImageQuery """ return self.submit_image_query( detector, @@ -319,7 +328,6 @@ def ask_ml( :type detector: Detector or str :param image: The image, in several possible formats: - - filename (string) of a jpeg file - byte array or BytesIO or BufferedReader with jpeg bytes - numpy array with values 0-255 and dimensions (H,W,3) in BGR order @@ -327,7 +335,13 @@ def ask_ml( - PIL Image Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. + :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray + :param wait: How long to wait (in seconds) for any answer. + :type wait: float + + :return ImageQuery + :rtype ImageQuery """ iq = self.submit_image_query( detector, @@ -356,14 +370,12 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments :type detector: Detector or str :param image: The image, in several possible formats: - - filename (string) of a jpeg file - byte array or BytesIO or BufferedReader with jpeg bytes - numpy array with values 0-255 and dimensions (H,W,3) in BGR order (Note OpenCV uses BGR not RGB. `img[:, :, ::-1]` will reverse the channels) - PIL Image: Any binary format must be JPEG-encoded already. Any pixel format will get converted to JPEG at high quality before sending to service. - :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray :param wait: How long to wait (in seconds) for a confident answer. @@ -448,8 +460,16 @@ def wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: f Currently this is done by polling with an exponential back-off. :param image_query: An ImageQuery object to poll + :type image_query: ImageQuery or str + :param confidence_threshold: The minimum confidence level required to return before the timeout. + :type confidence_threshold: float + :param timeout_sec: The maximum number of seconds to wait. + :type timeout_sec: float + + :return ImageQuery + :rtype ImageQuery """ return self._wait_for_result(image_query, condition=iq_is_answered, timeout_sec=timeout_sec) @@ -457,9 +477,18 @@ def _wait_for_result( self, image_query: Union[ImageQuery, str], condition: Callable, timeout_sec: float = 30.0 ) -> ImageQuery: """Performs polling with exponential back-off until the condition is met for the image query. + :param image_query: An ImageQuery object to poll + :type image_query: ImageQuery or str + :param condition: A callable that takes an ImageQuery and returns True or False whether to keep waiting for a better result. + :type condition: Callable + :param timeout_sec: The maximum number of seconds to wait. + :type timeout_sec: float + + :return ImageQuery + :rtype ImageQuery """ if isinstance(image_query, str): image_query = self.get_image_query(image_query) @@ -513,12 +542,27 @@ def add_label(self, image_query: Union[ImageQuery, str], label: Union[Label, str def start_inspection(self) -> str: """For users with Inspection Reports enabled only. Starts an inspection report and returns the id of the inspection. + + :return The unique identifier of the inspection. + :rtype str """ return self.api_client.start_inspection() def update_inspection_metadata(self, inspection_id: str, user_provided_key: str, user_provided_value: str) -> None: """For users with Inspection Reports enabled only. Add/update inspection metadata with the user_provided_key and user_provided_value. + + :param inspection_id: The unique identifier of the inspection. + :type inspection_id: str + + :param user_provided_key: the key in the key/value pair for the inspection metadata. + :type user_provided_key: str + + :param user_provided_value: the value in the key/value pair for the inspection metadata. + :type user_provided_value: str + + :return None + :rtype None """ self.api_client.update_inspection_metadata(inspection_id, user_provided_key, user_provided_value) @@ -526,10 +570,22 @@ def stop_inspection(self, inspection_id: str) -> str: """For users with Inspection Reports enabled only. Stops an inspection and raises an exception if the response from the server indicates that the inspection was not successfully stopped. - Returns a str with result of the inspection (either PASS or FAIL). + + :param inspection_id: The unique identifier of the inspection. + :type inspection_id: str + + :return "PASS" or "FAIL" depending on the result of the inspection. + :rtype str """ return self.api_client.stop_inspection(inspection_id) def update_detector_confidence_threshold(self, detector_id: str, confidence_threshold: float) -> None: - """Updates the confidence threshold of a detector given a detector_id.""" + """Updates the confidence threshold of a detector given a detector_id. + + :param detector_id: The unique identifier of the detector. + :type detector_id: str + + :return None + :rtype None + """ self.api_client.update_detector_confidence_threshold(detector_id, confidence_threshold) From ffb6a45c5e7e20e17585eda3790d1a8eaa9ace9f Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 15:33:14 -0700 Subject: [PATCH 10/17] ask_fast to ask_ml in the tests --- test/integration/test_groundlight.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 80e6b971..a4cdc526 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -169,8 +169,8 @@ def test_ask_confident(gl: Groundlight, detector: Detector): assert is_valid_display_result(_image_query.result) -def test_ask_fast(gl: Groundlight, detector: Detector): - _image_query = gl.ask_fast(detector=detector.id, image="test/assets/dog.jpeg", wait=10) +def test_ask_ml(gl: Groundlight, detector: Detector): + _image_query = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=10) assert str(_image_query) assert isinstance(_image_query, ImageQuery) assert is_valid_display_result(_image_query.result) @@ -474,7 +474,7 @@ def test_ask_method_quality(gl: Groundlight, detector: Detector): name = f"Test {datetime.utcnow()}" # Need a unique name query = "Is there a dog?" detector = gl.create_detector(name=name, query=query, confidence_threshold=0.8) - fast_iq = gl.ask_fast(detector=detector.id, image="test/assets/dog.jpeg", wait=0) + fast_iq = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=0) assert fast_iq.result.confidence > 0.5 confident_iq = gl.ask_confident(detector=detector.id, image="test/assets/dog.jpeg", wait=180) assert confident_iq.result.confidence > 0.8 From 10aba12c02fc1d82d2042800169fac63153b5ae9 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 15:35:30 -0700 Subject: [PATCH 11/17] fixed sphinx docstring return types --- src/groundlight/client.py | 76 +++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 47ed4106..3d04472f 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -76,8 +76,8 @@ def __init__(self, endpoint: Optional[str] = None, api_token: Optional[str] = No If unset, fallback to the environment variable "GROUNDLIGHT_API_TOKEN". :type api_token: str - :return Groundlight client - :rtype Groundlight + :return: Groundlight client + :rtype: Groundlight """ # Specify the endpoint self.endpoint = sanitize_endpoint_url(endpoint) @@ -118,8 +118,8 @@ def get_detector(self, id: Union[str, Detector]) -> Detector: # pylint: disable :param id: the detector id :type id: str or Detector - :return Detector - :rtype Detector + :return: Detector + :rtype: Detector """ if isinstance(id, Detector): @@ -135,8 +135,8 @@ def get_detector_by_name(self, name: str) -> Detector: :param name: the detector name :type name: str - :return Detector - :rtype Detector + :return: Detector + :rtype: Detector """ return self.api_client._get_detector_by_name(name) # pylint: disable=protected-access @@ -150,8 +150,8 @@ def list_detectors(self, page: int = 1, page_size: int = 10) -> PaginatedDetecto :param page_size: the page size :type page_size: int - :return PaginatedDetectorList - :rtype PaginatedDetectorList + :return: PaginatedDetectorList + :rtype: PaginatedDetectorList """ obj = self.detectors_api.list_detectors(page=page, page_size=page_size) return PaginatedDetectorList.parse_obj(obj.to_dict()) @@ -179,8 +179,8 @@ def create_detector( :param pipeline_config: the pipeline config :type pipeline_config: str - :return Detector - :rtype Detector + :return: Detector + :rtype: Detector """ detector_creation_input = DetectorCreationInput(name=name, query=query) if confidence_threshold is not None: @@ -215,8 +215,8 @@ def get_or_create_detector( :param pipeline_config: the pipeline config :type pipeline_config: str - :return Detector - :rtype Detector + :return: Detector + :rtype: Detector """ try: existing_detector = self.get_detector_by_name(name) @@ -254,8 +254,8 @@ def get_image_query(self, id: str) -> ImageQuery: # pylint: disable=redefined-b :param id: the image query id :type id: str - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ obj = self.image_queries_api.get_image_query(id=id) iq = ImageQuery.parse_obj(obj.to_dict()) @@ -271,8 +271,8 @@ def list_image_queries(self, page: int = 1, page_size: int = 10) -> PaginatedIma :param page_size: the page size :type page_size: int - :return PaginatedImageQueryList - :rtype PaginatedImageQueryList + :return: PaginatedImageQueryList + :rtype: PaginatedImageQueryList """ obj = self.image_queries_api.list_image_queries(page=page, page_size=page_size) image_queries = PaginatedImageQueryList.parse_obj(obj.to_dict()) @@ -307,8 +307,8 @@ def ask_confident( :param wait: How long to wait (in seconds) for a confident answer. :type wait: float - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ return self.submit_image_query( detector, @@ -340,8 +340,8 @@ def ask_ml( :param wait: How long to wait (in seconds) for any answer. :type wait: float - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ iq = self.submit_image_query( detector, @@ -391,8 +391,8 @@ def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments this is the ID of the inspection to associate with the image query. :type inspection_id: str - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ if wait is None: wait = self.DEFAULT_WAIT @@ -446,8 +446,8 @@ def wait_for_confident_result( :param timeout_sec: The maximum number of seconds to wait. :type timeout_sec: float - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ def confidence_above_thresh(iq): @@ -468,8 +468,8 @@ def wait_for_ml_result(self, image_query: Union[ImageQuery, str], timeout_sec: f :param timeout_sec: The maximum number of seconds to wait. :type timeout_sec: float - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ return self._wait_for_result(image_query, condition=iq_is_answered, timeout_sec=timeout_sec) @@ -487,8 +487,8 @@ def _wait_for_result( :param timeout_sec: The maximum number of seconds to wait. :type timeout_sec: float - :return ImageQuery - :rtype ImageQuery + :return: ImageQuery + :rtype: ImageQuery """ if isinstance(image_query, str): image_query = self.get_image_query(image_query) @@ -525,8 +525,8 @@ def add_label(self, image_query: Union[ImageQuery, str], label: Union[Label, str :param label: The string "YES" or the string "NO" in answer to the query. :type label: Label or str - :return None - :rtype None + :return: None + :rtype: None """ if isinstance(image_query, ImageQuery): image_query_id = image_query.id @@ -543,8 +543,8 @@ def start_inspection(self) -> str: """For users with Inspection Reports enabled only. Starts an inspection report and returns the id of the inspection. - :return The unique identifier of the inspection. - :rtype str + :return: The unique identifier of the inspection. + :rtype: str """ return self.api_client.start_inspection() @@ -561,8 +561,8 @@ def update_inspection_metadata(self, inspection_id: str, user_provided_key: str, :param user_provided_value: the value in the key/value pair for the inspection metadata. :type user_provided_value: str - :return None - :rtype None + :return: None + :rtype: None """ self.api_client.update_inspection_metadata(inspection_id, user_provided_key, user_provided_value) @@ -574,8 +574,8 @@ def stop_inspection(self, inspection_id: str) -> str: :param inspection_id: The unique identifier of the inspection. :type inspection_id: str - :return "PASS" or "FAIL" depending on the result of the inspection. - :rtype str + :return: "PASS" or "FAIL" depending on the result of the inspection. + :rtype: str """ return self.api_client.stop_inspection(inspection_id) @@ -585,7 +585,7 @@ def update_detector_confidence_threshold(self, detector_id: str, confidence_thre :param detector_id: The unique identifier of the detector. :type detector_id: str - :return None - :rtype None + :return: None + :rtype: None """ self.api_client.update_detector_confidence_threshold(detector_id, confidence_threshold) From 88d67eb179c4bb6008c4033e821118570fc5e2f8 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 15:47:22 -0700 Subject: [PATCH 12/17] Cleaning the lint trap --- src/groundlight/client.py | 6 +++--- test/integration/test_groundlight.py | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 3d04472f..a790736a 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -321,7 +321,7 @@ def ask_ml( self, detector: Union[Detector, str], image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], - wait: Optional[float] = 10, + wait: Optional[float] = None, ) -> ImageQuery: """Evaluates an image with Groundlight, getting the first answer Groundlight can provide. :param detector: the Detector object, or string id of a detector like `det_12345` @@ -350,8 +350,8 @@ def ask_ml( ) if iq_is_answered(iq): return iq - else: - return self.wait_for_ml_result(iq, timeout_sec=wait) + wait = self.DEFAULT_WAIT if wait is None else wait + return self.wait_for_ml_result(iq, timeout_sec=wait) def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments self, diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index a4cdc526..f8f8246d 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -8,12 +8,13 @@ import pytest from groundlight import Groundlight from groundlight.binary_labels import VALID_DISPLAY_LABELS, DeprecatedLabel, Label, convert_internal_label_to_display -from groundlight.internalapi import InternalApiError, NotFoundError +from groundlight.internalapi import InternalApiError, NotFoundError, iq_is_answered from groundlight.optional_imports import * from groundlight.status_codes import is_user_error from model import ClassificationResult, Detector, ImageQuery, PaginatedDetectorList, PaginatedImageQueryList DEFAULT_CONFIDENCE_THRESHOLD = 0.9 +IQ_IMPROVEMENT_THRESHOLD = 0.75 def is_valid_display_result(result: Any) -> bool: @@ -194,7 +195,7 @@ def validate_image_query(_image_query: ImageQuery): detector=detector.id, image="test/assets/dog.jpeg", wait=180, confidence_threshold=0.75 ) validate_image_query(_image_query) - assert _image_query.result.confidence >= 0.75 + assert _image_query.result.confidence >= IQ_IMPROVEMENT_THRESHOLD def test_submit_image_query_blocking(gl: Groundlight, detector: Detector): @@ -470,14 +471,14 @@ def submit_noisy_image(image, label=None): def test_ask_method_quality(gl: Groundlight, detector: Detector): # asks for some level of quality on how fast ask_ml is and that we will get a confident result from ask_confident fast_always_yes_iq = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=0) - assert fast_always_yes_iq.result.confidence > 0.5 + assert iq_is_answered(fast_always_yes_iq) name = f"Test {datetime.utcnow()}" # Need a unique name query = "Is there a dog?" detector = gl.create_detector(name=name, query=query, confidence_threshold=0.8) fast_iq = gl.ask_ml(detector=detector.id, image="test/assets/dog.jpeg", wait=0) - assert fast_iq.result.confidence > 0.5 + assert iq_is_answered(fast_iq) confident_iq = gl.ask_confident(detector=detector.id, image="test/assets/dog.jpeg", wait=180) - assert confident_iq.result.confidence > 0.8 + assert confident_iq.result.confidence > IQ_IMPROVEMENT_THRESHOLD def test_start_inspection(gl: Groundlight): From f9b7b9b7b6a33170b392eec35ba6fd6585fde66a Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 15:54:12 -0700 Subject: [PATCH 13/17] Last bits of lint --- src/groundlight/client.py | 9 ++++++--- src/groundlight/internalapi.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index a790736a..fe892e39 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -287,7 +287,8 @@ def ask_confident( confidence_threshold: Optional[float] = None, wait: Optional[float] = None, ) -> ImageQuery: - """Evaluates an image with Groundlight, waiting until an answer above the confidence threshold of the detector is reached or the wait period has passed. + """Evaluates an image with Groundlight waiting until an answer above the confidence threshold + of the detector is reached or the wait period has passed. :param detector: the Detector object, or string id of a detector like `det_12345` :type detector: Detector or str @@ -301,7 +302,8 @@ def ask_confident( converted to JPEG at high quality before sending to service. :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray - :param confidence_threshold: The confidence threshold to wait for. If not set, use the detector's confidence threshold. + :param confidence_threshold: The confidence threshold to wait for. + If not set, use the detector's confidence threshold. :type confidence_threshold: float :param wait: How long to wait (in seconds) for a confident answer. @@ -481,7 +483,8 @@ def _wait_for_result( :param image_query: An ImageQuery object to poll :type image_query: ImageQuery or str - :param condition: A callable that takes an ImageQuery and returns True or False whether to keep waiting for a better result. + :param condition: A callable that takes an ImageQuery and returns True or False + whether to keep waiting for a better result. :type condition: Callable :param timeout_sec: The maximum number of seconds to wait. diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index b09d9b2b..a324b444 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -78,7 +78,8 @@ def iq_is_answered(iq: ImageQuery) -> bool: if iq.result.confidence is None: # Human label return True - return iq.result.label > 0.5 + placeholder_confidence = 0.5 + return iq.result.label > placeholder_confidence class InternalApiError(ApiException, RuntimeError): From c70909911ed9419eb84d6cf37ded6a03e8b8bfc4 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 16:16:27 -0700 Subject: [PATCH 14/17] Making iq submission with inspection work with newly optional patience time --- src/groundlight/internalapi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index a324b444..67b5a057 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -79,7 +79,7 @@ def iq_is_answered(iq: ImageQuery) -> bool: # Human label return True placeholder_confidence = 0.5 - return iq.result.label > placeholder_confidence + return iq.result.confidence > placeholder_confidence class InternalApiError(ApiException, RuntimeError): @@ -243,9 +243,9 @@ def _get_detector_by_name(self, name: str) -> Detector: def submit_image_query_with_inspection( # noqa: PLR0913 # pylint: disable=too-many-arguments self, detector_id: str, - patience_time: float, body: ByteStreamWrapper, inspection_id: str, + patience_time: Optional[float] = None, human_review: str = "DEFAULT", ) -> str: """Submits an image query to the API and returns the ID of the image query. @@ -257,8 +257,9 @@ def submit_image_query_with_inspection( # noqa: PLR0913 # pylint: disable=too-m params: Dict[str, Union[str, float, bool]] = { "inspection_id": inspection_id, "predictor_id": detector_id, - "patience_time": patience_time, } + if patience_time is not None: + params["patience_time"]: patience_time # In the API, 'send_notification' is used to control human_review escalation. This will eventually # be deprecated, but for now we need to support it in the following manner: From cec6730c2b584f1df1d0653aa0c996cda3f8e855 Mon Sep 17 00:00:00 2001 From: brandon Date: Fri, 13 Oct 2023 16:23:21 -0700 Subject: [PATCH 15/17] single char typo --- src/groundlight/internalapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/groundlight/internalapi.py b/src/groundlight/internalapi.py index 67b5a057..3d85eb37 100644 --- a/src/groundlight/internalapi.py +++ b/src/groundlight/internalapi.py @@ -259,7 +259,7 @@ def submit_image_query_with_inspection( # noqa: PLR0913 # pylint: disable=too-m "predictor_id": detector_id, } if patience_time is not None: - params["patience_time"]: patience_time + params["patience_time"] = float(patience_time) # In the API, 'send_notification' is used to control human_review escalation. This will eventually # be deprecated, but for now we need to support it in the following manner: From 8b8c967b0608e82935f991ee3da51452a18e7863 Mon Sep 17 00:00:00 2001 From: brandon Date: Mon, 16 Oct 2023 18:14:00 -0700 Subject: [PATCH 16/17] Reorder functions to trick Git's LCS alg to be correct --- src/groundlight/client.py | 150 +++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 74 deletions(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index fe892e39..1e2ae952 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -30,7 +30,8 @@ class ApiTokenError(Exception): class Groundlight: - """Client for accessing the Groundlight cloud service. + """ + Client for accessing the Groundlight cloud service. The API token (auth) is specified through the **GROUNDLIGHT_API_TOKEN** environment variable by default. @@ -280,6 +281,80 @@ def list_image_queries(self, page: int = 1, page_size: int = 10) -> PaginatedIma image_queries.results = [self._fixup_image_query(iq) for iq in image_queries.results] return image_queries + def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments + self, + detector: Union[Detector, str], + image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], + wait: Optional[float] = None, + patience_time: Optional[float] = None, + confidence_threshold: Optional[float] = None, + human_review: Optional[str] = None, + inspection_id: Optional[str] = None, + ) -> ImageQuery: + """ + Evaluates an image with Groundlight. + + :param detector: the Detector object, or string id of a detector like `det_12345` + :type detector: Detector or str + + :param image: The image, in several possible formats: + - filename (string) of a jpeg file + - byte array or BytesIO or BufferedReader with jpeg bytes + - numpy array with values 0-255 and dimensions (H,W,3) in BGR order + (Note OpenCV uses BGR not RGB. `img[:, :, ::-1]` will reverse the channels) + - PIL Image: Any binary format must be JPEG-encoded already. + Any pixel format will get converted to JPEG at high quality before sending to service. + :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray + + :param wait: How long to wait (in seconds) for a confident answer. + :type wait: float + + :param human_review: If `None` or `DEFAULT`, send the image query for human review + only if the ML prediction is not confident. + If set to `ALWAYS`, always send the image query for human review. + If set to `NEVER`, never send the image query for human review. + :type human_review: str + + :param inspection_id: Most users will omit this. For accounts with Inspection Reports enabled, + this is the ID of the inspection to associate with the image query. + :type inspection_id: str + + :return: ImageQuery + :rtype: ImageQuery + """ + if wait is None: + wait = self.DEFAULT_WAIT + + detector_id = detector.id if isinstance(detector, Detector) else detector + + image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) + + params = {"detector_id": detector_id, "body": image_bytesio} + if patience_time is not None: + params["patience_time"] = patience_time + + if human_review is not None: + params["human_review"] = human_review + + # If no inspection_id is provided, we submit the image query using image_queries_api (autogenerated via OpenAPI) + # However, our autogenerated code does not currently support inspection_id, so if an inspection_id was + # provided, we use the private API client instead. + if inspection_id is None: + raw_image_query = self.image_queries_api.submit_image_query(**params) + image_query = ImageQuery.parse_obj(raw_image_query.to_dict()) + else: + params["inspection_id"] = inspection_id + iq_id = self.api_client.submit_image_query_with_inspection(**params) + image_query = self.get_image_query(iq_id) + + if wait: + if confidence_threshold is None: + threshold = self.get_detector(detector).confidence_threshold + else: + threshold = confidence_threshold + image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) + return self._fixup_image_query(image_query) + def ask_confident( self, detector: Union[Detector, str], @@ -355,79 +430,6 @@ def ask_ml( wait = self.DEFAULT_WAIT if wait is None else wait return self.wait_for_ml_result(iq, timeout_sec=wait) - def submit_image_query( # noqa: PLR0913 # pylint: disable=too-many-arguments - self, - detector: Union[Detector, str], - image: Union[str, bytes, Image.Image, BytesIO, BufferedReader, np.ndarray], - wait: Optional[float] = None, - patience_time: Optional[float] = None, - confidence_threshold: Optional[float] = None, - human_review: Optional[str] = None, - inspection_id: Optional[str] = None, - ) -> ImageQuery: - """ - Evaluates an image with Groundlight. - - :param detector: the Detector object, or string id of a detector like `det_12345` - :type detector: Detector or str - - :param image: The image, in several possible formats: - - filename (string) of a jpeg file - - byte array or BytesIO or BufferedReader with jpeg bytes - - numpy array with values 0-255 and dimensions (H,W,3) in BGR order - (Note OpenCV uses BGR not RGB. `img[:, :, ::-1]` will reverse the channels) - - PIL Image: Any binary format must be JPEG-encoded already. - Any pixel format will get converted to JPEG at high quality before sending to service. - :type image: str or bytes or Image.Image or BytesIO or BufferedReader or np.ndarray - - :param wait: How long to wait (in seconds) for a confident answer. - :type wait: float - - :param human_review: If `None` or `DEFAULT`, send the image query for human review - only if the ML prediction is not confident. - If set to `ALWAYS`, always send the image query for human review. - If set to `NEVER`, never send the image query for human review. - :type human_review: str - - :param inspection_id: Most users will omit this. For accounts with Inspection Reports enabled, - this is the ID of the inspection to associate with the image query. - :type inspection_id: str - - :return: ImageQuery - :rtype: ImageQuery - """ - if wait is None: - wait = self.DEFAULT_WAIT - - detector_id = detector.id if isinstance(detector, Detector) else detector - - image_bytesio: ByteStreamWrapper = parse_supported_image_types(image) - - params = {"detector_id": detector_id, "body": image_bytesio} - if patience_time is not None: - params["patience_time"] = patience_time - - if human_review is not None: - params["human_review"] = human_review - - # If no inspection_id is provided, we submit the image query using image_queries_api (autogenerated via OpenAPI) - # However, our autogenerated code does not currently support inspection_id, so if an inspection_id was - # provided, we use the private API client instead. - if inspection_id is None: - raw_image_query = self.image_queries_api.submit_image_query(**params) - image_query = ImageQuery.parse_obj(raw_image_query.to_dict()) - else: - params["inspection_id"] = inspection_id - iq_id = self.api_client.submit_image_query_with_inspection(**params) - image_query = self.get_image_query(iq_id) - - if wait: - if confidence_threshold is None: - threshold = self.get_detector(detector).confidence_threshold - else: - threshold = confidence_threshold - image_query = self.wait_for_confident_result(image_query, confidence_threshold=threshold, timeout_sec=wait) - return self._fixup_image_query(image_query) def wait_for_confident_result( self, From 45894628b414cbba1447740b7895db3f2f25b2e2 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Tue, 17 Oct 2023 01:29:38 +0000 Subject: [PATCH 17/17] Automatically reformatting code --- src/groundlight/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/groundlight/client.py b/src/groundlight/client.py index 381565c9..9ced39dd 100644 --- a/src/groundlight/client.py +++ b/src/groundlight/client.py @@ -443,7 +443,6 @@ def ask_ml( wait = self.DEFAULT_WAIT if wait is None else wait return self.wait_for_ml_result(iq, timeout_sec=wait) - def ask_async( self, detector: Union[Detector, str],