From e3227499ef97c1d83e18ce72d06fdc9d950fe9a2 Mon Sep 17 00:00:00 2001 From: Francine Wright Date: Thu, 1 May 2025 18:13:45 +0000 Subject: [PATCH 1/5] add bboxes detectors --- src/groundlight/experimental_api.py | 76 ++++++++++++++++++++++++++++ test/integration/test_groundlight.py | 2 + test/unit/test_experimental.py | 45 +++++++++++++++- 3 files changed, 122 insertions(+), 1 deletion(-) diff --git a/src/groundlight/experimental_api.py b/src/groundlight/experimental_api.py index 98c3754e..22f639a0 100644 --- a/src/groundlight/experimental_api.py +++ b/src/groundlight/experimental_api.py @@ -20,6 +20,7 @@ from groundlight_openapi_client.api.image_queries_api import ImageQueriesApi from groundlight_openapi_client.api.notes_api import NotesApi from groundlight_openapi_client.model.action_request import ActionRequest +from groundlight_openapi_client.model.bounding_box_mode_configuration import BoundingBoxModeConfiguration from groundlight_openapi_client.model.channel_enum import ChannelEnum from groundlight_openapi_client.model.condition_request import ConditionRequest from groundlight_openapi_client.model.count_mode_configuration import CountModeConfiguration @@ -974,6 +975,81 @@ def create_multiclass_detector( # noqa: PLR0913 # pylint: disable=too-many-argu obj = self.detectors_api.create_detector(detector_creation_input, _request_timeout=DEFAULT_REQUEST_TIMEOUT) return Detector.parse_obj(obj.to_dict()) + + def create_bounding_box_detector( # noqa: PLR0913 # pylint: disable=too-many-arguments, too-many-locals + self, + name: str, + query: str, + class_name: str, + *, + max_num_bboxes: Optional[int] = None, + group_name: Optional[str] = None, + confidence_threshold: Optional[float] = None, + patience_time: Optional[float] = None, + pipeline_config: Optional[str] = None, + metadata: Union[dict, str, None] = None, + ) -> Detector: + """ + Creates a bounding box detector that can detect objects in images up to a specified maximum number of bounding + boxes. + + **Example usage**:: + + gl = ExperimentalApi() + + # Create a detector that counts people up to 5 + detector = gl.create_bounding_box_detector( + name="people_counter", + query="Draw a bounding box around each person in the image", + class_name="person", + max_num_bboxes=5, + confidence_threshold=0.9, + patience_time=30.0 + ) + + # Use the detector to find people in an image + image_query = gl.ask_ml(detector, "path/to/image.jpg") + print(f"Confidence: {image_query.result.confidence}") + print(f"Bounding boxes: {image_query.result.rois}") + + :param name: A short, descriptive name for the detector. + :param query: A question about the object to detect in the image. + :param class_name: The class name of the object to detect. + :param max_num_bboxes: Maximum number of bounding boxes to detect (default: 10) + :param group_name: Optional name of a group to organize related detectors together. + :param confidence_threshold: A value that sets the minimum confidence level required for the ML model's + predictions. If confidence is below this threshold, the query may be sent for human review. + :param patience_time: The maximum time in seconds that Groundlight will attempt to generate a + confident prediction before falling back to human review. Defaults to 30 seconds. + :param pipeline_config: Advanced usage only. Configuration string needed to instantiate a specific + prediction pipeline for this detector. + :param metadata: A dictionary or JSON string containing custom key/value pairs to associate with + the detector (limited to 1KB). This metadata can be used to store additional + information like location, purpose, or related system IDs. You can retrieve this + metadata later by calling `get_detector()`. + + :return: The created Detector object + """ + + detector_creation_input = self._prep_create_detector( + name=name, + query=query, + group_name=group_name, + confidence_threshold=confidence_threshold, + patience_time=patience_time, + pipeline_config=pipeline_config, + metadata=metadata, + ) + detector_creation_input.mode = ModeEnum.BOUNDING_BOX + # TODO: pull the BE defined default + if max_num_bboxes is None: + max_num_bboxes = 10 + mode_config = BoundingBoxModeConfiguration(max_num_bboxes=max_num_bboxes, class_name=class_name) + detector_creation_input.mode_configuration = mode_config + obj = self.detectors_api.create_detector(detector_creation_input, _request_timeout=DEFAULT_REQUEST_TIMEOUT) + return Detector.parse_obj(obj.to_dict()) + + def _download_mlbinary_url(self, detector: Union[str, Detector]) -> EdgeModelInfo: """ Gets a temporary presigned URL to download the model binaries for the given detector, along diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index 545803ac..c51187e3 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -17,6 +17,7 @@ from ksuid import KsuidMs from model import ( BinaryClassificationResult, + BoundingBoxResult, CountingResult, Detector, ImageQuery, @@ -35,6 +36,7 @@ def is_valid_display_result(result: Any) -> bool: not isinstance(result, BinaryClassificationResult) and not isinstance(result, CountingResult) and not isinstance(result, MultiClassificationResult) + and not isinstance(result, BoundingBoxResult) ): return False diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 0b597649..4ef6fa3a 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -1,5 +1,5 @@ import time -from datetime import datetime +from datetime import datetime, timezone import pytest from groundlight import ExperimentalApi @@ -145,3 +145,46 @@ def test_multiclass_detector(gl_experimental: ExperimentalApi): mc_iq = gl_experimental.submit_image_query(created_detector, "test/assets/dog.jpeg") assert mc_iq.result.label is not None assert mc_iq.result.label in class_names + +@pytest.mark.skip( + reason=( + "General users currently currently can't use bounding box detectors. If you have questions, reach out" + " to Groundlight support, or upgrade your plan." + ) +) +def test_bounding_box_detector(gl_experimental: ExperimentalApi): + """ + Verify that we can create and submit to a bounding box detector + """ + name = f"test_bounding_box_detector_{datetime.now(timezone.utc)}" + created_detector = gl_experimental.create_bounding_box_detector(name, "Draw a bounding box around each dog in the image", "dog") + assert created_detector is not None + bbox_iq = gl_experimental.submit_image_query(created_detector, "test/assets/dog.jpeg") + assert bbox_iq.result.label is not None + assert bbox_iq.rois is not None + +@pytest.mark.skip( + reason=( + "General users currently currently can't use bounding box detectors. If you have questions, reach out" + " to Groundlight support, or upgrade your plan." + ) +) +def test_bounding_box_detector_async(gl_experimental: ExperimentalApi): + """ + Verify that we can create and submit to a bounding box detector with ask_async + """ + name = f"test_bounding_box_detector_async_{datetime.now(timezone.utc)}" + created_detector = gl_experimental.create_bounding_box_detector(name, "Draw a bounding box around each dog in the image", "dog") + assert created_detector is not None + async_iq = gl_experimental.ask_async(created_detector, "test/assets/dog.jpeg") + + # attempting to access fields within the result should raise an exception + with pytest.raises(AttributeError): + _ = async_iq.result.label # type: ignore + with pytest.raises(AttributeError): + _ = async_iq.result.confidence # type: ignore + + time.sleep(5) + # you should be able to get a "real" result by retrieving an updated image query object from the server + _image_query = gl_experimental.get_image_query(id=async_iq.id) + assert _image_query.result is not None \ No newline at end of file From e07bab2a56e81653ad385c7ed4f3c8db7b8bcd46 Mon Sep 17 00:00:00 2001 From: Auto-format Bot Date: Thu, 1 May 2025 18:14:39 +0000 Subject: [PATCH 2/5] Automatically reformatting code --- src/groundlight/experimental_api.py | 2 -- test/unit/test_experimental.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/groundlight/experimental_api.py b/src/groundlight/experimental_api.py index 22f639a0..a198e571 100644 --- a/src/groundlight/experimental_api.py +++ b/src/groundlight/experimental_api.py @@ -975,7 +975,6 @@ def create_multiclass_detector( # noqa: PLR0913 # pylint: disable=too-many-argu obj = self.detectors_api.create_detector(detector_creation_input, _request_timeout=DEFAULT_REQUEST_TIMEOUT) return Detector.parse_obj(obj.to_dict()) - def create_bounding_box_detector( # noqa: PLR0913 # pylint: disable=too-many-arguments, too-many-locals self, name: str, @@ -1049,7 +1048,6 @@ def create_bounding_box_detector( # noqa: PLR0913 # pylint: disable=too-many-ar obj = self.detectors_api.create_detector(detector_creation_input, _request_timeout=DEFAULT_REQUEST_TIMEOUT) return Detector.parse_obj(obj.to_dict()) - def _download_mlbinary_url(self, detector: Union[str, Detector]) -> EdgeModelInfo: """ Gets a temporary presigned URL to download the model binaries for the given detector, along diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 4ef6fa3a..32813a65 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -146,6 +146,7 @@ def test_multiclass_detector(gl_experimental: ExperimentalApi): assert mc_iq.result.label is not None assert mc_iq.result.label in class_names + @pytest.mark.skip( reason=( "General users currently currently can't use bounding box detectors. If you have questions, reach out" @@ -157,12 +158,15 @@ def test_bounding_box_detector(gl_experimental: ExperimentalApi): Verify that we can create and submit to a bounding box detector """ name = f"test_bounding_box_detector_{datetime.now(timezone.utc)}" - created_detector = gl_experimental.create_bounding_box_detector(name, "Draw a bounding box around each dog in the image", "dog") + created_detector = gl_experimental.create_bounding_box_detector( + name, "Draw a bounding box around each dog in the image", "dog" + ) assert created_detector is not None bbox_iq = gl_experimental.submit_image_query(created_detector, "test/assets/dog.jpeg") assert bbox_iq.result.label is not None assert bbox_iq.rois is not None + @pytest.mark.skip( reason=( "General users currently currently can't use bounding box detectors. If you have questions, reach out" @@ -174,7 +178,9 @@ def test_bounding_box_detector_async(gl_experimental: ExperimentalApi): Verify that we can create and submit to a bounding box detector with ask_async """ name = f"test_bounding_box_detector_async_{datetime.now(timezone.utc)}" - created_detector = gl_experimental.create_bounding_box_detector(name, "Draw a bounding box around each dog in the image", "dog") + created_detector = gl_experimental.create_bounding_box_detector( + name, "Draw a bounding box around each dog in the image", "dog" + ) assert created_detector is not None async_iq = gl_experimental.ask_async(created_detector, "test/assets/dog.jpeg") @@ -187,4 +193,4 @@ def test_bounding_box_detector_async(gl_experimental: ExperimentalApi): time.sleep(5) # you should be able to get a "real" result by retrieving an updated image query object from the server _image_query = gl_experimental.get_image_query(id=async_iq.id) - assert _image_query.result is not None \ No newline at end of file + assert _image_query.result is not None From 6b04dd634442a3e163e32d9719682e933bdc14bd Mon Sep 17 00:00:00 2001 From: Francine Wright Date: Thu, 1 May 2025 18:30:44 +0000 Subject: [PATCH 3/5] retrigger --- test/unit/test_experimental.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 32813a65..ee5a7940 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -157,7 +157,7 @@ def test_bounding_box_detector(gl_experimental: ExperimentalApi): """ Verify that we can create and submit to a bounding box detector """ - name = f"test_bounding_box_detector_{datetime.now(timezone.utc)}" + name = f"Test {datetime.now(timezone.utc)}" created_detector = gl_experimental.create_bounding_box_detector( name, "Draw a bounding box around each dog in the image", "dog" ) @@ -177,7 +177,7 @@ def test_bounding_box_detector_async(gl_experimental: ExperimentalApi): """ Verify that we can create and submit to a bounding box detector with ask_async """ - name = f"test_bounding_box_detector_async_{datetime.now(timezone.utc)}" + name = f"Test {datetime.now(timezone.utc)}" created_detector = gl_experimental.create_bounding_box_detector( name, "Draw a bounding box around each dog in the image", "dog" ) From d74168d6ba9f7ee40fe1f2a09d0f78ac6e2de599 Mon Sep 17 00:00:00 2001 From: Francine Wright Date: Thu, 1 May 2025 21:13:49 +0000 Subject: [PATCH 4/5] fix todos --- src/groundlight/experimental_api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/groundlight/experimental_api.py b/src/groundlight/experimental_api.py index a198e571..9e6a8319 100644 --- a/src/groundlight/experimental_api.py +++ b/src/groundlight/experimental_api.py @@ -903,10 +903,12 @@ def create_counting_detector( # noqa: PLR0913 # pylint: disable=too-many-argume metadata=metadata, ) detector_creation_input.mode = ModeEnum.COUNT - # TODO: pull the BE defined default + if max_count is None: - max_count = 10 - mode_config = CountModeConfiguration(max_count=max_count, class_name=class_name) + mode_config = CountModeConfiguration(class_name=class_name) + else: + mode_config = CountModeConfiguration(max_count=max_count, class_name=class_name) + detector_creation_input.mode_configuration = mode_config obj = self.detectors_api.create_detector(detector_creation_input, _request_timeout=DEFAULT_REQUEST_TIMEOUT) return Detector.parse_obj(obj.to_dict()) @@ -1040,10 +1042,12 @@ def create_bounding_box_detector( # noqa: PLR0913 # pylint: disable=too-many-ar metadata=metadata, ) detector_creation_input.mode = ModeEnum.BOUNDING_BOX - # TODO: pull the BE defined default + if max_num_bboxes is None: - max_num_bboxes = 10 - mode_config = BoundingBoxModeConfiguration(max_num_bboxes=max_num_bboxes, class_name=class_name) + mode_config = BoundingBoxModeConfiguration(class_name=class_name) + else: + mode_config = BoundingBoxModeConfiguration(max_num_bboxes=max_num_bboxes, class_name=class_name) + detector_creation_input.mode_configuration = mode_config obj = self.detectors_api.create_detector(detector_creation_input, _request_timeout=DEFAULT_REQUEST_TIMEOUT) return Detector.parse_obj(obj.to_dict()) From 1febdd8b0b43ddc6560c8405bc6557f3dbffaa93 Mon Sep 17 00:00:00 2001 From: Francine Wright Date: Wed, 7 May 2025 21:00:11 +0000 Subject: [PATCH 5/5] remove result for ROIs and add label --- src/groundlight/experimental_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/groundlight/experimental_api.py b/src/groundlight/experimental_api.py index 9e6a8319..c526b833 100644 --- a/src/groundlight/experimental_api.py +++ b/src/groundlight/experimental_api.py @@ -1011,7 +1011,8 @@ def create_bounding_box_detector( # noqa: PLR0913 # pylint: disable=too-many-ar # Use the detector to find people in an image image_query = gl.ask_ml(detector, "path/to/image.jpg") print(f"Confidence: {image_query.result.confidence}") - print(f"Bounding boxes: {image_query.result.rois}") + print(f"Label: {image_query.result.label}") + print(f"Bounding boxes: {image_query.rois}") :param name: A short, descriptive name for the detector. :param query: A question about the object to detect in the image.