From d08f2b3868b1ad3bd76a352f5871736183632980 Mon Sep 17 00:00:00 2001 From: Tyler Romero Date: Wed, 6 Dec 2023 15:48:04 -0800 Subject: [PATCH 1/4] Configure tests for edge-endpoint support --- test/conftest.py | 13 +++++++++++++ test/integration/test_groundlight.py | 26 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 test/conftest.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..446f4ca0 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,13 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--edge-endpoint", + action="store_true", + default=False, + help=( + "skip tests for features that are unsupported on the edge-endpoint. Additonally, run other tests that" + " ensure that good errors are returned from the edge-endpoint." + ), + ) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index f8438650..eb8ab69b 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -252,6 +252,10 @@ def test_submit_image_query_with_human_review_param(gl: Groundlight, detector: D assert is_valid_display_result(_image_query.result) +@pytest.mark.skipif( + condition=pytest.config.getoption("--edge-endpoint"), + reason="Skipping metadata tests because the edge-endpoint does not support metadata.", +) @pytest.mark.parametrize("metadata", [None, {}, {"a": 1}, '{"a": 1}']) def test_submit_image_query_with_metadata( gl: Groundlight, detector: Detector, image: str, metadata: Union[Dict, str, None] @@ -267,6 +271,10 @@ def test_submit_image_query_with_metadata( assert retrieved_iq.metadata == expected_metadata +@pytest.mark.skipif( + condition=pytest.config.getoption("--edge-endpoint"), + reason="Skipping metadata tests because the edge-endpoint does not support metadata.", +) def test_ask_methods_with_metadata(gl: Groundlight, detector: Detector, image: str): metadata = {"a": 1} iq = gl.ask_ml(detector=detector.id, image=image, metadata=metadata) @@ -280,12 +288,20 @@ def test_ask_methods_with_metadata(gl: Groundlight, detector: Detector, image: s # assert iq.metadata == metadata +@pytest.mark.skipif( + condition=pytest.config.getoption("--edge-endpoint"), + reason="Skipping metadata tests because the edge-endpoint does not support metadata.", +) @pytest.mark.parametrize("metadata", ["", "a", b'{"a": 1}']) def test_submit_image_query_with_invalid_metadata(gl: Groundlight, detector: Detector, image: str, metadata: Any): with pytest.raises(TypeError): gl.submit_image_query(detector=detector.id, image=image, human_review="NEVER", metadata=metadata) +@pytest.mark.skipif( + condition=pytest.config.getoption("--edge-endpoint"), + reason="Skipping metadata tests because the edge-endpoint does not support metadata.", +) def test_submit_image_query_with_metadata_too_large(gl: Groundlight, detector: Detector, image: str): with pytest.raises(ValueError): gl.submit_image_query( @@ -296,6 +312,16 @@ def test_submit_image_query_with_metadata_too_large(gl: Groundlight, detector: D ) +@pytest.mark.skipif( + condition=not pytest.config.getoption("--edge-endpoint"), + reason="Skipping test specific to the edge-endpoint.", +) +def test_submit_image_query_with_metadata_returns_user_error(gl: Groundlight, detector: Detector, image: str): + with pytest.raises(openapi_client.exceptions.ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image, human_review="NEVER", metadata={"a": 1}) + assert is_user_error(exc_info.value.status) + + def test_submit_image_query_jpeg_bytes(gl: Groundlight, detector: Detector): jpeg = open("test/assets/dog.jpeg", "rb").read() _image_query = gl.submit_image_query(detector=detector.id, image=jpeg, human_review="NEVER") From f952f601fcdc98301f84e808e4d2b0b5425e6308 Mon Sep 17 00:00:00 2001 From: Tyler Romero Date: Wed, 6 Dec 2023 16:51:05 -0800 Subject: [PATCH 2/4] Fixes --- Makefile | 21 ++++++++++++++++----- test/conftest.py | 13 ------------- test/integration/test_groundlight.py | 26 ++++++-------------------- 3 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 test/conftest.py diff --git a/Makefile b/Makefile index 8355c03e..19f97945 100644 --- a/Makefile +++ b/Makefile @@ -30,20 +30,31 @@ PYTEST=poetry run pytest -v # `make test TEST_ARGS="-k some_filter"` TEST_ARGS= +VALID_LABELS_FILTER="-m "valid_labels'" + +CLOUD_FILTERS = -m "not run_only_for_edge_endpoint" +EDGE_FILTERS = -m "not skip_for_edge_endpoint" + test: install ## Run tests against the prod API (needs GROUNDLIGHT_API_TOKEN) - ${PYTEST} ${TEST_ARGS} test + ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test + +test-4edge: install ## Run tests against the prod API via the edge-endpoint (needs GROUNDLIGHT_API_TOKEN) + ${PYTEST} ${TEST_ARGS} ${EDGE_FILTERS} test test-local: install ## Run tests against a localhost API (needs GROUNDLIGHT_API_TOKEN and a local API server) - GROUNDLIGHT_ENDPOINT="http://localhost:8000/" ${PYTEST} ${TEST_ARGS} test + GROUNDLIGHT_ENDPOINT="http://localhost:8000/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test test-integ: install ## Run tests against the integ API server (needs GROUNDLIGHT_API_TOKEN) - GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} test + GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test + +test-integ-4edge: install ## Run tests against the integ API server (needs GROUNDLIGHT_API_TOKEN) + GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} ${EDGE_FILTERS} test test-docs: install ## Run the example code and tests in our docs against the prod API (needs GROUNDLIGHT_API_TOKEN) - poetry run pytest -v --markdown-docs ${TEST_ARGS} docs README.md + ${PYTEST} --markdown-docs ${TEST_ARGS} ${CLOUD_FILTERS} docs README.md test-docs-integ: install ## Run the example code and tests in our docs against the integ API (needs GROUNDLIGHT_API_TOKEN) - GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" poetry run pytest -v --markdown-docs ${TEST_ARGS} docs README.md + GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} --markdown-docs ${TEST_ARGS} ${CLOUD_FILTERS} docs README.md # Adjust which paths we lint LINT_PATHS="src test bin samples" diff --git a/test/conftest.py b/test/conftest.py deleted file mode 100644 index 446f4ca0..00000000 --- a/test/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest - - -def pytest_addoption(parser): - parser.addoption( - "--edge-endpoint", - action="store_true", - default=False, - help=( - "skip tests for features that are unsupported on the edge-endpoint. Additonally, run other tests that" - " ensure that good errors are returned from the edge-endpoint." - ), - ) diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index eb8ab69b..8f74d391 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -252,10 +252,7 @@ def test_submit_image_query_with_human_review_param(gl: Groundlight, detector: D assert is_valid_display_result(_image_query.result) -@pytest.mark.skipif( - condition=pytest.config.getoption("--edge-endpoint"), - reason="Skipping metadata tests because the edge-endpoint does not support metadata.", -) +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") @pytest.mark.parametrize("metadata", [None, {}, {"a": 1}, '{"a": 1}']) def test_submit_image_query_with_metadata( gl: Groundlight, detector: Detector, image: str, metadata: Union[Dict, str, None] @@ -271,10 +268,7 @@ def test_submit_image_query_with_metadata( assert retrieved_iq.metadata == expected_metadata -@pytest.mark.skipif( - condition=pytest.config.getoption("--edge-endpoint"), - reason="Skipping metadata tests because the edge-endpoint does not support metadata.", -) +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") def test_ask_methods_with_metadata(gl: Groundlight, detector: Detector, image: str): metadata = {"a": 1} iq = gl.ask_ml(detector=detector.id, image=image, metadata=metadata) @@ -288,20 +282,14 @@ def test_ask_methods_with_metadata(gl: Groundlight, detector: Detector, image: s # assert iq.metadata == metadata -@pytest.mark.skipif( - condition=pytest.config.getoption("--edge-endpoint"), - reason="Skipping metadata tests because the edge-endpoint does not support metadata.", -) +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") @pytest.mark.parametrize("metadata", ["", "a", b'{"a": 1}']) def test_submit_image_query_with_invalid_metadata(gl: Groundlight, detector: Detector, image: str, metadata: Any): with pytest.raises(TypeError): gl.submit_image_query(detector=detector.id, image=image, human_review="NEVER", metadata=metadata) -@pytest.mark.skipif( - condition=pytest.config.getoption("--edge-endpoint"), - reason="Skipping metadata tests because the edge-endpoint does not support metadata.", -) +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") def test_submit_image_query_with_metadata_too_large(gl: Groundlight, detector: Detector, image: str): with pytest.raises(ValueError): gl.submit_image_query( @@ -312,11 +300,9 @@ def test_submit_image_query_with_metadata_too_large(gl: Groundlight, detector: D ) -@pytest.mark.skipif( - condition=not pytest.config.getoption("--edge-endpoint"), - reason="Skipping test specific to the edge-endpoint.", -) +@pytest.mark.run_only_for_edge_endpoint def test_submit_image_query_with_metadata_returns_user_error(gl: Groundlight, detector: Detector, image: str): + """On the edge-endpoint, we raise an exception if the user passes metadata.""" with pytest.raises(openapi_client.exceptions.ApiException) as exc_info: gl.submit_image_query(detector=detector.id, image=image, human_review="NEVER", metadata={"a": 1}) assert is_user_error(exc_info.value.status) From 2c0ae81efbef85746c9ccdc0a0c129cd3a4554f0 Mon Sep 17 00:00:00 2001 From: Tyler Romero Date: Wed, 6 Dec 2023 16:53:21 -0800 Subject: [PATCH 3/4] Apply suggestions from code review --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 19f97945..2d19f027 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,6 @@ PYTEST=poetry run pytest -v # `make test TEST_ARGS="-k some_filter"` TEST_ARGS= -VALID_LABELS_FILTER="-m "valid_labels'" - CLOUD_FILTERS = -m "not run_only_for_edge_endpoint" EDGE_FILTERS = -m "not skip_for_edge_endpoint" From b6ced50c0deff022b298f81b17ce9c6c8479a7c8 Mon Sep 17 00:00:00 2001 From: Tyler Romero Date: Thu, 7 Dec 2023 14:47:49 -0800 Subject: [PATCH 4/4] Address PR comments --- Makefile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2d19f027..58948952 100644 --- a/Makefile +++ b/Makefile @@ -45,14 +45,11 @@ test-local: install ## Run tests against a localhost API (needs GROUNDLIGHT_API test-integ: install ## Run tests against the integ API server (needs GROUNDLIGHT_API_TOKEN) GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test -test-integ-4edge: install ## Run tests against the integ API server (needs GROUNDLIGHT_API_TOKEN) - GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} ${EDGE_FILTERS} test - test-docs: install ## Run the example code and tests in our docs against the prod API (needs GROUNDLIGHT_API_TOKEN) - ${PYTEST} --markdown-docs ${TEST_ARGS} ${CLOUD_FILTERS} docs README.md + ${PYTEST} --markdown-docs ${TEST_ARGS} docs README.md test-docs-integ: install ## Run the example code and tests in our docs against the integ API (needs GROUNDLIGHT_API_TOKEN) - GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} --markdown-docs ${TEST_ARGS} ${CLOUD_FILTERS} docs README.md + GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} --markdown-docs ${TEST_ARGS} docs README.md # Adjust which paths we lint LINT_PATHS="src test bin samples"