diff --git a/Makefile b/Makefile index 8355c03e..58948952 100644 --- a/Makefile +++ b/Makefile @@ -30,20 +30,26 @@ PYTEST=poetry run pytest -v # `make test TEST_ARGS="-k some_filter"` TEST_ARGS= +CLOUD_FILTERS = -m "not run_only_for_edge_endpoint" +EDGE_FILTERS = -m "not skip_for_edge_endpoint" + test: install ## Run tests against the prod API (needs GROUNDLIGHT_API_TOKEN) - ${PYTEST} ${TEST_ARGS} test + ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test + +test-4edge: install ## Run tests against the prod API via the edge-endpoint (needs GROUNDLIGHT_API_TOKEN) + ${PYTEST} ${TEST_ARGS} ${EDGE_FILTERS} test test-local: install ## Run tests against a localhost API (needs GROUNDLIGHT_API_TOKEN and a local API server) - GROUNDLIGHT_ENDPOINT="http://localhost:8000/" ${PYTEST} ${TEST_ARGS} test + GROUNDLIGHT_ENDPOINT="http://localhost:8000/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test test-integ: install ## Run tests against the integ API server (needs GROUNDLIGHT_API_TOKEN) - GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} test + GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test test-docs: install ## Run the example code and tests in our docs against the prod API (needs GROUNDLIGHT_API_TOKEN) - poetry run pytest -v --markdown-docs ${TEST_ARGS} docs README.md + ${PYTEST} --markdown-docs ${TEST_ARGS} docs README.md test-docs-integ: install ## Run the example code and tests in our docs against the integ API (needs GROUNDLIGHT_API_TOKEN) - GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" poetry run pytest -v --markdown-docs ${TEST_ARGS} docs README.md + GROUNDLIGHT_ENDPOINT="https://api.integ.groundlight.ai/" ${PYTEST} --markdown-docs ${TEST_ARGS} docs README.md # Adjust which paths we lint LINT_PATHS="src test bin samples" diff --git a/test/integration/test_groundlight.py b/test/integration/test_groundlight.py index f8438650..8f74d391 100644 --- a/test/integration/test_groundlight.py +++ b/test/integration/test_groundlight.py @@ -252,6 +252,7 @@ def test_submit_image_query_with_human_review_param(gl: Groundlight, detector: D assert is_valid_display_result(_image_query.result) +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") @pytest.mark.parametrize("metadata", [None, {}, {"a": 1}, '{"a": 1}']) def test_submit_image_query_with_metadata( gl: Groundlight, detector: Detector, image: str, metadata: Union[Dict, str, None] @@ -267,6 +268,7 @@ def test_submit_image_query_with_metadata( assert retrieved_iq.metadata == expected_metadata +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") def test_ask_methods_with_metadata(gl: Groundlight, detector: Detector, image: str): metadata = {"a": 1} iq = gl.ask_ml(detector=detector.id, image=image, metadata=metadata) @@ -280,12 +282,14 @@ def test_ask_methods_with_metadata(gl: Groundlight, detector: Detector, image: s # assert iq.metadata == metadata +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") @pytest.mark.parametrize("metadata", ["", "a", b'{"a": 1}']) def test_submit_image_query_with_invalid_metadata(gl: Groundlight, detector: Detector, image: str, metadata: Any): with pytest.raises(TypeError): gl.submit_image_query(detector=detector.id, image=image, human_review="NEVER", metadata=metadata) +@pytest.mark.skip_for_edge_endpoint(reason="The edge-endpoint does not support passing metadata.") def test_submit_image_query_with_metadata_too_large(gl: Groundlight, detector: Detector, image: str): with pytest.raises(ValueError): gl.submit_image_query( @@ -296,6 +300,14 @@ def test_submit_image_query_with_metadata_too_large(gl: Groundlight, detector: D ) +@pytest.mark.run_only_for_edge_endpoint +def test_submit_image_query_with_metadata_returns_user_error(gl: Groundlight, detector: Detector, image: str): + """On the edge-endpoint, we raise an exception if the user passes metadata.""" + with pytest.raises(openapi_client.exceptions.ApiException) as exc_info: + gl.submit_image_query(detector=detector.id, image=image, human_review="NEVER", metadata={"a": 1}) + assert is_user_error(exc_info.value.status) + + def test_submit_image_query_jpeg_bytes(gl: Groundlight, detector: Detector): jpeg = open("test/assets/dog.jpeg", "rb").read() _image_query = gl.submit_image_query(detector=detector.id, image=jpeg, human_review="NEVER")