diff --git a/Makefile b/Makefile index 82a1588f..ae002dc8 100644 --- a/Makefile +++ b/Makefile @@ -37,11 +37,16 @@ TEST_ARGS= CLOUD_FILTERS = -m "not run_only_for_edge_endpoint" EDGE_FILTERS = -m "not skip_for_edge_endpoint" +# Record information about the slowest 25 tests (but don't show anything slower than 0.1 seconds) +PROFILING_ARGS = \ + --durations 25 \ + --durations-min 0.1 + test: install ## Run tests against the prod API (needs GROUNDLIGHT_API_TOKEN) - ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test + ${PYTEST} ${PROFILING_ARGS} ${TEST_ARGS} ${CLOUD_FILTERS} test test-4edge: install ## Run tests against the prod API via the edge-endpoint (needs GROUNDLIGHT_API_TOKEN) - ${PYTEST} ${TEST_ARGS} ${EDGE_FILTERS} test + ${PYTEST} ${PROFILING_ARGS} ${TEST_ARGS} ${EDGE_FILTERS} test test-local: install ## Run tests against a localhost API (needs GROUNDLIGHT_API_TOKEN and a local API server) GROUNDLIGHT_ENDPOINT="http://localhost:8000/" ${PYTEST} ${TEST_ARGS} ${CLOUD_FILTERS} test diff --git a/test/unit/test_actions.py b/test/unit/test_actions.py index 08a79606..d620255b 100644 --- a/test/unit/test_actions.py +++ b/test/unit/test_actions.py @@ -13,9 +13,6 @@ def test_create_action(gl_experimental: ExperimentalApi): rule = gl_experimental.create_rule(det, f"test_rule_{name}", "EMAIL", "test@example.com") rule2 = gl_experimental.get_rule(rule.id) assert rule == rule2 - gl_experimental.delete_rule(rule.id) - with pytest.raises(NotFoundException) as _: - gl_experimental.get_rule(rule.id) @pytest.mark.skip(reason="actions are global on account, the test matrix collides with itself") # type: ignore @@ -45,6 +42,13 @@ def test_create_action_with_human_review(gl_experimental: ExperimentalApi): ) rule2 = gl_experimental.get_rule(rule.id) assert rule == rule2 + + +@pytest.mark.skip(reason="actions are global on account, the test matrix collides with itself") # type: ignore +def test_delete_action(gl_experimental: ExperimentalApi): + name = f"Test {datetime.utcnow()}" + det = gl_experimental.get_or_create_detector(name, "test_query") + rule = gl_experimental.create_rule(det, f"test_rule_{name}", "EMAIL", "test@example.com") gl_experimental.delete_rule(rule.id) with pytest.raises(NotFoundException) as _: gl_experimental.get_rule(rule.id) diff --git a/test/unit/test_detector_reset.py b/test/unit/test_detector_reset.py index 65f4d0f9..67b88514 100644 --- a/test/unit/test_detector_reset.py +++ b/test/unit/test_detector_reset.py @@ -6,6 +6,7 @@ from groundlight_openapi_client.exceptions import NotFoundException +@pytest.mark.skip(reason="This is an expensive test, reset may take some time") def test_reset_retry(gl_experimental: ExperimentalApi): # Reset the detector, retrying in case the reset is still ongoing det = gl_experimental.create_detector(f"Test {datetime.utcnow()}", "test_query") @@ -25,6 +26,7 @@ def test_reset_retry(gl_experimental: ExperimentalApi): raise Exception("Failed to reset detector") +@pytest.mark.skip(reason="This test does not work with strong 0 shot models, enabled by default based on your account") def test_reset_training(gl_experimental: ExperimentalApi): # If we reset a detector, we should have low confidence after the reset low_confidence_threshold = 0.6 diff --git a/test/unit/test_experimental.py b/test/unit/test_experimental.py index 82d4c977..4cebbbd6 100644 --- a/test/unit/test_experimental.py +++ b/test/unit/test_experimental.py @@ -56,6 +56,12 @@ def test_counting_detector(gl_experimental: ExperimentalApi): assert count_iq.result.count is not None +@pytest.mark.skip( + reason=( + "General users currently currently can't use multiclass detectors. If you have questions, reach out" + " to Groundlight support, or upgrade your plan." + ) +) def test_multiclass_detector(gl_experimental: ExperimentalApi): """ verify that we can create and submit to a multi-class detector