From e3b7ad92b41c052b8a3810f3845b3ac6bcf25f85 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 09:41:26 +0100 Subject: [PATCH 1/9] update is quarantined tag according to RFC --- .../datadog/trace/civisibility/domain/TestImpl.java | 2 +- .../civisibility/events/TestEventsHandlerImpl.java | 2 +- .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../test-quarantined-failed-parameterized/events.ftl | 4 ++-- .../resources/test-quarantined-failed/events.ftl | 2 +- .../src/test/resources/test-parallel/coverages.ftl | 12 ++++++------ .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../test-quarantined-failed-parameterized/events.ftl | 2 +- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../test-quarantined-failed-parameterized/events.ftl | 4 ++-- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed-atr/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../resources/test-quarantined-failed/events.ftl | 2 +- .../resources/test-quarantined-failed-7/events.ftl | 2 +- .../test-quarantined-failed-atr-7/events.ftl | 10 +++++----- .../resources/test-quarantined-failed-efd/events.ftl | 6 +++--- .../test-quarantined-failed-known/events.ftl | 2 +- .../test-quarantined-failed-parameterized/events.ftl | 4 ++-- .../trace/bootstrap/instrumentation/api/Tags.java | 3 ++- 41 files changed, 98 insertions(+), 97 deletions(-) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java index aad4395aff6..d60228dfe3b 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java @@ -277,7 +277,7 @@ public void end(@Nullable Long endTime) { EventType.TEST, span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null, span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null, - span.getTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null, + span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null, span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null, span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null, retryReason instanceof TagValue ? (TagValue) retryReason : null, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java index d5c9693e3dc..92f2b870c54 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java @@ -168,7 +168,7 @@ public void onTestStart( } if (testModule.isQuarantined(thisTest)) { - test.setTag(Tags.TEST_MANAGEMENT_IS_QUARANTINED, true); + test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED, true); } if (testExecutionHistory != null) { diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl index 88f48a49fac..ecfa9f13d11 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -49,11 +49,11 @@ "span.kind" : "test", "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -97,12 +97,12 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -146,12 +146,12 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -195,12 +195,12 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -245,12 +245,12 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl index 6e29663a772..873b71b6552 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -50,11 +50,11 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -99,12 +99,12 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.retry_reason" : "efd", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -150,12 +150,12 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.retry_reason" : "efd", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl index 9fe8a3800b8..ad3d947ca17 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl @@ -49,11 +49,11 @@ "span.kind" : "test", "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed/events.ftl index b748a83b9b7..8a75ade8bc0 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-quarantined-failed/events.ftl @@ -49,11 +49,11 @@ "span.kind" : "test", "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-4", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl index 98dbb49f3cd..829b39c272a 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -122,12 +122,12 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -172,13 +172,13 @@ "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -223,13 +223,13 @@ "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -274,13 +274,13 @@ "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -326,13 +326,13 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl index 40b5ab897ce..155dcf123d3 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -125,12 +125,12 @@ "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -176,13 +176,13 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.retry_reason" : "efd", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -229,13 +229,13 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.retry_reason" : "efd", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl index 73c135c74fd..b72db7bc168 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed-known/events.ftl @@ -124,12 +124,12 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed/events.ftl index b2746c28117..b4ff261d152 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/resources/test-quarantined-failed/events.ftl @@ -122,12 +122,12 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "munit", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "munit-junit-4", "test.name" : "Calculator.add", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailedMUnit", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"myTag\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-atr/events.ftl index 2aedb07c62e..7735299037c 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -121,13 +121,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -172,7 +172,6 @@ "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -180,6 +179,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -224,7 +224,6 @@ "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -232,6 +231,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -276,7 +276,6 @@ "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -284,6 +283,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -329,7 +329,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -337,6 +336,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-efd/events.ftl index 8b8a874e065..55665fad7e0 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -124,13 +124,13 @@ "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -176,7 +176,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.retry_reason" : "efd", @@ -184,6 +183,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -230,7 +230,6 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.retry_reason" : "efd", @@ -238,6 +237,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-known/events.ftl index a6573e540be..9daaa37af8e 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-known/events.ftl @@ -123,13 +123,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-parameterized/events.ftl index e20c3d1c73e..7b619a55bd5 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed-parameterized/events.ftl @@ -122,7 +122,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed_parameterized", "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized[0 0 42]\"}}", @@ -130,6 +129,7 @@ "test.source.method" : "test_failed_parameterized()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -171,7 +171,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed_parameterized", "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized[21 21 42]\"}}", @@ -179,6 +178,7 @@ "test.source.method" : "test_failed_parameterized()V", "test.status" : "pass", "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed/events.ftl index 6e9ec79858a..f640662a917 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-quarantined-failed/events.ftl @@ -121,13 +121,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit4", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-4.10", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl index 4b612855288..8e1e6d30f80 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl @@ -1,15 +1,15 @@ [ { "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic.feature" + "filename" : "org/example/cucumber/calculator/basic_arithmetic_skipped.feature" } ], - "span_id" : ${content_span_id}, + "span_id" : ${content_span_id_3}, "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id} + "test_suite_id" : ${content_test_suite_id_2} }, { "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic_skipped.feature" + "filename" : "org/example/cucumber/calculator/basic_arithmetic.feature" } ], - "span_id" : ${content_span_id_3}, + "span_id" : ${content_span_id}, "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id_2} + "test_suite_id" : ${content_test_suite_id} } ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl index b50f425d4c5..9d3c0c2dfa7 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -49,11 +49,11 @@ "span.kind" : "test", "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -97,12 +97,12 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -146,12 +146,12 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -195,12 +195,12 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -245,12 +245,12 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.retry_reason" : "atr", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl index 597e8d6c292..aac9b69030b 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -50,11 +50,11 @@ "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -99,12 +99,12 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.retry_reason" : "efd", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" @@ -150,12 +150,12 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.retry_reason" : "efd", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl index 84d9bf701f2..6db10e84448 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl @@ -49,11 +49,11 @@ "span.kind" : "test", "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed/events.ftl index c83cc0440c6..bbb5e4373d2 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-quarantined-failed/events.ftl @@ -49,11 +49,11 @@ "span.kind" : "test", "test.framework" : "cucumber", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "cucumber-junit-5", "test.name" : "Addition", "test.status" : "fail", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_quarantined" : "true", "test.traits" : "{\"category\":[\"foo\"]}", "test.type" : "test", "test_session.name" : "session-name" diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl index bfe1263f1ff..2055dc5a541 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -54,13 +54,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -106,7 +106,6 @@ "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.retry_reason" : "atr", @@ -114,6 +113,7 @@ "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -159,7 +159,6 @@ "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.retry_reason" : "atr", @@ -167,6 +166,7 @@ "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -212,7 +212,6 @@ "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.retry_reason" : "atr", @@ -220,6 +219,7 @@ "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -266,7 +266,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.retry_reason" : "atr", @@ -274,6 +273,7 @@ "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl index c4808da6720..a2690cc3948 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -55,13 +55,13 @@ "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -108,7 +108,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.retry_reason" : "efd", @@ -116,6 +115,7 @@ "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -163,7 +163,6 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.retry_reason" : "efd", @@ -171,6 +170,7 @@ "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl index 2015aa5d8b2..eb8ffdc0dc6 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-known/events.ftl @@ -54,13 +54,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-parameterized/events.ftl index 076e6d5e932..377165d01bb 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed-parameterized/events.ftl @@ -102,7 +102,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test add 4 and 4", "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 4 and 4\"}}", @@ -110,6 +109,7 @@ "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", "test.status" : "fail", "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed/events.ftl index 44f26a9dacc..8f7f343f202 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-quarantined-failed/events.ftl @@ -54,13 +54,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "spock-junit-5", "test.name" : "test failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-atr/events.ftl index 0513a9cb6eb..fc217cf84b1 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -122,13 +122,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -174,7 +174,6 @@ "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -182,6 +181,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -227,7 +227,6 @@ "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -235,6 +234,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -280,7 +280,6 @@ "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -288,6 +287,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -334,7 +334,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -342,6 +341,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-efd/events.ftl index d67333861ea..699fd078c02 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -125,13 +125,13 @@ "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -178,7 +178,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.retry_reason" : "efd", @@ -186,6 +185,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -233,7 +233,6 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.retry_reason" : "efd", @@ -241,6 +240,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-known/events.ftl index 4cd3042664f..7ef33f46b15 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-known/events.ftl @@ -124,13 +124,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-parameterized/events.ftl index 631f415b2ac..06fcc1510f7 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed-parameterized/events.ftl @@ -122,7 +122,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed_parameterized", "test.parameters" : "{\"metadata\":{\"test_name\":\"[1] 0, 0, 42\"}}", @@ -130,6 +129,7 @@ "test.source.method" : "test_failed_parameterized(III)V", "test.status" : "fail", "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -174,7 +174,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed_parameterized", "test.parameters" : "{\"metadata\":{\"test_name\":\"[2] 1, 1, 42\"}}", @@ -182,6 +181,7 @@ "test.source.method" : "test_failed_parameterized(III)V", "test.status" : "fail", "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed/events.ftl index f66021f3e58..a0d646a996a 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-quarantined-failed/events.ftl @@ -122,13 +122,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "junit5", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "junit-5.3", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-quarantined-failed/events.ftl index 49653072786..8b8eef8a86b 100644 --- a/dd-java-agent/instrumentation/karate/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-quarantined-failed/events.ftl @@ -139,11 +139,11 @@ "span.kind" : "test", "test.framework" : "karate", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "karate", "test.name" : "second scenario", "test.status" : "fail", "test.suite" : "[org/example/test_failed] test failed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-atr/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-atr/events.ftl index f1fe335b033..5af1f9c5fb9 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-atr/events.ftl +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-atr/events.ftl @@ -54,12 +54,12 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -103,13 +103,13 @@ "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -153,13 +153,13 @@ "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -203,13 +203,13 @@ "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -254,13 +254,13 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.retry_reason" : "atr", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-efd/events.ftl index 5f98ac2cd75..11a21ee17f1 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -55,12 +55,12 @@ "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -105,13 +105,13 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.retry_reason" : "efd", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -157,13 +157,13 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.retry_reason" : "efd", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-known/events.ftl index e1ece69ed19..953ebd8155f 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed-known/events.ftl @@ -54,12 +54,12 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed/events.ftl index 421a001aae5..9df71b1598b 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed/events.ftl +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-quarantined-failed/events.ftl @@ -54,12 +54,12 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "scalatest", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", "test.source.file" : "dummy_source_path", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-7/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-7/events.ftl index c4161a5e22a..3d8fc3bdf1b 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-7/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-7/events.ftl @@ -54,13 +54,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-atr-7/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-atr-7/events.ftl index 2e0cd3b7280..0fbd7cba05e 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-atr-7/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-atr-7/events.ftl @@ -54,13 +54,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -106,7 +106,6 @@ "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -114,6 +113,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -159,7 +159,6 @@ "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -167,6 +166,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -213,7 +213,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.has_failed_all_retries" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -221,6 +220,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -266,7 +266,6 @@ "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.retry_reason" : "atr", @@ -274,6 +273,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-efd/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-efd/events.ftl index 8d957840a86..fe5c19923e5 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-efd/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-efd/events.ftl @@ -55,13 +55,13 @@ "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -109,7 +109,6 @@ "test.has_failed_all_retries" : "true", "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.retry_reason" : "efd", @@ -117,6 +116,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -163,7 +163,6 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.is_new" : "true", "test.is_retry" : "true", - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.retry_reason" : "efd", @@ -171,6 +170,7 @@ "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-known/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-known/events.ftl index 8c3400fc9ac..854ab9d281e 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-known/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-known/events.ftl @@ -54,13 +54,13 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "test_failed", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "org.example.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-parameterized/events.ftl index c9a8492b62f..50b7efe07e7 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-quarantined-failed-parameterized/events.ftl @@ -51,7 +51,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "parameterized_test_succeed", "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}", @@ -59,6 +58,7 @@ "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", "test.status" : "pass", "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, @@ -102,7 +102,6 @@ "test.codeowners" : "[\"owner1\",\"owner2\"]", "test.framework" : "testng", "test.framework_version" : ${content_meta_test_framework_version}, - "test.management.is_quarantined" : "true", "test.module" : "testng-7", "test.name" : "parameterized_test_succeed", "test.parameters" : "{\"arguments\":{\"0\":\"\\\"goodbye\\\"\",\"1\":\"false\"}}", @@ -110,6 +109,7 @@ "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", "test.status" : "fail", "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "session-name" }, diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index dfaeb60bcf3..0be2715f2d4 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -97,8 +97,9 @@ public class Tags { public static final String TEST_RETRY_REASON = "test.retry_reason"; public static final String TEST_IS_MODIFIED = "test.is_modified"; public static final String TEST_HAS_FAILED_ALL_RETRIES = "test.has_failed_all_retries"; - public static final String TEST_MANAGEMENT_IS_QUARANTINED = "test.management.is_quarantined"; public static final String TEST_TEST_MANAGEMENT_ENABLED = "test.test_management.enabled"; + public static final String TEST_TEST_MANAGEMENT_IS_QUARANTINED = + "test.test_management.is_quarantined"; public static final String CI_PROVIDER_NAME = "ci.provider.name"; public static final String CI_PIPELINE_ID = "ci.pipeline.id"; From a5a065ce09fb56121b1fdabef5d1b02e4dff6c6d Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 11:59:37 +0100 Subject: [PATCH 2/9] disabled tests --- .../domain/TestFrameworkModule.java | 2 + .../trace/civisibility/domain/TestImpl.java | 2 + .../domain/buildsystem/ProxyTestModule.java | 5 + .../domain/headless/HeadlessTestModule.java | 5 + .../events/TestEventsHandlerImpl.java | 4 + .../civisibility/test/ExecutionStrategy.java | 6 + .../CiVisibilityInstrumentationTest.groovy | 7 +- .../src/test/groovy/CucumberTest.groovy | 27 ++- .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 146 ++++++++++++ .../src/test/groovy/MUnitTest.groovy | 11 +- .../src/test/groovy/JUnit4Test.groovy | 20 +- .../coverages.ftl | 1 + .../events.ftl | 205 +++++++++++++++++ .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 154 +++++++++++++ .../src/test/groovy/CucumberTest.groovy | 27 ++- .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 146 ++++++++++++ .../src/test/groovy/SpockTest.groovy | 20 +- .../coverages.ftl | 1 + .../events.ftl | 203 +++++++++++++++++ .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 155 +++++++++++++ .../src/test/groovy/JUnit5Test.groovy | 20 +- .../coverages.ftl | 1 + .../events.ftl | 155 +++++++++++++ .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 154 +++++++++++++ .../karate/src/test/groovy/KarateTest.groovy | 15 +- .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 211 ++++++++++++++++++ .../src/test/groovy/ScalatestTest.groovy | 19 +- .../test-disabled-failed/coverages.ftl | 1 + .../resources/test-disabled-failed/events.ftl | 151 +++++++++++++ .../instrumentation/testng/TestNGTest.groovy | 22 +- .../test-disabled-failed-7/coverages.ftl | 1 + .../test-disabled-failed-7/events.ftl | 154 +++++++++++++ .../coverages.ftl | 1 + .../events.ftl | 205 +++++++++++++++++ .../telemetry/CiVisibilityCountMetric.java | 2 + .../telemetry/tag/IsDisabled.java | 12 + .../telemetry/tag/SkipReason.java | 5 +- .../bootstrap/instrumentation/api/Tags.java | 2 + 44 files changed, 2229 insertions(+), 55 deletions(-) create mode 100644 dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/events.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/coverages.ftl create mode 100644 dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/events.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/coverages.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/events.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/coverages.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/events.ftl create mode 100644 internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/IsDisabled.java diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java index 5b7fc2b1de5..8f98b2b4dd1 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java @@ -33,6 +33,8 @@ TestSuiteImpl testSuiteStart( boolean isQuarantined(TestIdentifier test); + boolean isDisabled(TestIdentifier test); + /** * Returns the reason for skipping a test, IF it can be skipped. * diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java index d60228dfe3b..ecddadae84e 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java @@ -19,6 +19,7 @@ import datadog.trace.api.civisibility.telemetry.tag.BrowserDriver; import datadog.trace.api.civisibility.telemetry.tag.EventType; import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries; +import datadog.trace.api.civisibility.telemetry.tag.IsDisabled; import datadog.trace.api.civisibility.telemetry.tag.IsModified; import datadog.trace.api.civisibility.telemetry.tag.IsNew; import datadog.trace.api.civisibility.telemetry.tag.IsQuarantined; @@ -278,6 +279,7 @@ public void end(@Nullable Long endTime) { span.getTag(Tags.TEST_IS_NEW) != null ? IsNew.TRUE : null, span.getTag(Tags.TEST_IS_MODIFIED) != null ? IsModified.TRUE : null, span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED) != null ? IsQuarantined.TRUE : null, + span.getTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED) != null ? IsDisabled.TRUE : null, span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null, span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null, retryReason instanceof TagValue ? (TagValue) retryReason : null, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java index ff1467272cf..0c9b3c78935 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java @@ -109,6 +109,11 @@ public boolean isQuarantined(TestIdentifier test) { return executionStrategy.isQuarantined(test); } + @Override + public boolean isDisabled(TestIdentifier test) { + return executionStrategy.isDisabled(test); + } + @Nullable @Override public SkipReason skipReason(TestIdentifier test) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java index 3cb57b5854a..df066c5a973 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java @@ -94,6 +94,11 @@ public boolean isQuarantined(TestIdentifier test) { return executionStrategy.isQuarantined(test); } + @Override + public boolean isDisabled(TestIdentifier test) { + return executionStrategy.isDisabled(test); + } + @Nullable @Override public SkipReason skipReason(TestIdentifier test) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java index 92f2b870c54..a33e5d6748f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java @@ -171,6 +171,10 @@ public void onTestStart( test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_QUARANTINED, true); } + if (testModule.isDisabled(thisTest)) { + test.setTag(Tags.TEST_TEST_MANAGEMENT_IS_TEST_DISABLED, true); + } + if (testExecutionHistory != null) { RetryReason retryReason = testExecutionHistory.currentExecutionRetryReason(); if (retryReason != null) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java index c1cb17ab1c5..a54d12cfbf5 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java @@ -90,9 +90,15 @@ public SkipReason skipReason(TestIdentifier test) { if (test == null) { return null; } + + if (isDisabled(test)) { + return SkipReason.DISABLED; + } + if (!executionSettings.isTestSkippingEnabled()) { return null; } + Map skippableTests = executionSettings.getSkippableTests(); TestMetadata testMetadata = skippableTests.get(test); if (testMetadata == null) { diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index e35c294871b..e6f7393ffdf 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -294,14 +294,17 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { def givenQuarantinedTests(List tests) { quarantinedTests.addAll(tests) + testManagementEnabled = true } def givenDisabledTests(List tests) { disabledTests.addAll(tests) + testManagementEnabled = true } def givenAttemptToFixTests(List tests) { attemptToFixTests.addAll(tests) + testManagementEnabled = true } def givenDiff(Diff diff) { @@ -320,10 +323,6 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled } - def givenTestManagementEnabled(boolean testManagementEnabled) { - this.testManagementEnabled = testManagementEnabled - } - def givenTestsOrder(String testsOrder) { injectSysConfig(CiVisibilityConfig.CIVISIBILITY_TEST_ORDER, testsOrder) } diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy index 271e648ad60..f55933bc14c 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/groovy/CucumberTest.groovy @@ -91,7 +91,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runFeatures(features, true) @@ -106,7 +105,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -118,16 +116,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | features | quarantined | retried + testcaseName | features | quarantined | retried "test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") - ] | [ + ] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -139,15 +136,29 @@ class CucumberTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | features | quarantined | known + testcaseName | features | quarantined | known "test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") - ] | [ + ] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] "test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") - ] | [] + ] | [] + } + + def "test disabled #testcaseName"() { + givenDisabledTests(disabled) + + runFeatures(features, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | disabled + "test-disabled-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") + ] } private String version() { diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..e7bc82f2a6e --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,146 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "skip", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "junit.test_suite", + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.name" : "Addition", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.status" : "skip", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_test_disabled" : "true", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "cucumber-junit-4", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-4", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "junit.test_module", + "resource" : "cucumber-junit-4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy index de264157aa9..1b66f1816e3 100644 --- a/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/munit-junit-4/src/test/groovy/MUnitTest.groovy @@ -9,9 +9,9 @@ import org.example.TestFailedAssumptionMUnit import org.example.TestFailedMUnit import org.example.TestFailedThenSucceedMUnit import org.example.TestSkippedMUnit -import org.example.TestSucceedMUnitSlow import org.example.TestSkippedSuiteMUnit import org.example.TestSucceedMUnit +import org.example.TestSucceedMUnitSlow import org.junit.runner.JUnitCore @DisableTestTrace(reason = "avoid self-tracing") @@ -80,7 +80,6 @@ class MUnitTest extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, true) @@ -93,7 +92,6 @@ class MUnitTest extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -105,12 +103,11 @@ class MUnitTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | retried + testcaseName | tests | quarantined | retried "test-quarantined-failed-atr" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -122,9 +119,9 @@ class MUnitTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | known + testcaseName | tests | quarantined | known "test-quarantined-failed-known" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] - "test-quarantined-failed-efd" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [] + "test-quarantined-failed-efd" | [TestFailedMUnit] | [new TestFQN("org.example.TestFailedMUnit", "Calculator.add")] | [] } private void runTests(Collection> tests, boolean expectSuccess = true) { diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy index f0c2490b8a6..5e21ac5f390 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/groovy/JUnit4Test.groovy @@ -128,7 +128,6 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, true) @@ -142,7 +141,6 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -154,12 +152,11 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | retried + testcaseName | tests | quarantined | retried "test-quarantined-failed-atr" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -171,11 +168,24 @@ class JUnit4Test extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | known + testcaseName | tests | quarantined | known "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [] } + def "test disabled #testcaseName"() { + givenDisabledTests(disabled) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | disabled + "test-disabled-failed" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-disabled-failed-parameterized" | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "test_failed_parameterized")] + } + private void runTests(Collection> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() try { diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/events.ftl new file mode 100644 index 00000000000..16ad8909c37 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed-parameterized/events.ftl @@ -0,0 +1,205 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-4.10", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized[0 0 42]\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized[21 21 42]\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..ae7473fb54f --- /dev/null +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,154 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-4.10", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-4.10", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit4", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-4.10", + "test.name" : "test_failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy index bb517db5ae5..3437fedf68a 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy @@ -101,7 +101,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runFeatures(features, false, true) @@ -116,7 +115,6 @@ class CucumberTest extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -128,16 +126,15 @@ class CucumberTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | features | quarantined | retried + testcaseName | features | quarantined | retried "test-quarantined-failed-atr" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") - ] | [ + ] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -149,15 +146,29 @@ class CucumberTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | features | quarantined | known + testcaseName | features | quarantined | known "test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") - ] | [ + ] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") ] "test-quarantined-failed-efd" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") - ] | [] + ] | [] + } + + def "test disabled #testcaseName"() { + givenDisabledTests(disabled) + + runFeatures(features, false, true) + + assertSpansData(testcaseName) + + where: + testcaseName | features | disabled + "test-disabled-failed" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [ + new TestFQN("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition") + ] } private String parameterizedTestNameSuffix() { diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..1eecdb66ac7 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,146 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-5", + "test.status" : "skip", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "junit.test_suite", + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-5", + "test.name" : "Addition", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.status" : "skip", + "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", + "test.test_management.is_test_disabled" : "true", + "test.traits" : "{\"category\":[\"foo\"]}", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "cucumber-junit-5", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "cucumber-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "cucumber", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "cucumber-junit-5", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "junit.test_module", + "resource" : "cucumber-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy index dbfa08b7554..01a45cc1aae 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/groovy/SpockTest.groovy @@ -133,7 +133,6 @@ class SpockTest extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, true) @@ -147,7 +146,6 @@ class SpockTest extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -159,12 +157,11 @@ class SpockTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | retried + testcaseName | tests | quarantined | retried "test-quarantined-failed-atr" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] | [new TestFQN("org.example.TestFailedSpock", "test failed")] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -176,11 +173,24 @@ class SpockTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | known + testcaseName | tests | quarantined | known "test-quarantined-failed-known" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] | [new TestFQN("org.example.TestFailedSpock", "test failed")] "test-quarantined-failed-efd" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] | [] } + def "test disabled #testcaseName"() { + givenDisabledTests(disabled) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | disabled + "test-disabled-failed" | [TestFailedSpock] | [new TestFQN("org.example.TestFailedSpock", "test failed")] + "test-disabled-failed-parameterized" | [TestFailedParameterizedSpock] | [new TestFQN("org.example.TestFailedParameterizedSpock", "test add 4 and 4")] + } + private static void runTests(List> classes, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/events.ftl new file mode 100644 index 00000000000..1e5013b9858 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed-parameterized/events.ftl @@ -0,0 +1,203 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.source.file" : "dummy_source_path", + "test.status" : "pass", + "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedParameterizedSpock", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.name" : "test add 1 and 2", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 1 and 2\"}}", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.status" : "pass", + "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterizedSpock.test add 1 and 2", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.name" : "test add 4 and 4", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 4 and 4\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterizedSpock", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterizedSpock.test add 4 and 4", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "spock-junit-5", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.status" : "pass", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..1839256d0cb --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,155 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedSpock", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedSpock", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.name" : "test failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test failed()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedSpock", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedSpock.test failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "spock-junit-5", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "spock", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "spock-junit-5", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "junit.test_module", + "resource" : "spock-junit-5", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy index 08d6b877a3f..53417a48fb9 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/groovy/JUnit5Test.groovy @@ -166,7 +166,6 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, true) @@ -180,7 +179,6 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -192,12 +190,11 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | retried + testcaseName | tests | quarantined | retried "test-quarantined-failed-atr" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -209,11 +206,24 @@ class JUnit5Test extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | known + testcaseName | tests | quarantined | known "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [] } + def "test disabled #testcaseName"() { + givenDisabledTests(disabled) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | disabled + "test-disabled-failed" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-disabled-failed-parameterized" | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "test_failed_parameterized")] + } + protected void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.startForcefully() diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/events.ftl new file mode 100644 index 00000000000..9154f5df50f --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed-parameterized/events.ftl @@ -0,0 +1,155 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-5.3", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.name" : "test_failed_parameterized", + "test.parameters" : "{\"metadata\":{\"test_name\":\"test_failed_parameterized(int, int, int)\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed_parameterized(III)V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.test_failed_parameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..65c6c242b85 --- /dev/null +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,154 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "junit-5.3", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "junit.test_session", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2} + }, + "name" : "junit.test_module", + "resource" : "junit-5.3", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "junit.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "junit", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "junit5", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "junit-5.3", + "test.name" : "test_failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy index df54a3f96df..857e844dffc 100644 --- a/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/src/test/groovy/KarateTest.groovy @@ -97,7 +97,6 @@ class KarateTest extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, true) @@ -109,6 +108,20 @@ class KarateTest extends CiVisibilityInstrumentationTest { "test-quarantined-failed" | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")] } + def "test disabled #testcaseName"() { + Assumptions.assumeTrue(isSkippingSupported(FileUtils.KARATE_VERSION)) + + givenDisabledTests(disabled) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | disabled + "test-disabled-failed" | [TestFailedKarate] | [new TestFQN("[org/example/test_failed] test failed", "second scenario")] + } + private void runTests(List> tests, boolean expectSuccess = true) { TestEventsHandlerHolder.start() diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..c0c21c703f5 --- /dev/null +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,211 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "component" : "karate", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "step.name" : "* print 'first'" + }, + "metrics" : { + "step.endLine" : 4, + "step.startLine" : 4 + }, + "name" : "karate.step", + "parent_id" : ${content_parent_id}, + "resource" : "* print 'first'", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start}, + "trace_id" : ${content_trace_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.status" : "pass", + "test.suite" : "[org/example/test_failed] test failed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count} + }, + "name" : "karate.test_suite", + "resource" : "[org/example/test_failed] test failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.name" : "first scenario", + "test.status" : "pass", + "test.suite" : "[org/example/test_failed] test failed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_2}, + "resource" : "[org/example/test_failed] test failed.first scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_parent_id}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.name" : "second scenario", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.status" : "skip", + "test.suite" : "[org/example/test_failed] test failed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test", + "parent_id" : ${content_parent_id_2}, + "resource" : "[org/example/test_failed] test failed.second scenario", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "karate", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "pass", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "karate.test_session", + "resource" : "karate", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "karate", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "karate", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "karate", + "test.status" : "pass", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "karate.test_module", + "resource" : "karate", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_6}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy index 62f67ae5efa..5271da60db9 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy +++ b/dd-java-agent/instrumentation/scalatest/src/test/groovy/ScalatestTest.groovy @@ -103,7 +103,6 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { } def "test quarantined #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, true) @@ -116,7 +115,6 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { } def "test quarantined auto-retries #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -128,12 +126,11 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | retried + testcaseName | tests | quarantined | retried "test-quarantined-failed-atr" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] } def "test quarantined early flakiness detection #testcaseName"() { - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -145,11 +142,23 @@ class ScalatestTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | known + testcaseName | tests | quarantined | known "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] | [] } + def "test disabled #testcaseName"() { + givenDisabledTests(disabled) + + runTests(tests, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | disabled + "test-disabled-failed" | [TestFailed] | [new TestFQN("org.example.TestFailed", "Example.add adds two numbers")] + } + @Override String instrumentedLibraryName() { return "scalatest" diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/coverages.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/events.ftl new file mode 100644 index 00000000000..bbf66a80914 --- /dev/null +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-disabled-failed/events.ftl @@ -0,0 +1,151 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "scalatest.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.name" : "Example.add adds two numbers", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.Example.add adds two numbers", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "scalatest", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "scalatest.test_session", + "resource" : "scalatest", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "scalatest", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "scalatest", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "scalatest", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "scalatest.test_module", + "resource" : "scalatest", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index dd8e1541027..170b6580af4 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -158,7 +158,6 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { def "test quarantined #testcaseName"() { Assumptions.assumeTrue(isExceptionSuppressionSupported()) - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) runTests(tests, null, true) @@ -174,7 +173,6 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { def "test quarantined auto-retries #testcaseName"() { Assumptions.assumeTrue(isExceptionSuppressionSupported()) - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenFlakyRetryEnabled(true) @@ -186,14 +184,13 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | retried + testcaseName | tests | quarantined | retried "test-quarantined-failed-atr-${version()}" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] } def "test quarantined early flakiness detection #testcaseName"() { Assumptions.assumeTrue(isExceptionSuppressionSupported()) - givenTestManagementEnabled(true) givenQuarantinedTests(quarantined) givenEarlyFlakinessDetectionEnabled(true) @@ -205,11 +202,26 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { assertSpansData(testcaseName) where: - testcaseName | tests | quarantined | known + testcaseName | tests | quarantined | known "test-quarantined-failed-known" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [new TestFQN("org.example.TestFailed", "test_failed")] "test-quarantined-failed-efd" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] | [] } + def "test disabled #testcaseName"() { + Assumptions.assumeTrue(isExceptionSuppressionSupported()) + + givenDisabledTests(disabled) + + runTests(tests, null, true) + + assertSpansData(testcaseName) + + where: + testcaseName | tests | disabled + "test-disabled-failed-${version()}" | [TestFailed] | [new TestFQN("org.example.TestFailed", "test_failed")] + "test-disabled-failed-parameterized" | [TestFailedParameterized] | [new TestFQN("org.example.TestFailedParameterized", "parameterized_test_succeed")] + } + private static boolean isEFDSupported() { currentTestNGVersion >= testNGv75 } diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/events.ftl new file mode 100644 index 00000000000..83e2f2be3c1 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-7/events.ftl @@ -0,0 +1,154 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.name" : "test_failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-7", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "testng.test_module", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/events.ftl new file mode 100644 index 00000000000..a2dd675ebec --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-disabled-failed-parameterized/events.ftl @@ -0,0 +1,205 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.name" : "parameterized_test_succeed", + "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.parameterized_test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.name" : "parameterized_test_succeed", + "test.parameters" : "{\"arguments\":{\"0\":\"\\\"goodbye\\\"\",\"1\":\"false\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.parameterized_test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-7", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-7", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "testng.test_module", + "resource" : "testng-7", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java index 6442aeed041..84dced79146 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/CiVisibilityCountMetric.java @@ -17,6 +17,7 @@ import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner; import datadog.trace.api.civisibility.telemetry.tag.HasFailedAllRetries; import datadog.trace.api.civisibility.telemetry.tag.ImpactedTestsDetectionEnabled; +import datadog.trace.api.civisibility.telemetry.tag.IsDisabled; import datadog.trace.api.civisibility.telemetry.tag.IsHeadless; import datadog.trace.api.civisibility.telemetry.tag.IsModified; import datadog.trace.api.civisibility.telemetry.tag.IsNew; @@ -73,6 +74,7 @@ public enum CiVisibilityCountMetric { IsNew.class, IsModified.class, IsQuarantined.class, + IsDisabled.class, IsRetry.class, HasFailedAllRetries.class, RetryReason.class, diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/IsDisabled.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/IsDisabled.java new file mode 100644 index 00000000000..eff53a82b35 --- /dev/null +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/IsDisabled.java @@ -0,0 +1,12 @@ +package datadog.trace.api.civisibility.telemetry.tag; + +import datadog.trace.api.civisibility.telemetry.TagValue; + +public enum IsDisabled implements TagValue { + TRUE; + + @Override + public String asString() { + return "is_disabled:true"; + } +} diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/SkipReason.java b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/SkipReason.java index 258370d0d97..0457b3a6067 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/SkipReason.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/telemetry/tag/SkipReason.java @@ -3,14 +3,15 @@ import datadog.trace.api.civisibility.telemetry.TagValue; public enum SkipReason implements TagValue { - ITR("Skipped by Datadog Intelligent Test Runner"); + ITR("Skipped by Datadog Test Impact Analysis"), + DISABLED("Flaky test is disabled by Datadog"); private final String s; private final String description; SkipReason(String description) { this.description = description; - this.s = "retry_reason:" + name().toLowerCase(); + this.s = "skip_reason:" + name().toLowerCase(); } public String getDescription() { diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java index 0be2715f2d4..5c15a92f206 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/Tags.java @@ -100,6 +100,8 @@ public class Tags { public static final String TEST_TEST_MANAGEMENT_ENABLED = "test.test_management.enabled"; public static final String TEST_TEST_MANAGEMENT_IS_QUARANTINED = "test.test_management.is_quarantined"; + public static final String TEST_TEST_MANAGEMENT_IS_TEST_DISABLED = + "test.test_management.is_test_disabled"; public static final String CI_PROVIDER_NAME = "ci.provider.name"; public static final String CI_PIPELINE_ID = "ci.pipeline.id"; From 58ec02a23b3326e837a8a6724ef7ef6b0ca3dfe5 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 11:59:54 +0100 Subject: [PATCH 3/9] update TIA skip reason --- .../src/test/resources/test-itr-skipping/events.ftl | 2 +- .../test-itr-skipping-parameterized/events.ftl | 2 +- .../src/test/resources/test-itr-skipping/events.ftl | 4 ++-- .../src/test/resources/test-itr-skipping/events.ftl | 2 +- .../src/test/resources/test-parallel/coverages.ftl | 12 ++++++------ .../events.ftl | 2 +- .../test-itr-not-skipping-spec-setup/events.ftl | 2 +- .../test-itr-skipping-parameterized/events.ftl | 2 +- .../test-itr-skipping-spec-setup/events.ftl | 6 +++--- .../src/test/resources/test-itr-skipping/events.ftl | 4 ++-- .../test-itr-skipping-parametrized/events.ftl | 2 +- .../src/test/resources/test-itr-skipping/events.ftl | 4 ++-- .../test-itr-skipping-parameterized/events.ftl | 2 +- .../src/test/resources/test-itr-skipping/events.ftl | 2 +- .../src/test/resources/test-itr-skipping/events.ftl | 2 +- .../events.ftl | 2 +- .../test-itr-skipping-parameterized/events.ftl | 2 +- .../src/test/resources/test-itr-skipping/events.ftl | 4 ++-- .../events.ftl | 2 +- .../test-itr-skipping-parameterized/events.ftl | 2 +- .../src/test/resources/test-itr-skipping/events.ftl | 4 ++-- 21 files changed, 33 insertions(+), 33 deletions(-) diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/events.ftl index 3a03dcf7ddc..badbf34c0ac 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/test/resources/test-itr-skipping/events.ftl @@ -49,7 +49,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "cucumber-junit-4", "test.name" : "Addition", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.status" : "skip", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping-parameterized/events.ftl index 99279c2186d..73997857d7c 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping-parameterized/events.ftl @@ -129,7 +129,7 @@ "test.module" : "junit-4.10", "test.name" : "parameterized_test_succeed", "test.parameters" : "{\"metadata\":{\"test_name\":\"parameterized_test_succeed[str1]\"}}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "parameterized_test_succeed()V", diff --git a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping/events.ftl index 6501b944066..7e827553f98 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/junit-4.10/src/test/resources/test-itr-skipping/events.ftl @@ -128,7 +128,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "junit-4.10", "test.name" : "test_another_succeed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_another_succeed()V", @@ -178,7 +178,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "junit-4.10", "test.name" : "test_failed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-skipping/events.ftl index e23503e1f50..fc457bb16e8 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-itr-skipping/events.ftl @@ -49,7 +49,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "cucumber-junit-5", "test.name" : "Addition", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.status" : "skip", "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic.feature:Basic Arithmetic", diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl index 8e1e6d30f80..4b612855288 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-parallel/coverages.ftl @@ -1,15 +1,15 @@ [ { "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic_skipped.feature" + "filename" : "org/example/cucumber/calculator/basic_arithmetic.feature" } ], - "span_id" : ${content_span_id_3}, + "span_id" : ${content_span_id}, "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id_2} + "test_suite_id" : ${content_test_suite_id} }, { "files" : [ { - "filename" : "org/example/cucumber/calculator/basic_arithmetic.feature" + "filename" : "org/example/cucumber/calculator/basic_arithmetic_skipped.feature" } ], - "span_id" : ${content_span_id}, + "span_id" : ${content_span_id_3}, "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id} + "test_suite_id" : ${content_test_suite_id_2} } ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl index 3cc08985e18..f9a5b184a06 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-parameterized-spec-setup/events.ftl @@ -55,7 +55,7 @@ "test.module" : "spock-junit-5", "test.name" : "test add 1 and 2", "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 1 and 2\"}}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl index 85ae2fefc79..e896c6521ca 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-not-skipping-spec-setup/events.ftl @@ -102,7 +102,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "spock-junit-5", "test.name" : "test success", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test success()V", diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized/events.ftl index 26294a20d35..9931463d676 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-parameterized/events.ftl @@ -55,7 +55,7 @@ "test.module" : "spock-junit-5", "test.name" : "test add 1 and 2", "test.parameters" : "{\"metadata\":{\"test_name\":\"test add 1 and 2\"}}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test add #a and #b(Ljava/lang/Object;Ljava/lang/Object;Ljava/lang/Object;)V", diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl index 87d0bcc6414..d9e1bc83f10 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping-spec-setup/events.ftl @@ -13,7 +13,7 @@ "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "spock-junit-5", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.source.file" : "dummy_source_path", "test.status" : "skip", "test.suite" : "org.example.TestSucceedSetupSpecSpock", @@ -55,7 +55,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "spock-junit-5", "test.name" : "test another success", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test another success()V", @@ -105,7 +105,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "spock-junit-5", "test.name" : "test success", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test success()V", diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping/events.ftl index 97962a20be1..c7fcdbd7b39 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/test/resources/test-itr-skipping/events.ftl @@ -13,7 +13,7 @@ "test.framework" : "spock", "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "spock-junit-5", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.source.file" : "dummy_source_path", "test.status" : "skip", "test.suite" : "org.example.TestSucceedSpock", @@ -55,7 +55,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "spock-junit-5", "test.name" : "test success", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test success()V", diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping-parametrized/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping-parametrized/events.ftl index d68208122b0..ac6c2e94a43 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping-parametrized/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping-parametrized/events.ftl @@ -129,7 +129,7 @@ "test.module" : "junit-5.3", "test.name" : "test_parameterized", "test.parameters" : "{\"metadata\":{\"test_name\":\"[1] 0, 0, 0, some:\\\"parameter\\\"\"}}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_parameterized(IIILjava/lang/String;)V", diff --git a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping/events.ftl index 41ebfa06810..ae30335e16f 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/junit-5.3/src/test/resources/test-itr-skipping/events.ftl @@ -128,7 +128,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "junit-5.3", "test.name" : "test_another_succeed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_another_succeed()V", @@ -178,7 +178,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "junit-5.3", "test.name" : "test_failed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping-parameterized/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping-parameterized/events.ftl index 2bbfe0eb1e1..5300e051e4c 100644 --- a/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping-parameterized/events.ftl @@ -122,7 +122,7 @@ "test.module" : "karate", "test.name" : "first scenario as an outline", "test.parameters" : "{\"param\":\"'a'\",\"value\":\"aa\"}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.status" : "skip", "test.suite" : "[org/example/test_parameterized] test parameterized", diff --git a/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping/events.ftl index a22d4e55e40..2a493942a66 100644 --- a/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/karate/src/test/resources/test-itr-skipping/events.ftl @@ -74,7 +74,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "karate", "test.name" : "first scenario", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.status" : "skip", "test.suite" : "[org/example/test_succeed] test succeed", diff --git a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-itr-skipping/events.ftl index 833d60de85e..14499b7b4d5 100644 --- a/dd-java-agent/instrumentation/scalatest/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/scalatest/src/test/resources/test-itr-skipping/events.ftl @@ -54,7 +54,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "scalatest", "test.name" : "Example.add adds two numbers", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.status" : "skip", diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl index 77b2298b0dc..101803b9da0 100644 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl @@ -54,7 +54,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "testng-6", "test.name" : "testMethod", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "testMethod()V", diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-parameterized/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-parameterized/events.ftl index 3c9beb20747..3ef242cced7 100644 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping-parameterized/events.ftl @@ -55,7 +55,7 @@ "test.module" : "testng-6", "test.name" : "parameterized_test_succeed", "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping/events.ftl index 3e5a6aa4ebe..f411899b59b 100644 --- a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-itr-skipping/events.ftl @@ -54,7 +54,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "testng-6", "test.name" : "test_another_succeed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_another_succeed()V", @@ -104,7 +104,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "testng-6", "test.name" : "test_failed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl index 12689eb4bdd..617beac8e9f 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-factory-data-provider/events.ftl @@ -54,7 +54,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "testng-7", "test.name" : "testMethod", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "testMethod()V", diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-parameterized/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-parameterized/events.ftl index 94d435101c2..d2e96b31a3a 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-parameterized/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping-parameterized/events.ftl @@ -55,7 +55,7 @@ "test.module" : "testng-7", "test.name" : "parameterized_test_succeed", "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", diff --git a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping/events.ftl b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping/events.ftl index fb3b3bf0ecc..07fe8900d80 100644 --- a/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping/events.ftl +++ b/dd-java-agent/instrumentation/testng/testng-7/src/test/resources/test-itr-skipping/events.ftl @@ -54,7 +54,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "testng-7", "test.name" : "test_another_succeed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_another_succeed()V", @@ -104,7 +104,7 @@ "test.framework_version" : ${content_meta_test_framework_version}, "test.module" : "testng-7", "test.name" : "test_failed", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "dummy_source_path", "test.source.method" : "test_failed()V", From 52acaed4e17688c03152b457b4b6294a97143cb4 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 14:29:04 +0100 Subject: [PATCH 4/9] add disabled smoke test --- .../datadog/smoketest/MockBackend.groovy | 12 +- .../datadog/smoketest/MavenSmokeTest.groovy | 19 +- .../coverages.ftl | 1 + .../events.ftl | 326 ++++++++++++++++++ .../pom.xml | 62 ++++ .../main/java/datadog/smoke/Calculator.java | 11 + .../test/java/datadog/smoke/TestFailed.java | 14 + .../events.ftl | 2 +- 8 files changed, 444 insertions(+), 3 deletions(-) create mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl create mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl create mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/pom.xml create mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/main/java/datadog/smoke/Calculator.java create mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java diff --git a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy index 428882bb5a0..3156aa7fa09 100644 --- a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy +++ b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy @@ -95,6 +95,15 @@ class MockBackend implements AutoCloseable { ]) } + void givenDisabledTests(String module, String suite, String name) { + testManagement.add([ + "module": module, + "suite": suite, + "name": name, + "properties": ["disabled": true] + ]) + } + String getIntakeUrl() { return intakeServer.address.toString() } @@ -245,7 +254,8 @@ class MockBackend implements AutoCloseable { for (Map test : testManagement) { Map suites = modules.computeIfAbsent("${test.module}", k -> [:]).computeIfAbsent("suites", k -> [:]) Map tests = suites.computeIfAbsent("${test.suite}", k -> [:]).computeIfAbsent("tests", k -> [:]) - tests.computeIfAbsent("${test.name}", k -> [:]).put("properties", test.properties) + Map properties = tests.computeIfAbsent("${test.name}", k -> [:]).computeIfAbsent("properties", k -> [:]) + properties.putAll(test.properties) } String testManagementResponse = """{ "modules": ${JSON_MAPPER.writeValueAsString(modules)} }""" diff --git a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy index 1615c6cf955..fc1f230c9da 100644 --- a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy +++ b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy @@ -1,7 +1,6 @@ package datadog.smoketest import datadog.trace.api.Config -import datadog.trace.api.civisibility.config.TestFQN import datadog.trace.api.config.CiVisibilityConfig import datadog.trace.api.config.GeneralConfig import datadog.trace.civisibility.CiVisibilitySmokeTest @@ -146,6 +145,24 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { "test_successful_maven_run_quarantined_tests" | "3.9.9" } + def "test disabled"() { + givenWrapperPropertiesFile(mavenVersion) + givenMavenProjectFiles(projectName) + givenMavenDependenciesAreLoaded(projectName, mavenVersion) + + mockBackend.givenTestManagement(true) + mockBackend.givenDisabledTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") + + def exitCode = whenRunningMavenBuild([], []) + assert exitCode == 0 + + verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(8), mockBackend.waitForCoverages(0)) + + where: + projectName | mavenVersion + "test_successful_maven_run_disabled_tests" | "3.9.9" + } + private void givenWrapperPropertiesFile(String mavenVersion) { def distributionUrl = "https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${mavenVersion}/apache-maven-${mavenVersion}-bin.zip" diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl new file mode 100644 index 00000000000..03c19cb49ab --- /dev/null +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl @@ -0,0 +1,326 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "maven", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_session_end", + "test.code_coverage.enabled" : "true", + "test.command" : "mvn -B test", + "test.framework" : "junit4", + "test.framework_version" : "4.13.2", + "test.itr.tests_skipping.enabled" : "true", + "test.itr.tests_skipping.type" : "test", + "test.status" : "pass", + "test.test_management.enabled" : "true", + "test.toolchain" : ${content_meta_test_toolchain}, + "test.type" : "test", + "test_session.name" : "mvn -B test" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.itr.tests_skipping.count" : 0 + }, + "name" : "maven.test_session", + "resource" : "Maven Smoke Tests Project", + "service" : "test-maven-service", + "start" : ${content_start}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.test.is_user_provided_service" : "true", + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "maven", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_module_end", + "test.code_coverage.enabled" : "true", + "test.command" : "mvn -B test", + "test.execution" : "maven-surefire-plugin:test:default-test", + "test.framework" : "junit4", + "test.framework_version" : "4.13.2", + "test.itr.tests_skipping.enabled" : "true", + "test.itr.tests_skipping.type" : "test", + "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", + "test.status" : "pass", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "mvn -B test" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "test.itr.tests_skipping.count" : 0 + }, + "name" : "maven.test_module", + "resource" : "Maven Smoke Tests Project maven-surefire-plugin default-test", + "service" : "test-maven-service", + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "_dd.test.is_user_provided_service" : "true", + "env" : "integration-test", + "execution" : "default-compile", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "plugin" : "maven-compiler-plugin", + "project" : "Maven Smoke Tests Project", + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version} + }, + "metrics" : { }, + "name" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_compile", + "parent_id" : ${content_test_session_id}, + "resource" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_compile", + "service" : "test-maven-service", + "span_id" : ${content_span_id}, + "start" : ${content_start_3}, + "trace_id" : ${content_test_session_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_4}, + "_dd.test.is_user_provided_service" : "true", + "env" : "integration-test", + "execution" : "default-testCompile", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "plugin" : "maven-compiler-plugin", + "project" : "Maven Smoke Tests Project", + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version} + }, + "metrics" : { }, + "name" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_testCompile", + "parent_id" : ${content_test_session_id}, + "resource" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_testCompile", + "service" : "test-maven-service", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_4}, + "trace_id" : ${content_test_session_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_5}, + "_dd.test.is_user_provided_service" : "true", + "env" : "integration-test", + "execution" : "default-resources", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "plugin" : "maven-resources-plugin", + "project" : "Maven Smoke Tests Project", + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version} + }, + "metrics" : { }, + "name" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_resources", + "parent_id" : ${content_test_session_id}, + "resource" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_resources", + "service" : "test-maven-service", + "span_id" : ${content_span_id_3}, + "start" : ${content_start_5}, + "trace_id" : ${content_test_session_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_6}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_6}, + "_dd.test.is_user_provided_service" : "true", + "env" : "integration-test", + "execution" : "default-testResources", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "plugin" : "maven-resources-plugin", + "project" : "Maven Smoke Tests Project", + "runtime-id" : ${content_meta_runtime_id}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version} + }, + "metrics" : { }, + "name" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_testResources", + "parent_id" : ${content_test_session_id}, + "resource" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_testResources", + "service" : "test-maven-service", + "span_id" : ${content_span_id_4}, + "start" : ${content_start_6}, + "trace_id" : ${content_test_session_id} + }, + "type" : "span", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_7}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_7}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "junit", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id_2}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test_suite_end", + "test.framework" : "junit4", + "test.framework_version" : "4.13.2", + "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", + "test.source.file" : "src/test/java/datadog/smoke/TestFailed.java", + "test.status" : "skip", + "test.suite" : "datadog.smoke.TestFailed", + "test.type" : "test", + "test_session.name" : "mvn -B test" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id_2}, + "test.source.end" : 14, + "test.source.start" : 7 + }, + "name" : "junit.test_suite", + "resource" : "datadog.smoke.TestFailed", + "service" : "test-maven-service", + "start" : ${content_start_7}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_8}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_8}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "junit", + "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id_2}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test", + "test.framework" : "junit4", + "test.framework_version" : "4.13.2", + "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", + "test.name" : "test_failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "src/test/java/datadog/smoke/TestFailed.java", + "test.source.method" : "test_failed()V", + "test.status" : "skip", + "test.suite" : "datadog.smoke.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "mvn -B test" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id_2}, + "test.source.end" : 12, + "test.source.start" : 9 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.smoke.TestFailed.test_failed", + "service" : "test-maven-service", + "span_id" : ${content_span_id_5}, + "start" : ${content_start_8}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +} ] \ No newline at end of file diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/pom.xml b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/pom.xml new file mode 100644 index 00000000000..48f92df3632 --- /dev/null +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/pom.xml @@ -0,0 +1,62 @@ + + + + 4.0.0 + com.datadog.ci.test + maven-smoke-test + 1.0-SNAPSHOT + Maven Smoke Tests Project + + + 8 + 8 + UTF-8 + + + + + + + false + + central + Central Repository + https://repo.maven.apache.org/maven2 + + + + + + never + + + false + + central + Central Repository + https://repo.maven.apache.org/maven2 + + + + + + junit + junit + 4.13.2 + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0 + + + + + diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/main/java/datadog/smoke/Calculator.java b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/main/java/datadog/smoke/Calculator.java new file mode 100644 index 00000000000..2f4461a279d --- /dev/null +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/main/java/datadog/smoke/Calculator.java @@ -0,0 +1,11 @@ +package datadog.smoke; + +public class Calculator { + public static int add(int a, int b) { + return a + b; + } + + public static int subtract(int a, int b) { + return a - b; + } +} diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java new file mode 100644 index 00000000000..10580282e38 --- /dev/null +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java @@ -0,0 +1,14 @@ +package datadog.smoke; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class TestFailed { + + @Test + public void test_failed() { + assertTrue(Calculator.add(2, 2) == 5); + } + +} diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl index eda5ccb7cd7..97d0aebcff6 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl @@ -293,13 +293,13 @@ "span.kind" : "test", "test.framework" : "junit4", "test.framework_version" : "4.13.2", - "test.management.is_quarantined" : "true", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_failed", "test.source.file" : "src/test/java/datadog/smoke/TestFailed.java", "test.source.method" : "test_failed()V", "test.status" : "fail", "test.suite" : "datadog.smoke.TestFailed", + "test.test_management.is_quarantined" : "true", "test.type" : "test", "test_session.name" : "mvn -B test" }, From aac26fa015632c90ed1d429268938b1e330c9009 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 15:03:48 +0100 Subject: [PATCH 5/9] update TIA skip reason in smoke tests --- .../resources/test-succeed-junit-5/events.ftl | 2 +- .../events.ftl | 2 +- .../coverages.ftl | 22 +++++++++---------- .../events.ftl | 2 +- .../test-succeed-old-gradle/events.ftl | 2 +- .../test_successful_maven_run/events.ftl | 2 +- .../events.ftl | 2 +- .../events.ftl | 2 +- .../events.ftl | 2 +- .../events.ftl | 2 +- .../events.ftl | 2 +- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/dd-smoke-tests/gradle/src/test/resources/test-succeed-junit-5/events.ftl b/dd-smoke-tests/gradle/src/test/resources/test-succeed-junit-5/events.ftl index e691dd11c14..d7ccd416122 100644 --- a/dd-smoke-tests/gradle/src/test/resources/test-succeed-junit-5/events.ftl +++ b/dd-smoke-tests/gradle/src/test/resources/test-succeed-junit-5/events.ftl @@ -308,7 +308,7 @@ "test.framework_version" : "5.9.3", "test.module" : ":test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/gradle/src/test/resources/test-succeed-legacy-instrumentation/events.ftl b/dd-smoke-tests/gradle/src/test/resources/test-succeed-legacy-instrumentation/events.ftl index a3674672a94..667146df0b8 100644 --- a/dd-smoke-tests/gradle/src/test/resources/test-succeed-legacy-instrumentation/events.ftl +++ b/dd-smoke-tests/gradle/src/test/resources/test-succeed-legacy-instrumentation/events.ftl @@ -258,7 +258,7 @@ "test.framework_version" : "4.10", "test.module" : ":test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/gradle/src/test/resources/test-succeed-multi-forks-legacy-instrumentation/coverages.ftl b/dd-smoke-tests/gradle/src/test/resources/test-succeed-multi-forks-legacy-instrumentation/coverages.ftl index 2fe41431bbf..567ef02b425 100644 --- a/dd-smoke-tests/gradle/src/test/resources/test-succeed-multi-forks-legacy-instrumentation/coverages.ftl +++ b/dd-smoke-tests/gradle/src/test/resources/test-succeed-multi-forks-legacy-instrumentation/coverages.ftl @@ -1,15 +1,4 @@ [ { - "files" : [ { - "bitmap" : "gAw=", - "filename" : "src/test/java/datadog/smoke/TestSucceed.java" - }, { - "bitmap" : "IA==", - "filename" : "src/main/java/datadog/smoke/Calculator.java" - } ], - "span_id" : ${content_span_id_4}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id} -}, { "files" : [ { "bitmap" : "AAE=", "filename" : "src/main/java/datadog/smoke/Calculator.java" @@ -20,4 +9,15 @@ "span_id" : ${content_span_id_5}, "test_session_id" : ${content_test_session_id}, "test_suite_id" : ${content_test_suite_id_2} +}, { + "files" : [ { + "bitmap" : "gAw=", + "filename" : "src/test/java/datadog/smoke/TestSucceed.java" + }, { + "bitmap" : "IA==", + "filename" : "src/main/java/datadog/smoke/Calculator.java" + } ], + "span_id" : ${content_span_id_4}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} } ] \ No newline at end of file diff --git a/dd-smoke-tests/gradle/src/test/resources/test-succeed-new-instrumentation/events.ftl b/dd-smoke-tests/gradle/src/test/resources/test-succeed-new-instrumentation/events.ftl index 81a424a8f40..7c9a2b80b0c 100644 --- a/dd-smoke-tests/gradle/src/test/resources/test-succeed-new-instrumentation/events.ftl +++ b/dd-smoke-tests/gradle/src/test/resources/test-succeed-new-instrumentation/events.ftl @@ -308,7 +308,7 @@ "test.framework_version" : "4.10", "test.module" : ":test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/gradle/src/test/resources/test-succeed-old-gradle/events.ftl b/dd-smoke-tests/gradle/src/test/resources/test-succeed-old-gradle/events.ftl index e2388b5df4a..09d5d7de93f 100644 --- a/dd-smoke-tests/gradle/src/test/resources/test-succeed-old-gradle/events.ftl +++ b/dd-smoke-tests/gradle/src/test/resources/test-succeed-old-gradle/events.ftl @@ -258,7 +258,7 @@ "test.framework_version" : "4.10", "test.module" : ":test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run/events.ftl index e3521cbff09..ea917278854 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run/events.ftl @@ -382,7 +382,7 @@ "test.framework_version" : "4.13.2", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_builtin_coverage/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_builtin_coverage/events.ftl index e9f52afdec2..05eef47fe75 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_builtin_coverage/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_builtin_coverage/events.ftl @@ -346,7 +346,7 @@ "test.framework_version" : "4.13.2", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_multiple_forks/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_multiple_forks/events.ftl index daad9434fc5..0662a83efbb 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_multiple_forks/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_multiple_forks/events.ftl @@ -382,7 +382,7 @@ "test.framework_version" : "5.9.2", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_0_0/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_0_0/events.ftl index e3521cbff09..ea917278854 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_0_0/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_0_0/events.ftl @@ -382,7 +382,7 @@ "test.framework_version" : "4.13.2", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_5_0/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_5_0/events.ftl index e3521cbff09..ea917278854 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_5_0/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_surefire_3_5_0/events.ftl @@ -382,7 +382,7 @@ "test.framework_version" : "4.13.2", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_with_jacoco_and_argline/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_with_jacoco_and_argline/events.ftl index 4726148dc8c..108563f0ddb 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_with_jacoco_and_argline/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_with_jacoco_and_argline/events.ftl @@ -382,7 +382,7 @@ "test.framework_version" : "4.13.2", "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", "test.name" : "test_to_skip_with_itr", - "test.skip_reason" : "Skipped by Datadog Intelligent Test Runner", + "test.skip_reason" : "Skipped by Datadog Test Impact Analysis", "test.skipped_by_itr" : "true", "test.source.file" : "src/test/java/datadog/smoke/TestSucceed.java", "test.source.method" : "test_to_skip_with_itr()V", From 8eb4cc13eb83c88f78c4d226c4280377eecf24de Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 15:04:27 +0100 Subject: [PATCH 6/9] add test management check to skip instrumentations --- .../junit4/JUnit4CucumberSkipInstrumentation.java | 4 +++- .../instrumentation/junit4/JUnit4SkipInstrumentation.java | 4 +++- .../junit5/JUnit5CucumberSkipInstrumentation.java | 4 +++- .../junit5/JUnit5SpockSkipInstrumentation.java | 4 +++- .../instrumentation/junit5/JUnit5SkipInstrumentation.java | 4 +++- .../trace/instrumentation/karate/KarateTracingHook.java | 3 ++- .../scalatest/ScalatestSkipInstrumentation.java | 4 +++- .../instrumentation/testng/TestNGSkipInstrumentation.java | 4 +++- 8 files changed, 23 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java index cdb999f531a..81c701b841f 100644 --- a/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java @@ -34,7 +34,9 @@ public JUnit4CucumberSkipInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override diff --git a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4SkipInstrumentation.java b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4SkipInstrumentation.java index 87cd9394c48..560c99f2a3f 100644 --- a/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4SkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-4.10/src/main/java/datadog/trace/instrumentation/junit4/JUnit4SkipInstrumentation.java @@ -36,7 +36,9 @@ public JUnit4SkipInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override diff --git a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java index f2c68d8e1d3..08a1843ab2d 100644 --- a/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5CucumberSkipInstrumentation.java @@ -38,7 +38,9 @@ public ElementMatcher.Junction classLoaderMatcher() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override diff --git a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockSkipInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockSkipInstrumentation.java index 298aec07f05..4166c767878 100644 --- a/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/spock-junit-5/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SpockSkipInstrumentation.java @@ -38,7 +38,9 @@ public ElementMatcher.Junction classLoaderMatcher() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override diff --git a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SkipInstrumentation.java b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SkipInstrumentation.java index ba0c9fdbcba..39a44f76c0a 100644 --- a/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SkipInstrumentation.java +++ b/dd-java-agent/instrumentation/junit-5.3/src/main/java/datadog/trace/instrumentation/junit5/JUnit5SkipInstrumentation.java @@ -34,7 +34,9 @@ public JUnit5SkipInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override diff --git a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java index 475fc0f9166..c81b8c0d1da 100644 --- a/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java +++ b/dd-java-agent/instrumentation/karate/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java @@ -114,7 +114,8 @@ public boolean beforeScenario(ScenarioRuntime sr) { String parameters = KarateUtils.getParameters(scenario); Collection categories = scenario.getTagsEffective().getTagKeys(); - if (Config.get().isCiVisibilityTestSkippingEnabled()) { + if (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()) { TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario); SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest); diff --git a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java index 5c5ee59dd63..e885d99079f 100644 --- a/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/scalatest/src/main/java/datadog/trace/instrumentation/scalatest/ScalatestSkipInstrumentation.java @@ -30,7 +30,9 @@ public ScalatestSkipInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override diff --git a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java index c029e399f54..245fd81cfdb 100644 --- a/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java +++ b/dd-java-agent/instrumentation/testng/src/main/java/datadog/trace/instrumentation/testng/TestNGSkipInstrumentation.java @@ -27,7 +27,9 @@ public TestNGSkipInstrumentation() { @Override public boolean isApplicable(Set enabledSystems) { - return super.isApplicable(enabledSystems) && Config.get().isCiVisibilityTestSkippingEnabled(); + return super.isApplicable(enabledSystems) + && (Config.get().isCiVisibilityTestSkippingEnabled() + || Config.get().isCiVisibilityTestManagementEnabled()); } @Override From 8d340fb279206f257301119a3589b65390a11bf5 Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 15:17:06 +0100 Subject: [PATCH 7/9] include test management tests reset in smoke tests --- .../src/main/groovy/datadog/smoketest/MockBackend.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy index 3156aa7fa09..3bf5e2412dd 100644 --- a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy +++ b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy @@ -50,6 +50,7 @@ class MockBackend implements AutoCloseable { skippableTests.clear() flakyTests.clear() + testManagement.clear() changedFiles.clear() } From 80a7054cd31629e3a172d6c1d645dd952526409d Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Wed, 12 Feb 2025 15:57:27 +0100 Subject: [PATCH 8/9] join quarantine and disable maven smoke tests to test management --- .../datadog/smoketest/MavenSmokeTest.groovy | 25 +- .../coverages.ftl | 1 - .../events.ftl | 326 ------------------ .../test/java/datadog/smoke/TestFailed.java | 14 - .../pom.xml | 62 ---- .../main/java/datadog/smoke/Calculator.java | 11 - .../coverages.ftl | 2 +- .../events.ftl | 67 +++- .../pom.xml | 0 .../main/java/datadog/smoke/Calculator.java | 0 .../test/java/datadog/smoke/TestFailed.java | 4 + 11 files changed, 70 insertions(+), 442 deletions(-) delete mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl delete mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl delete mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java delete mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/pom.xml delete mode 100644 dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/main/java/datadog/smoke/Calculator.java rename dd-smoke-tests/maven/src/test/resources/{test_successful_maven_run_quarantined_tests => test_successful_maven_run_test_management}/coverages.ftl (87%) rename dd-smoke-tests/maven/src/test/resources/{test_successful_maven_run_quarantined_tests => test_successful_maven_run_test_management}/events.ftl (85%) rename dd-smoke-tests/maven/src/test/resources/{test_successful_maven_run_disabled_tests => test_successful_maven_run_test_management}/pom.xml (100%) rename dd-smoke-tests/maven/src/test/resources/{test_successful_maven_run_disabled_tests => test_successful_maven_run_test_management}/src/main/java/datadog/smoke/Calculator.java (100%) rename dd-smoke-tests/maven/src/test/resources/{test_successful_maven_run_quarantined_tests => test_successful_maven_run_test_management}/src/test/java/datadog/smoke/TestFailed.java (69%) diff --git a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy index fc1f230c9da..291e2b929b9 100644 --- a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy +++ b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy @@ -127,40 +127,23 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { "test_successful_maven_run_impacted_tests" | "3.9.9" } - def "test quarantine"() { + def "test test management"() { givenWrapperPropertiesFile(mavenVersion) givenMavenProjectFiles(projectName) givenMavenDependenciesAreLoaded(projectName, mavenVersion) mockBackend.givenTestManagement(true) mockBackend.givenQuarantinedTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") + mockBackend.givenDisabledTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_another_failed") def exitCode = whenRunningMavenBuild([], []) assert exitCode == 0 - verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(8), mockBackend.waitForCoverages(1)) + verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(9), mockBackend.waitForCoverages(1)) where: projectName | mavenVersion - "test_successful_maven_run_quarantined_tests" | "3.9.9" - } - - def "test disabled"() { - givenWrapperPropertiesFile(mavenVersion) - givenMavenProjectFiles(projectName) - givenMavenDependenciesAreLoaded(projectName, mavenVersion) - - mockBackend.givenTestManagement(true) - mockBackend.givenDisabledTests("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") - - def exitCode = whenRunningMavenBuild([], []) - assert exitCode == 0 - - verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(8), mockBackend.waitForCoverages(0)) - - where: - projectName | mavenVersion - "test_successful_maven_run_disabled_tests" | "3.9.9" + "test_successful_maven_run_test_management" | "3.9.9" } private void givenWrapperPropertiesFile(String mavenVersion) { diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl deleted file mode 100644 index 8878e547a79..00000000000 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/coverages.ftl +++ /dev/null @@ -1 +0,0 @@ -[ ] \ No newline at end of file diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl deleted file mode 100644 index 03c19cb49ab..00000000000 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/events.ftl +++ /dev/null @@ -1,326 +0,0 @@ -[ { - "content" : { - "duration" : ${content_duration}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid}, - "_dd.test.is_user_provided_service" : "true", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "ci.workspace_path" : ${content_meta_ci_workspace_path}, - "component" : "maven", - "env" : "integration-test", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "runtime-id" : ${content_meta_runtime_id}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version}, - "span.kind" : "test_session_end", - "test.code_coverage.enabled" : "true", - "test.command" : "mvn -B test", - "test.framework" : "junit4", - "test.framework_version" : "4.13.2", - "test.itr.tests_skipping.enabled" : "true", - "test.itr.tests_skipping.type" : "test", - "test.status" : "pass", - "test.test_management.enabled" : "true", - "test.toolchain" : ${content_meta_test_toolchain}, - "test.type" : "test", - "test_session.name" : "mvn -B test" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id}, - "test.itr.tests_skipping.count" : 0 - }, - "name" : "maven.test_session", - "resource" : "Maven Smoke Tests Project", - "service" : "test-maven-service", - "start" : ${content_start}, - "test_session_id" : ${content_test_session_id} - }, - "type" : "test_session_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_2}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_2}, - "_dd.test.is_user_provided_service" : "true", - "ci.workspace_path" : ${content_meta_ci_workspace_path}, - "component" : "maven", - "env" : "integration-test", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "runtime-id" : ${content_meta_runtime_id}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version}, - "span.kind" : "test_module_end", - "test.code_coverage.enabled" : "true", - "test.command" : "mvn -B test", - "test.execution" : "maven-surefire-plugin:test:default-test", - "test.framework" : "junit4", - "test.framework_version" : "4.13.2", - "test.itr.tests_skipping.enabled" : "true", - "test.itr.tests_skipping.type" : "test", - "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", - "test.status" : "pass", - "test.test_management.enabled" : "true", - "test.type" : "test", - "test_session.name" : "mvn -B test" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, - "test.itr.tests_skipping.count" : 0 - }, - "name" : "maven.test_module", - "resource" : "Maven Smoke Tests Project maven-surefire-plugin default-test", - "service" : "test-maven-service", - "start" : ${content_start_2}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id} - }, - "type" : "test_module_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_3}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_3}, - "_dd.test.is_user_provided_service" : "true", - "env" : "integration-test", - "execution" : "default-compile", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "plugin" : "maven-compiler-plugin", - "project" : "Maven Smoke Tests Project", - "runtime-id" : ${content_meta_runtime_id}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version} - }, - "metrics" : { }, - "name" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_compile", - "parent_id" : ${content_test_session_id}, - "resource" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_compile", - "service" : "test-maven-service", - "span_id" : ${content_span_id}, - "start" : ${content_start_3}, - "trace_id" : ${content_test_session_id} - }, - "type" : "span", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_4}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_4}, - "_dd.test.is_user_provided_service" : "true", - "env" : "integration-test", - "execution" : "default-testCompile", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "plugin" : "maven-compiler-plugin", - "project" : "Maven Smoke Tests Project", - "runtime-id" : ${content_meta_runtime_id}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version} - }, - "metrics" : { }, - "name" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_testCompile", - "parent_id" : ${content_test_session_id}, - "resource" : "Maven_Smoke_Tests_Project_maven_compiler_plugin_default_testCompile", - "service" : "test-maven-service", - "span_id" : ${content_span_id_2}, - "start" : ${content_start_4}, - "trace_id" : ${content_test_session_id} - }, - "type" : "span", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_5}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_5}, - "_dd.test.is_user_provided_service" : "true", - "env" : "integration-test", - "execution" : "default-resources", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "plugin" : "maven-resources-plugin", - "project" : "Maven Smoke Tests Project", - "runtime-id" : ${content_meta_runtime_id}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version} - }, - "metrics" : { }, - "name" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_resources", - "parent_id" : ${content_test_session_id}, - "resource" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_resources", - "service" : "test-maven-service", - "span_id" : ${content_span_id_3}, - "start" : ${content_start_5}, - "trace_id" : ${content_test_session_id} - }, - "type" : "span", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_6}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_6}, - "_dd.test.is_user_provided_service" : "true", - "env" : "integration-test", - "execution" : "default-testResources", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "plugin" : "maven-resources-plugin", - "project" : "Maven Smoke Tests Project", - "runtime-id" : ${content_meta_runtime_id}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version} - }, - "metrics" : { }, - "name" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_testResources", - "parent_id" : ${content_test_session_id}, - "resource" : "Maven_Smoke_Tests_Project_maven_resources_plugin_default_testResources", - "service" : "test-maven-service", - "span_id" : ${content_span_id_4}, - "start" : ${content_start_6}, - "trace_id" : ${content_test_session_id} - }, - "type" : "span", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_7}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_7}, - "_dd.test.is_user_provided_service" : "true", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "ci.workspace_path" : ${content_meta_ci_workspace_path}, - "component" : "junit", - "env" : "integration-test", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "runtime-id" : ${content_meta_runtime_id_2}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version}, - "span.kind" : "test_suite_end", - "test.framework" : "junit4", - "test.framework_version" : "4.13.2", - "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", - "test.source.file" : "src/test/java/datadog/smoke/TestFailed.java", - "test.status" : "skip", - "test.suite" : "datadog.smoke.TestFailed", - "test.type" : "test", - "test_session.name" : "mvn -B test" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id_2}, - "test.source.end" : 14, - "test.source.start" : 7 - }, - "name" : "junit.test_suite", - "resource" : "datadog.smoke.TestFailed", - "service" : "test-maven-service", - "start" : ${content_start_7}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id} - }, - "type" : "test_suite_end", - "version" : 1 -}, { - "content" : { - "duration" : ${content_duration_8}, - "error" : 0, - "meta" : { - "_dd.p.tid" : ${content_meta__dd_p_tid_8}, - "_dd.test.is_user_provided_service" : "true", - "_dd.tracer_host" : ${content_meta__dd_tracer_host}, - "ci.workspace_path" : ${content_meta_ci_workspace_path}, - "component" : "junit", - "env" : "integration-test", - "language" : "jvm", - "library_version" : ${content_meta_library_version}, - "os.architecture" : ${content_meta_os_architecture}, - "os.platform" : ${content_meta_os_platform}, - "os.version" : ${content_meta_os_version}, - "runtime-id" : ${content_meta_runtime_id_2}, - "runtime.name" : ${content_meta_runtime_name}, - "runtime.vendor" : ${content_meta_runtime_vendor}, - "runtime.version" : ${content_meta_runtime_version}, - "span.kind" : "test", - "test.framework" : "junit4", - "test.framework_version" : "4.13.2", - "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", - "test.name" : "test_failed", - "test.skip_reason" : "Flaky test is disabled by Datadog", - "test.source.file" : "src/test/java/datadog/smoke/TestFailed.java", - "test.source.method" : "test_failed()V", - "test.status" : "skip", - "test.suite" : "datadog.smoke.TestFailed", - "test.test_management.is_test_disabled" : "true", - "test.type" : "test", - "test_session.name" : "mvn -B test" - }, - "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, - "_dd.profiling.enabled" : 0, - "_dd.trace_span_attribute_schema" : 0, - "process_id" : ${content_metrics_process_id_2}, - "test.source.end" : 12, - "test.source.start" : 9 - }, - "name" : "junit.test", - "parent_id" : ${content_parent_id}, - "resource" : "datadog.smoke.TestFailed.test_failed", - "service" : "test-maven-service", - "span_id" : ${content_span_id_5}, - "start" : ${content_start_8}, - "test_module_id" : ${content_test_module_id}, - "test_session_id" : ${content_test_session_id}, - "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id} - }, - "type" : "test", - "version" : 2 -} ] \ No newline at end of file diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java deleted file mode 100644 index 10580282e38..00000000000 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/test/java/datadog/smoke/TestFailed.java +++ /dev/null @@ -1,14 +0,0 @@ -package datadog.smoke; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -public class TestFailed { - - @Test - public void test_failed() { - assertTrue(Calculator.add(2, 2) == 5); - } - -} diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/pom.xml b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/pom.xml deleted file mode 100644 index 48f92df3632..00000000000 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/pom.xml +++ /dev/null @@ -1,62 +0,0 @@ - - - - 4.0.0 - com.datadog.ci.test - maven-smoke-test - 1.0-SNAPSHOT - Maven Smoke Tests Project - - - 8 - 8 - UTF-8 - - - - - - - false - - central - Central Repository - https://repo.maven.apache.org/maven2 - - - - - - never - - - false - - central - Central Repository - https://repo.maven.apache.org/maven2 - - - - - - junit - junit - 4.13.2 - test - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0 - - - - - diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/main/java/datadog/smoke/Calculator.java b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/main/java/datadog/smoke/Calculator.java deleted file mode 100644 index 2f4461a279d..00000000000 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/main/java/datadog/smoke/Calculator.java +++ /dev/null @@ -1,11 +0,0 @@ -package datadog.smoke; - -public class Calculator { - public static int add(int a, int b) { - return a + b; - } - - public static int subtract(int a, int b) { - return a - b; - } -} diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/coverages.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/coverages.ftl similarity index 87% rename from dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/coverages.ftl rename to dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/coverages.ftl index 4f8fc4c15f5..7a24021c56a 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/coverages.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/coverages.ftl @@ -4,7 +4,7 @@ }, { "filename" : "src/test/java/datadog/smoke/TestFailed.java" } ], - "span_id" : ${content_span_id_5}, + "span_id" : ${content_span_id_6}, "test_session_id" : ${content_test_session_id}, "test_suite_id" : ${content_test_suite_id} } ] \ No newline at end of file diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/events.ftl similarity index 85% rename from dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl rename to dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/events.ftl index 97d0aebcff6..6c22e89dc6c 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/events.ftl +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/events.ftl @@ -255,7 +255,7 @@ "_dd.profiling.enabled" : 0, "_dd.trace_span_attribute_schema" : 0, "process_id" : ${content_metrics_process_id_2}, - "test.source.end" : 14, + "test.source.end" : 18, "test.source.start" : 7 }, "name" : "junit.test_suite", @@ -271,7 +271,7 @@ }, { "content" : { "duration" : ${content_duration_8}, - "error" : 1, + "error" : 0, "meta" : { "_dd.p.tid" : ${content_meta__dd_p_tid_8}, "_dd.test.is_user_provided_service" : "true", @@ -279,6 +279,61 @@ "ci.workspace_path" : ${content_meta_ci_workspace_path}, "component" : "junit", "env" : "integration-test", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "os.architecture" : ${content_meta_os_architecture}, + "os.platform" : ${content_meta_os_platform}, + "os.version" : ${content_meta_os_version}, + "runtime-id" : ${content_meta_runtime_id_2}, + "runtime.name" : ${content_meta_runtime_name}, + "runtime.vendor" : ${content_meta_runtime_vendor}, + "runtime.version" : ${content_meta_runtime_version}, + "span.kind" : "test", + "test.framework" : "junit4", + "test.framework_version" : "4.13.2", + "test.module" : "Maven Smoke Tests Project maven-surefire-plugin default-test", + "test.name" : "test_another_failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "src/test/java/datadog/smoke/TestFailed.java", + "test.source.method" : "test_another_failed()V", + "test.status" : "skip", + "test.suite" : "datadog.smoke.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "mvn -B test" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id_2}, + "test.source.end" : 17, + "test.source.start" : 14 + }, + "name" : "junit.test", + "parent_id" : ${content_parent_id}, + "resource" : "datadog.smoke.TestFailed.test_another_failed", + "service" : "test-maven-service", + "span_id" : ${content_span_id_5}, + "start" : ${content_start_8}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_9}, + "error" : 1, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_9}, + "_dd.test.is_user_provided_service" : "true", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "ci.workspace_path" : ${content_meta_ci_workspace_path}, + "component" : "junit", + "env" : "integration-test", "error.stack" : ${content_meta_error_stack}, "error.type" : "java.lang.AssertionError", "language" : "jvm", @@ -304,7 +359,7 @@ "test_session.name" : "mvn -B test" }, "metrics" : { - "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5}, "_dd.profiling.enabled" : 0, "_dd.trace_span_attribute_schema" : 0, "process_id" : ${content_metrics_process_id_2}, @@ -315,12 +370,12 @@ "parent_id" : ${content_parent_id}, "resource" : "datadog.smoke.TestFailed.test_failed", "service" : "test-maven-service", - "span_id" : ${content_span_id_5}, - "start" : ${content_start_8}, + "span_id" : ${content_span_id_6}, + "start" : ${content_start_9}, "test_module_id" : ${content_test_module_id}, "test_session_id" : ${content_test_session_id}, "test_suite_id" : ${content_test_suite_id}, - "trace_id" : ${content_trace_id} + "trace_id" : ${content_trace_id_2} }, "type" : "test", "version" : 2 diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/pom.xml b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/pom.xml similarity index 100% rename from dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/pom.xml rename to dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/pom.xml diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/main/java/datadog/smoke/Calculator.java b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/src/main/java/datadog/smoke/Calculator.java similarity index 100% rename from dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_disabled_tests/src/main/java/datadog/smoke/Calculator.java rename to dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/src/main/java/datadog/smoke/Calculator.java diff --git a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/test/java/datadog/smoke/TestFailed.java b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/src/test/java/datadog/smoke/TestFailed.java similarity index 69% rename from dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/test/java/datadog/smoke/TestFailed.java rename to dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/src/test/java/datadog/smoke/TestFailed.java index 10580282e38..bdf29316598 100644 --- a/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_quarantined_tests/src/test/java/datadog/smoke/TestFailed.java +++ b/dd-smoke-tests/maven/src/test/resources/test_successful_maven_run_test_management/src/test/java/datadog/smoke/TestFailed.java @@ -11,4 +11,8 @@ public void test_failed() { assertTrue(Calculator.add(2, 2) == 5); } + @Test + public void test_another_failed() { + assertTrue(Calculator.add(2, 2) == 5); + } } From 8c8d26980a2359f48ebcc98302bc0f0d957216de Mon Sep 17 00:00:00 2001 From: Daniel Mohedano Date: Thu, 13 Feb 2025 10:13:09 +0100 Subject: [PATCH 9/9] pr fixes --- .../instrumentation/testng/TestNGTest.groovy | 2 - .../test-disabled-failed-6/coverages.ftl | 1 + .../test-disabled-failed-6/events.ftl | 154 +++++++++++++ .../coverages.ftl | 1 + .../events.ftl | 205 ++++++++++++++++++ .../datadog/smoketest/MavenSmokeTest.groovy | 2 +- 6 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/coverages.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/events.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/coverages.ftl create mode 100644 dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/events.ftl diff --git a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy index 170b6580af4..21b990cd4b2 100644 --- a/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy +++ b/dd-java-agent/instrumentation/testng/src/testFixtures/groovy/datadog/trace/instrumentation/testng/TestNGTest.groovy @@ -208,8 +208,6 @@ abstract class TestNGTest extends CiVisibilityInstrumentationTest { } def "test disabled #testcaseName"() { - Assumptions.assumeTrue(isExceptionSuppressionSupported()) - givenDisabledTests(disabled) runTests(tests, null, true) diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/events.ftl new file mode 100644 index 00000000000..34a62ee621d --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-6/events.ftl @@ -0,0 +1,154 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestFailed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.name" : "test_failed", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "test_failed()V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailed", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailed.test_failed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-6", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-6", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_3}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4} + }, + "name" : "testng.test_module", + "resource" : "testng-6", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/coverages.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/coverages.ftl new file mode 100644 index 00000000000..8878e547a79 --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/coverages.ftl @@ -0,0 +1 @@ +[ ] \ No newline at end of file diff --git a/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/events.ftl b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/events.ftl new file mode 100644 index 00000000000..49e969a5cfb --- /dev/null +++ b/dd-java-agent/instrumentation/testng/testng-6/src/test/resources/test-disabled-failed-parameterized/events.ftl @@ -0,0 +1,205 @@ +[ { + "content" : { + "duration" : ${content_duration}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_suite_end", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.source.file" : "dummy_source_path", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}, + "test.source.end" : 19, + "test.source.start" : 11 + }, + "name" : "testng.test_suite", + "resource" : "org.example.TestFailedParameterized", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id} + }, + "type" : "test_suite_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_2}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.name" : "parameterized_test_succeed", + "test.parameters" : "{\"arguments\":{\"0\":\"hello\",\"1\":\"true\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.parameterized_test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id}, + "start" : ${content_start_2}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_3}, + "error" : 0, + "meta" : { + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test", + "test.codeowners" : "[\"owner1\",\"owner2\"]", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.name" : "parameterized_test_succeed", + "test.parameters" : "{\"arguments\":{\"0\":\"\\\"goodbye\\\"\",\"1\":\"false\"}}", + "test.skip_reason" : "Flaky test is disabled by Datadog", + "test.source.file" : "dummy_source_path", + "test.source.method" : "parameterized_test_succeed(Ljava/lang/String;Z)V", + "test.status" : "skip", + "test.suite" : "org.example.TestFailedParameterized", + "test.test_management.is_test_disabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id}, + "test.source.end" : 18, + "test.source.start" : 12 + }, + "name" : "testng.test", + "parent_id" : ${content_parent_id}, + "resource" : "org.example.TestFailedParameterized.parameterized_test_succeed", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "span_id" : ${content_span_id_2}, + "start" : ${content_start_3}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id}, + "test_suite_id" : ${content_test_suite_id}, + "trace_id" : ${content_trace_id_2} + }, + "type" : "test", + "version" : 2 +}, { + "content" : { + "duration" : ${content_duration_4}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_2}, + "_dd.profiling.ctx" : "test", + "_dd.tracer_host" : ${content_meta__dd_tracer_host}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "language" : "jvm", + "library_version" : ${content_meta_library_version}, + "runtime-id" : ${content_meta_runtime_id}, + "span.kind" : "test_session_end", + "test.command" : "testng-6", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}, + "_dd.profiling.enabled" : 0, + "_dd.trace_span_attribute_schema" : 0, + "process_id" : ${content_metrics_process_id} + }, + "name" : "testng.test_session", + "resource" : "testng-6", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_4}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_session_end", + "version" : 1 +}, { + "content" : { + "duration" : ${content_duration_5}, + "error" : 0, + "meta" : { + "_dd.p.tid" : ${content_meta__dd_p_tid_3}, + "component" : "testng", + "dummy_ci_tag" : "dummy_ci_tag_value", + "env" : "none", + "library_version" : ${content_meta_library_version}, + "span.kind" : "test_module_end", + "test.framework" : "testng", + "test.framework_version" : ${content_meta_test_framework_version}, + "test.module" : "testng-6", + "test.status" : "skip", + "test.test_management.enabled" : "true", + "test.type" : "test", + "test_session.name" : "session-name" + }, + "metrics" : { + "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_5} + }, + "name" : "testng.test_module", + "resource" : "testng-6", + "service" : "worker.org.gradle.process.internal.worker.gradleworkermain", + "start" : ${content_start_5}, + "test_module_id" : ${content_test_module_id}, + "test_session_id" : ${content_test_session_id} + }, + "type" : "test_module_end", + "version" : 1 +} ] \ No newline at end of file diff --git a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy index 291e2b929b9..4a82bcd2b04 100644 --- a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy +++ b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy @@ -143,7 +143,7 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { where: projectName | mavenVersion - "test_successful_maven_run_test_management" | "3.9.9" + "test_successful_maven_run_test_management" | "3.9.9" } private void givenWrapperPropertiesFile(String mavenVersion) {