[test optimization] Fix EFD retries in jest (#7637)

Author: juan-fernandez

integration-tests/ci-visibility/test-early-flake-detection/instant-test.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const assert = require('assert')
+
+describe('efd slow retries', () => {
+  it('instant test', () => {
+    assert.strictEqual(1 + 1, 2)
+  })
+})

integration-tests/ci-visibility/test-early-flake-detection/slightly-slow-test.js

@@ -0,0 +1,10 @@
+'use strict'
+
+const assert = require('assert')
+
+describe('efd slow retries', () => {
+  it('slightly slow test', async () => {
+    await new Promise(resolve => setTimeout(resolve, 5100))
+    assert.strictEqual(1 + 1, 2)
+  })
+})

integration-tests/jest/jest.spec.js

@@ -3776,6 +3776,135 @@ describe(`jest@${JEST_VERSION} commonJS`, () => {
       const [[exitCode]] = await Promise.all([once(childProcess, 'exit'), testAssertionsPromise])
       assert.strictEqual(exitCode, 1, 'exit code 1 when suite fails (resolution error, EFD)')
     })
+
+    it('retries a fast new test using the count from the matching slow_test_retries bucket', async () => {
+      receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
+      receiver.setKnownTests({ jest: {} })
+      receiver.setSettings({
+        early_flake_detection: {
+          enabled: true,
+          slow_test_retries: {
+            '5s': 2,
+            '10s': 1,
+          },
+          faulty_session_threshold: 100,
+        },
+        known_tests_enabled: true,
+      })
+
+      const eventsPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+          const events = payloads.flatMap(({ payload }) => payload.events)
+          const tests = events.filter(event => event.type === 'test').map(event => event.content)
+          const testEvents = tests.filter(t => t.resource?.includes('instant-test'))
+          // 1 original + 2 retries from the '5s' bucket (fast test < 5 s)
+          assert.strictEqual(testEvents.length, 3)
+          const efdRetries = testEvents.filter(t =>
+            t.meta[TEST_IS_RETRY] === 'true' && t.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.efd
+          )
+          assert.strictEqual(efdRetries.length, 2)
+          testEvents.forEach(t => assert.strictEqual(t.meta[TEST_IS_NEW], 'true'))
+        })
+
+      childProcess = exec(
+        runTestsCommand,
+        {
+          cwd,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port),
+            TESTS_TO_RUN: 'test-early-flake-detection/instant-test',
+          },
+        }
+      )
+
+      await Promise.all([once(childProcess, 'exit'), eventsPromise])
+    })
+
+    it('retries a slightly slow new test using the count from the matching slow_test_retries bucket', async () => {
+      receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
+      receiver.setKnownTests({ jest: {} })
+      receiver.setSettings({
+        early_flake_detection: {
+          enabled: true,
+          slow_test_retries: {
+            '5s': 3,
+            '10s': 1,
+          },
+          faulty_session_threshold: 100,
+        },
+        known_tests_enabled: true,
+      })
+
+      const eventsPromise = receiver
+        // test runs for ~5100 ms × 2 executions; allow extra time for jest startup + reporting
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+          const events = payloads.flatMap(({ payload }) => payload.events)
+          const tests = events.filter(event => event.type === 'test').map(event => event.content)
+          const testEvents = tests.filter(t => t.resource?.includes('slightly-slow-test'))
+          // 1 original + 1 retry from the '10s' bucket (test takes ~5100 ms, between 5 s and 10 s)
+          assert.strictEqual(testEvents.length, 2)
+          const efdRetries = testEvents.filter(t =>
+            t.meta[TEST_IS_RETRY] === 'true' && t.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.efd
+          )
+          assert.strictEqual(efdRetries.length, 1)
+          testEvents.forEach(t => assert.strictEqual(t.meta[TEST_IS_NEW], 'true'))
+        }, 25_000)
+
+      childProcess = exec(
+        runTestsCommand,
+        {
+          cwd,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port),
+            TESTS_TO_RUN: 'test-early-flake-detection/slightly-slow-test',
+          },
+        }
+      )
+
+      await Promise.all([once(childProcess, 'exit'), eventsPromise])
+    })
+
+    it('aborts retries and tags the test when the test is too slow for any slow_test_retries bucket', async () => {
+      receiver.setInfoResponse({ endpoints: ['/evp_proxy/v4'] })
+      receiver.setKnownTests({ jest: {} })
+      receiver.setSettings({
+        early_flake_detection: {
+          enabled: true,
+          slow_test_retries: {
+            '5s': 3,
+            '10s': 0,
+          },
+          faulty_session_threshold: 100,
+        },
+        known_tests_enabled: true,
+      })
+
+      const eventsPromise = receiver
+        .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
+          const events = payloads.flatMap(({ payload }) => payload.events)
+          const tests = events.filter(event => event.type === 'test').map(event => event.content)
+          const testEvents = tests.filter(t => t.resource?.includes('slightly-slow-test'))
+          // 0 retries — bucket value is 0
+          assert.strictEqual(testEvents.length, 1)
+          const [testEvent] = testEvents
+          assert.strictEqual(testEvent.meta[TEST_IS_NEW], 'true')
+          assert.strictEqual(testEvent.meta[TEST_EARLY_FLAKE_ABORT_REASON], 'slow')
+          assert.ok(!(TEST_IS_RETRY in testEvent.meta), 'should not be retried')
+        }, 20_000)
+
+      childProcess = exec(
+        runTestsCommand,
+        {
+          cwd,
+          env: {
+            ...getCiVisAgentlessConfig(receiver.port),
+            TESTS_TO_RUN: 'test-early-flake-detection/slightly-slow-test',
+          },
+        }
+      )
+
+      await Promise.all([once(childProcess, 'exit'), eventsPromise])
+    })
   })
 
   context('flaky test retries', () => {

packages/datadog-instrumentations/src/jest.js

@@ -21,6 +21,7 @@ const {
   getFormattedJestTestParameters,
   getJestTestName,
   getJestSuitesToRun,
+  getEfdRetryCount,
 } = require('../../datadog-plugin-jest/src/util')
 const { addHook, channel } = require('./helpers/instrument')
 
@@ -76,6 +77,7 @@ let hasUnskippableSuites = false
 let hasForcedToRunSuites = false
 let isEarlyFlakeDetectionEnabled = false
 let earlyFlakeDetectionNumRetries = 0
+let earlyFlakeDetectionSlowTestRetries = {}
 let earlyFlakeDetectionFaultyThreshold = 30
 let isEarlyFlakeDetectionFaulty = false
 let hasFilteredSkippableSuites = false
@@ -95,6 +97,12 @@ const attemptToFixRetriedTestsStatuses = new Map()
 const wrappedWorkers = new WeakSet()
 const testSuiteMockedFiles = new Map()
 const testsToBeRetried = new Set()
+// Per-test: how many EFD retries were determined after the first execution.
+const efdDeterminedRetries = new Map()
+// Tests whose first run exceeded the 5-min threshold — tagged "slow".
+const efdSlowAbortedTests = new Set()
+// Tests added as EFD new-test candidates (not ATF, not impacted).
+const efdNewTestCandidates = new Set()
 const testSuiteAbsolutePathsWithFastCheck = new Set()
 const testSuiteJestObjects = new Map()
 
@@ -197,7 +205,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
       this.isImpactedTestsEnabled = this.testEnvironmentOptions._ddIsImpactedTestsEnabled
 
       if (this.isKnownTestsEnabled) {
-        earlyFlakeDetectionNumRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionNumRetries
+        earlyFlakeDetectionSlowTestRetries = this.testEnvironmentOptions._ddEarlyFlakeDetectionSlowTestRetries ?? {}
         try {
           this.knownTestsForThisSuite = this.getKnownTestsForSuite(this.testEnvironmentOptions._ddKnownTests)
 
@@ -543,11 +551,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
             retriedTestsToNumAttempts.set(testFullName, 0)
             if (this.isEarlyFlakeDetectionEnabled) {
               testsToBeRetried.add(testFullName)
-              this.retryTest({
-                jestEvent: event,
-                retryCount: earlyFlakeDetectionNumRetries,
-                retryType: 'Early flake detection',
-              })
+              efdNewTestCandidates.add(testFullName)
+              // Cloning is deferred to test_done after the first execution,
+              // when we know the duration and can choose the right retry count.
             }
           }
         }
@@ -572,8 +578,8 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
         let attemptToFixFailed = false
         let failedAllTests = false
         let isAttemptToFix = false
+        const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
         if (this.isTestManagementTestsEnabled) {
-          const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
           isAttemptToFix = this.testManagementTestsForThisSuite?.attemptToFix?.includes(testName)
           if (isAttemptToFix) {
             if (attemptToFixRetriedTestsStatuses.has(testName)) {
@@ -598,9 +604,53 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           }
         }
 
+        // EFD dynamic cloning: on first execution of a new EFD candidate,
+        // determine the retry count from the test's duration.
+        if (
+          this.isEarlyFlakeDetectionEnabled &&
+          this.isKnownTestsEnabled &&
+          efdNewTestCandidates.has(testName) &&
+          event.test.invocations === 1 &&
+          !efdDeterminedRetries.has(testName)
+        ) {
+          const durationMs = event.test.duration ?? 0
+          const retryCount = getEfdRetryCount(durationMs, earlyFlakeDetectionSlowTestRetries)
+          efdDeterminedRetries.set(testName, retryCount)
+          if (retryCount > 0) {
+            // Temporarily adjust jest-circus state so that retry tests are registered
+            // into the correct describe block and bypass the "tests have started" guard.
+            //
+            // Problem 1 (jest-circus ≤24): currentDescribeBlock points to ROOT during
+            // execution, and ROOT's tests loop already finished before children ran.
+            //
+            // Problem 2 (jest-circus ≥27): `hasStarted = true` causes `test()` to throw
+            // "Cannot add a test after tests have started running".
+            //
+            // Fix: temporarily point currentDescribeBlock to the test's parent (so retries
+            // land in the still-iterating children array) and set hasStarted = false (so the
+            // guard is bypassed). Both are restored immediately after scheduling the retries.
+            const originalDescribeBlock = state.currentDescribeBlock
+            const originalHasStarted = state.hasStarted
+            state.currentDescribeBlock = event.test.parent ?? originalDescribeBlock
+            state.hasStarted = false
+            this.retryTest({
+              jestEvent: {
+                testName: event.test.name,
+                fn: event.test.fn,
+                timeout: event.test.timeout,
+              },
+              retryCount,
+              retryType: 'Early flake detection',
+            })
+            state.currentDescribeBlock = originalDescribeBlock
+            state.hasStarted = originalHasStarted
+          } else {
+            efdSlowAbortedTests.add(testName)
+          }
+        }
+
         let isEfdRetry = false
         // We'll store the test statuses of the retries
-        const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
         if (this.isKnownTestsEnabled) {
           const isNewTest = retriedTestsToNumAttempts.has(testName)
           if (isNewTest) {
@@ -613,7 +663,8 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
             const testStatuses = newTestsTestStatuses.get(testName)
             // Check if this is the last EFD retry.
             // If it is, we'll set the failedAllTests flag to true if all the tests failed
-            if (testStatuses.length === earlyFlakeDetectionNumRetries + 1 &&
+            const efdRetryCount = efdDeterminedRetries.get(testName) ?? 0
+            if (efdRetryCount > 0 && testStatuses.length === efdRetryCount + 1 &&
               testStatuses.every(status => status === 'fail')) {
               failedAllTests = true
             }
@@ -671,6 +722,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           attemptToFixFailed,
           isAtrRetry,
           finalStatus,
+          earlyFlakeAbortReason: efdSlowAbortedTests.has(testName) ? 'slow' : undefined,
         })
 
         if (promises.isProbeReady) {
@@ -682,6 +734,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
           test.errors = errors
         }
         atrSuppressedErrors.clear()
+        efdDeterminedRetries.clear()
+        efdSlowAbortedTests.clear()
+        efdNewTestCandidates.clear()
       }
       if (event.name === 'test_skip' || event.name === 'test_todo') {
         const testName = getJestTestName(event.test, this.getShouldStripSeedFromTestName())
@@ -702,7 +757,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) {
     getEfdResult ({ testName, isNewTest, isModifiedTest, isEfdRetry, numberOfExecutedRetries }) {
       const isEfdEnabled = this.isEarlyFlakeDetectionEnabled
       const isEfdActive = isEfdEnabled && (isNewTest || isModifiedTest)
-      const isLastEfdRetry = isEfdRetry && numberOfExecutedRetries >= (earlyFlakeDetectionNumRetries + 1)
+      const retryCount = efdDeterminedRetries.get(testName) ?? 0
+      const isSlowAbort = efdSlowAbortedTests.has(testName)
+      const isLastEfdRetry = (isEfdRetry && numberOfExecutedRetries >= (retryCount + 1)) || isSlowAbort
       const isFinalEfdTestExecution = isEfdActive && isLastEfdRetry
 
       let finalStatus
@@ -939,6 +996,7 @@ function getCliWrapper (isNewJestVersion) {
           isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled
           isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled
           earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries
+          earlyFlakeDetectionSlowTestRetries = libraryConfig.earlyFlakeDetectionSlowTestRetries ?? {}
           earlyFlakeDetectionFaultyThreshold = libraryConfig.earlyFlakeDetectionFaultyThreshold
           isKnownTestsEnabled = libraryConfig.isKnownTestsEnabled
           isTestManagementTestsEnabled = libraryConfig.isTestManagementEnabled
@@ -1514,7 +1572,7 @@ addHook({
       _ddItrCorrelationId,
       _ddKnownTests,
       _ddIsEarlyFlakeDetectionEnabled,
-      _ddEarlyFlakeDetectionNumRetries,
+      _ddEarlyFlakeDetectionSlowTestRetries,
       _ddRepositoryRoot,
       _ddIsFlakyTestRetriesEnabled,
       _ddFlakyTestRetriesCount,

packages/datadog-plugin-jest/src/index.js

@@ -185,7 +185,7 @@ class JestPlugin extends CiPlugin {
         config._ddRequestErrorTags = this.getSessionRequestErrorTags()
         config._ddItrCorrelationId = this.itrCorrelationId
         config._ddIsEarlyFlakeDetectionEnabled = !!this.libraryConfig?.isEarlyFlakeDetectionEnabled
-        config._ddEarlyFlakeDetectionNumRetries = this.libraryConfig?.earlyFlakeDetectionNumRetries ?? 0
+        config._ddEarlyFlakeDetectionSlowTestRetries = this.libraryConfig?.earlyFlakeDetectionSlowTestRetries ?? {}
         config._ddRepositoryRoot = this.repositoryRoot
         config._ddIsFlakyTestRetriesEnabled = this.libraryConfig?.isFlakyTestRetriesEnabled ?? false
         config._ddIsTestManagementTestsEnabled = this.libraryConfig?.isTestManagementEnabled ?? false
@@ -395,6 +395,7 @@ class JestPlugin extends CiPlugin {
       attemptToFixFailed,
       isAtrRetry,
       finalStatus,
+      earlyFlakeAbortReason,
     }) => {
       span.setTag(TEST_STATUS, status)
       if (finalStatus) {
@@ -415,6 +416,9 @@ class JestPlugin extends CiPlugin {
         span.setTag(TEST_IS_RETRY, 'true')
         span.setTag(TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.atr)
       }
+      if (earlyFlakeAbortReason) {
+        span.setTag(TEST_EARLY_FLAKE_ABORT_REASON, earlyFlakeAbortReason)
+      }
 
       this.telemetry.ciVisEvent(
         TELEMETRY_EVENT_FINISHED,

packages/datadog-plugin-jest/src/util.js

@@ -3,7 +3,7 @@
 const { readFileSync } = require('fs')
 const { parse } = require('../../../vendor/dist/jest-docblock')
 
-const { getTestSuitePath } = require('../../dd-trace/src/plugins/util/test')
+const { getTestSuitePath, getEfdRetryCount } = require('../../dd-trace/src/plugins/util/test')
 const log = require('../../dd-trace/src/log')
 
 /**
@@ -172,4 +172,5 @@ module.exports = {
   getJestTestName,
   getJestSuitesToRun,
   isMarkedAsUnskippable,
+  getEfdRetryCount,
 }