Skip to content

Commit 8f81db6

Browse files
juan-fernandezjuan-fernandez
authored
[test optimization] Prevent ATF tests from being incorrectly tagged as new in cucumber (#7961)
1 parent abc727b commit 8f81db6

6 files changed

Lines changed: 266 additions & 1 deletion

File tree

‎integration-tests/cucumber/cucumber.spec.js‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2596,6 +2596,54 @@ describe(`cucumber@${version} commonJS`, () => {
25962596
})
25972597
})
25982598

2599+
it('does not tag known attempt to fix tests as new', async () => {
2600+
receiver.setKnownTests({
2601+
cucumber: {
2602+
'ci-visibility/features-test-management/attempt-to-fix.feature': [
2603+
'Say attempt to fix',
2604+
],
2605+
},
2606+
})
2607+
receiver.setSettings({
2608+
test_management: { enabled: true, attempt_to_fix_retries: 2 },
2609+
early_flake_detection: {
2610+
enabled: true,
2611+
slow_test_retries: { '5s': 2 },
2612+
faulty_session_threshold: 100,
2613+
},
2614+
known_tests_enabled: true,
2615+
})
2616+
2617+
const eventsPromise = receiver
2618+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
2619+
const events = payloads.flatMap(({ payload }) => payload.events)
2620+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2621+
const atfTests = tests.filter(
2622+
t => t.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true'
2623+
)
2624+
assert.ok(atfTests.length > 0)
2625+
for (const test of atfTests) {
2626+
assert.ok(
2627+
!(TEST_IS_NEW in test.meta),
2628+
'ATF test that is in known tests should not be tagged as new'
2629+
)
2630+
}
2631+
})
2632+
2633+
childProcess = exec(
2634+
'./node_modules/.bin/cucumber-js ci-visibility/features-test-management/attempt-to-fix.feature',
2635+
{
2636+
cwd,
2637+
env: getCiVisAgentlessConfig(receiver.port),
2638+
}
2639+
)
2640+
2641+
await Promise.all([
2642+
once(childProcess, 'exit'),
2643+
eventsPromise,
2644+
])
2645+
})
2646+
25992647
it('does not fail retry if a test is quarantined', (done) => {
26002648
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
26012649
receiver.setTestManagementTests({

‎integration-tests/cypress/cypress.spec.js‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3274,6 +3274,62 @@ moduleTypes.forEach(({
32743274
await runAttemptToFixTest({ extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } })
32753275
})
32763276

3277+
it('does not tag known attempt to fix tests as new', async () => {
3278+
receiver.setKnownTests({
3279+
cypress: {
3280+
'cypress/e2e/attempt-to-fix.js': [
3281+
'attempt to fix is attempt to fix',
3282+
],
3283+
},
3284+
})
3285+
receiver.setSettings({
3286+
test_management: { enabled: true, attempt_to_fix_retries: 2 },
3287+
early_flake_detection: {
3288+
enabled: true,
3289+
slow_test_retries: { '5s': 2 },
3290+
faulty_session_threshold: 100,
3291+
},
3292+
known_tests_enabled: true,
3293+
})
3294+
3295+
const eventsPromise = receiver
3296+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
3297+
const events = payloads.flatMap(({ payload }) => payload.events)
3298+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
3299+
const atfTests = tests.filter(
3300+
t => t.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true'
3301+
)
3302+
assert.ok(atfTests.length > 0)
3303+
for (const test of atfTests) {
3304+
assert.ok(
3305+
!(TEST_IS_NEW in test.meta),
3306+
'ATF test that is in known tests should not be tagged as new'
3307+
)
3308+
}
3309+
}, 25000)
3310+
3311+
const envVars = getCiVisEvpProxyConfig(receiver.port)
3312+
const specToRun = 'cypress/e2e/attempt-to-fix.js'
3313+
3314+
childProcess = exec(
3315+
version === 'latest' ? testCommand : `${testCommand} --spec ${specToRun}`,
3316+
{
3317+
cwd,
3318+
env: {
3319+
...envVars,
3320+
CYPRESS_BASE_URL: `http://localhost:${webAppPort}`,
3321+
SPEC_PATTERN: specToRun,
3322+
CYPRESS_SHOULD_ALWAYS_PASS: '1',
3323+
},
3324+
}
3325+
)
3326+
3327+
await Promise.all([
3328+
once(childProcess, 'exit'),
3329+
eventsPromise,
3330+
])
3331+
})
3332+
32773333
/**
32783334
* TODO:
32793335
* The spec says that quarantined tests that are not attempted to fix should be run and their result ignored.

‎integration-tests/mocha/mocha.spec.js‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4153,6 +4153,59 @@ describe(`mocha@${MOCHA_VERSION}`, function () {
41534153
runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } })
41544154
})
41554155

4156+
onlyLatestIt('does not tag known attempt to fix tests as new', async () => {
4157+
receiver.setKnownTests({
4158+
mocha: {
4159+
'ci-visibility/test-management/test-attempt-to-fix-1.js': [
4160+
'attempt to fix tests can attempt to fix a test',
4161+
],
4162+
},
4163+
})
4164+
receiver.setSettings({
4165+
test_management: { enabled: true, attempt_to_fix_retries: 2 },
4166+
early_flake_detection: {
4167+
enabled: true,
4168+
slow_test_retries: { '5s': 2 },
4169+
faulty_session_threshold: 100,
4170+
},
4171+
known_tests_enabled: true,
4172+
})
4173+
4174+
const eventsPromise = receiver
4175+
.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => {
4176+
const events = payloads.flatMap(({ payload }) => payload.events)
4177+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
4178+
const atfTests = tests.filter(
4179+
t => t.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true'
4180+
)
4181+
assert.ok(atfTests.length > 0)
4182+
for (const test of atfTests) {
4183+
assert.ok(
4184+
!(TEST_IS_NEW in test.meta),
4185+
'ATF test that is in known tests should not be tagged as new'
4186+
)
4187+
}
4188+
})
4189+
4190+
childProcess = exec(
4191+
runTestsCommand,
4192+
{
4193+
cwd,
4194+
env: {
4195+
...getCiVisAgentlessConfig(receiver.port),
4196+
TESTS_TO_RUN: JSON.stringify([
4197+
'./test-management/test-attempt-to-fix-1.js',
4198+
]),
4199+
},
4200+
}
4201+
)
4202+
4203+
await Promise.all([
4204+
once(childProcess, 'exit'),
4205+
eventsPromise,
4206+
])
4207+
})
4208+
41564209
onlyLatestIt('does not fail retry if a test is quarantined', (done) => {
41574210
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
41584211
receiver.setTestManagementTests({

‎integration-tests/playwright/playwright.spec.js‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1660,6 +1660,59 @@ versions.forEach((version) => {
16601660
await runAttemptToFixTest({ extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } })
16611661
})
16621662

1663+
it('does not tag known attempt to fix tests as new', async () => {
1664+
receiver.setKnownTests({
1665+
playwright: {
1666+
'attempt-to-fix-test.js': [
1667+
'attempt to fix should attempt to fix failed test',
1668+
'attempt to fix should attempt to fix passed test',
1669+
],
1670+
},
1671+
})
1672+
receiver.setSettings({
1673+
test_management: { enabled: true, attempt_to_fix_retries: 2 },
1674+
early_flake_detection: {
1675+
enabled: true,
1676+
slow_test_retries: { '5s': 2 },
1677+
faulty_session_threshold: 100,
1678+
},
1679+
known_tests_enabled: true,
1680+
})
1681+
1682+
const eventsPromise = receiver
1683+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
1684+
const events = payloads.flatMap(({ payload }) => payload.events)
1685+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1686+
const atfTests = tests.filter(
1687+
t => t.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true'
1688+
)
1689+
assert.ok(atfTests.length > 0)
1690+
for (const test of atfTests) {
1691+
assert.ok(
1692+
!(TEST_IS_NEW in test.meta),
1693+
'ATF test that is in known tests should not be tagged as new'
1694+
)
1695+
}
1696+
})
1697+
1698+
childProcess = exec(
1699+
'./node_modules/.bin/playwright test -c playwright.config.js attempt-to-fix-test.js',
1700+
{
1701+
cwd,
1702+
env: {
1703+
...getCiVisAgentlessConfig(receiver.port),
1704+
PW_BASE_URL: `http://localhost:${webAppPort}`,
1705+
TEST_DIR: './ci-visibility/playwright-tests-test-management',
1706+
},
1707+
}
1708+
)
1709+
1710+
await Promise.all([
1711+
once(childProcess, 'exit'),
1712+
eventsPromise,
1713+
])
1714+
})
1715+
16631716
it('does not fail retry if a test is quarantined', async () => {
16641717
receiver.setSettings({
16651718
test_management: { enabled: true, attempt_to_fix_retries: ATTEMPT_TO_FIX_NUM_RETRIES },

‎integration-tests/vitest/vitest.spec.js‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1931,6 +1931,58 @@ versions.forEach((version) => {
19311931
runAttemptToFixTest(done, { extraEnvVars: { DD_TEST_MANAGEMENT_ENABLED: '0' } })
19321932
})
19331933

1934+
it('does not tag known attempt to fix tests as new', async () => {
1935+
receiver.setKnownTests({
1936+
vitest: {
1937+
'ci-visibility/vitest-tests/test-attempt-to-fix.mjs': [
1938+
'attempt to fix tests can attempt to fix a test',
1939+
],
1940+
},
1941+
})
1942+
receiver.setSettings({
1943+
test_management: { enabled: true, attempt_to_fix_retries: 2 },
1944+
early_flake_detection: {
1945+
enabled: true,
1946+
slow_test_retries: { '5s': 2 },
1947+
faulty_session_threshold: 100,
1948+
},
1949+
known_tests_enabled: true,
1950+
})
1951+
1952+
const eventsPromise = receiver
1953+
.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', (payloads) => {
1954+
const events = payloads.flatMap(({ payload }) => payload.events)
1955+
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1956+
const atfTests = tests.filter(
1957+
t => t.meta[TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX] === 'true'
1958+
)
1959+
assert.ok(atfTests.length > 0)
1960+
for (const test of atfTests) {
1961+
assert.ok(
1962+
!(TEST_IS_NEW in test.meta),
1963+
'ATF test that is in known tests should not be tagged as new'
1964+
)
1965+
}
1966+
})
1967+
1968+
childProcess = exec(
1969+
'./node_modules/.bin/vitest run',
1970+
{
1971+
cwd,
1972+
env: {
1973+
...getCiVisAgentlessConfig(receiver.port),
1974+
TEST_DIR: 'ci-visibility/vitest-tests/test-attempt-to-fix*',
1975+
NODE_OPTIONS: '--import dd-trace/register.js -r dd-trace/ci/init --no-warnings',
1976+
},
1977+
}
1978+
)
1979+
1980+
await Promise.all([
1981+
once(childProcess, 'exit'),
1982+
eventsPromise,
1983+
])
1984+
})
1985+
19341986
it('does not fail retry if a test is quarantined', (done) => {
19351987
receiver.setSettings({ test_management: { enabled: true, attempt_to_fix_retries: 3 } })
19361988
receiver.setTestManagementTests({

‎packages/datadog-instrumentations/src/cucumber.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ const numRetriesByPickleId = new Map()
6161
const numAttemptToCtx = new Map()
6262
const newTestsByTestFullname = new Map()
6363
const modifiedTestsByPickleId = new Map()
64+
// Pickle IDs for tests that are genuinely new (not in known tests list).
65+
const newTestPickleIds = new Set()
6466

6567
let eventDataCollector = null
6668
let pickleByFile = {}
@@ -359,7 +361,7 @@ function wrapRun (pl, isLatestVersion, version) {
359361
}
360362

361363
if (isKnownTestsEnabled && status !== 'skip') {
362-
isNew = numRetries !== undefined
364+
isNew = newTestPickleIds.has(this.pickle.id)
363365
}
364366

365367
if (isNew || isModified) {
@@ -714,6 +716,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = fa
714716
if (isKnownTestsEnabled && !isAttemptToFix) {
715717
isNew = isNewTest(testSuitePath, pickle.name)
716718
if (isNew) {
719+
newTestPickleIds.add(pickle.id)
717720
numRetriesByPickleId.set(pickle.id, 0)
718721
}
719722
}

0 commit comments

Comments
 (0)