Skip to content

Commit 38f62e9

Browse files
MariovidoMariovido
authored
[test-optimization] [SDTEST-1830] Add @test.retry_reason to the retries of different frameworks (#5550)
1 parent 5ab3df2 commit 38f62e9

19 files changed

Lines changed: 248 additions & 88 deletions

File tree

‎integration-tests/cucumber/cucumber.spec.js‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ const {
5555
DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX,
5656
TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
5757
TEST_HAS_FAILED_ALL_RETRIES,
58-
TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED
58+
TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED,
59+
TEST_RETRY_REASON_TYPES
5960
} = require('../../packages/dd-trace/src/plugins/util/test')
6061
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
6162

@@ -897,7 +898,7 @@ versions.forEach(version => {
897898
)
898899
assert.equal(retriedTests.length, NUM_RETRIES_EFD)
899900
retriedTests.forEach(test => {
900-
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd')
901+
assert.propertyVal(test.meta, TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.efd)
901902
})
902903
// Test name does not change
903904
newTests.forEach(test => {
@@ -1486,6 +1487,9 @@ versions.forEach(version => {
14861487
// All but the first one are retries
14871488
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
14881489
assert.equal(retriedTests.length, 2)
1490+
assert.equal(retriedTests.filter(
1491+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1492+
).length, 2)
14891493
})
14901494

14911495
childProcess = exec(
@@ -1523,7 +1527,9 @@ versions.forEach(version => {
15231527

15241528
assert.equal(tests.length, 1)
15251529

1526-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1530+
const retriedTests = tests.filter(
1531+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1532+
)
15271533
assert.equal(retriedTests.length, 0)
15281534
})
15291535

@@ -1572,7 +1578,9 @@ versions.forEach(version => {
15721578
assert.equal(passedTests.length, 0)
15731579

15741580
// All but the first one are retries
1575-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1581+
const retriedTests = tests.filter(
1582+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1583+
)
15761584
assert.equal(retriedTests.length, 1)
15771585
})
15781586

@@ -1608,7 +1616,9 @@ versions.forEach(version => {
16081616
const events = payloads.flatMap(({ payload }) => payload.events)
16091617

16101618
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1611-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1619+
const retriedTests = tests.filter(
1620+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1621+
)
16121622

16131623
assert.equal(retriedTests.length, 1)
16141624
const [retriedTest] = retriedTests
@@ -1657,7 +1667,9 @@ versions.forEach(version => {
16571667
const events = payloads.flatMap(({ payload }) => payload.events)
16581668

16591669
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1660-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1670+
const retriedTests = tests.filter(
1671+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1672+
)
16611673

16621674
assert.equal(retriedTests.length, 1)
16631675
const [retriedTest] = retriedTests
@@ -1706,7 +1718,9 @@ versions.forEach(version => {
17061718

17071719
const tests = events.filter(event => event.type === 'test').map(event => event.content)
17081720

1709-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1721+
const retriedTests = tests.filter(
1722+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1723+
)
17101724

17111725
assert.equal(retriedTests.length, 1)
17121726
const [retriedTest] = retriedTests
@@ -1784,7 +1798,9 @@ versions.forEach(version => {
17841798
const events = payloads.flatMap(({ payload }) => payload.events)
17851799

17861800
const tests = events.filter(event => event.type === 'test').map(event => event.content)
1787-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
1801+
const retriedTests = tests.filter(
1802+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1803+
)
17881804

17891805
assert.equal(retriedTests.length, 1)
17901806
const [retriedTest] = retriedTests
@@ -2109,7 +2125,7 @@ versions.forEach(version => {
21092125
assert.propertyVal(test.meta, TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX, 'true')
21102126
if (!isFirstAttempt) {
21112127
assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
2112-
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
2128+
assert.propertyVal(test.meta, TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.atf)
21132129
}
21142130
if (isLastAttempt) {
21152131
if (shouldFailSometimes) {

‎integration-tests/cypress/cypress.spec.js‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ const {
5151
DD_CAPABILITIES_AUTO_TEST_RETRIES,
5252
DD_CAPABILITIES_TEST_MANAGEMENT_QUARANTINE,
5353
DD_CAPABILITIES_TEST_MANAGEMENT_DISABLE,
54-
DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX
54+
DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX,
55+
TEST_RETRY_REASON_TYPES
5556
} = require('../../packages/dd-trace/src/plugins/util/test')
5657
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
5758
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -1107,7 +1108,7 @@ moduleTypes.forEach(({
11071108
assert.equal(retriedTests.length, NUM_RETRIES_EFD)
11081109

11091110
retriedTests.forEach((retriedTest) => {
1110-
assert.equal(retriedTest.meta[TEST_RETRY_REASON], 'efd')
1111+
assert.equal(retriedTest.meta[TEST_RETRY_REASON], TEST_RETRY_REASON_TYPES.efd)
11111112
})
11121113

11131114
newTests.forEach(newTest => {
@@ -1432,6 +1433,9 @@ moduleTypes.forEach(({
14321433
assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 2)
14331434
assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 1)
14341435
assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2)
1436+
assert.equal(eventuallyPassingTest.filter(test =>
1437+
test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1438+
).length, 2)
14351439

14361440
const neverPassingTest = tests.filter(
14371441
test => test.resource === 'cypress/e2e/flaky-test-retries.js.flaky test retry never passes'
@@ -1440,6 +1444,9 @@ moduleTypes.forEach(({
14401444
assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 6)
14411445
assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 0)
14421446
assert.equal(neverPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 5)
1447+
assert.equal(neverPassingTest.filter(
1448+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
1449+
).length, 5)
14431450
})
14441451

14451452
const {
@@ -1495,7 +1502,7 @@ moduleTypes.forEach(({
14951502
'cypress/e2e/flaky-test-retries.js.flaky test retry never passes',
14961503
'cypress/e2e/flaky-test-retries.js.flaky test retry always passes'
14971504
])
1498-
assert.equal(tests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 0)
1505+
assert.equal(tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr).length, 0)
14991506
})
15001507

15011508
const {
@@ -1553,7 +1560,7 @@ moduleTypes.forEach(({
15531560
'cypress/e2e/flaky-test-retries.js.flaky test retry always passes'
15541561
])
15551562

1556-
assert.equal(tests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2)
1563+
assert.equal(tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr).length, 2)
15571564
})
15581565

15591566
const {
@@ -1863,7 +1870,7 @@ moduleTypes.forEach(({
18631870
assert.notProperty(test.meta, TEST_RETRY_REASON)
18641871
} else {
18651872
assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
1866-
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
1873+
assert.propertyVal(test.meta, TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.atf)
18671874
}
18681875
if (isLastAttempt) {
18691876
if (shouldFailSometimes) {

‎integration-tests/jest/jest.spec.js‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ const {
5252
DD_CAPABILITIES_TEST_MANAGEMENT_ATTEMPT_TO_FIX,
5353
TEST_MANAGEMENT_IS_ATTEMPT_TO_FIX,
5454
TEST_HAS_FAILED_ALL_RETRIES,
55-
TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED
55+
TEST_MANAGEMENT_ATTEMPT_TO_FIX_PASSED,
56+
TEST_RETRY_REASON_TYPES
5657
} = require('../../packages/dd-trace/src/plugins/util/test')
5758
const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/env')
5859
const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants')
@@ -531,7 +532,7 @@ describe('jest CommonJS', () => {
531532
const events = payloads.flatMap(({ payload }) => payload.events)
532533

533534
const tests = events.filter(event => event.type === 'test').map(event => event.content)
534-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
535+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
535536

536537
assert.equal(retriedTests.length, 2)
537538
const retriedTest = retriedTests.find(test => test.meta[TEST_SUITE].includes('test-hit-breakpoint.js'))
@@ -1666,7 +1667,7 @@ describe('jest CommonJS', () => {
16661667
)
16671668
assert.equal(retriedTests.length, NUM_RETRIES_EFD)
16681669
retriedTests.forEach(test => {
1669-
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'efd')
1670+
assert.propertyVal(test.meta, TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.efd)
16701671
})
16711672
// Test name does not change
16721673
newTests.forEach(test => {
@@ -2406,6 +2407,9 @@ describe('jest CommonJS', () => {
24062407
assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 2)
24072408
assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 1)
24082409
assert.equal(eventuallyPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2)
2410+
assert.equal(eventuallyPassingTest.filter(test =>
2411+
test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
2412+
).length, 2)
24092413

24102414
const neverPassingTest = tests.filter(
24112415
test => test.resource ===
@@ -2415,6 +2419,9 @@ describe('jest CommonJS', () => {
24152419
assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'fail').length, 6)
24162420
assert.equal(neverPassingTest.filter(test => test.meta[TEST_STATUS] === 'pass').length, 0)
24172421
assert.equal(neverPassingTest.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 5)
2422+
assert.equal(neverPassingTest.filter(
2423+
test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr
2424+
).length, 5)
24182425

24192426
const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content)
24202427

@@ -2473,7 +2480,7 @@ describe('jest CommonJS', () => {
24732480
'ci-visibility/jest-flaky/flaky-fails.js.test-flaky-test-retries can retry failed tests'
24742481
])
24752482

2476-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2483+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
24772484

24782485
assert.equal(retriedTests.length, 0)
24792486
})
@@ -2514,6 +2521,8 @@ describe('jest CommonJS', () => {
25142521
const events = payloads.flatMap(({ payload }) => payload.events)
25152522

25162523
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2524+
assert.equal(tests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 2)
2525+
assert.equal(tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr).length, 2)
25172526

25182527
assert.equal(tests.length, 5)
25192528
// only one retry
@@ -2556,7 +2565,7 @@ describe('jest CommonJS', () => {
25562565
const events = payloads.flatMap(({ payload }) => payload.events)
25572566

25582567
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2559-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2568+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
25602569

25612570
assert.equal(retriedTests.length, 1)
25622571
const [retriedTest] = retriedTests
@@ -2605,7 +2614,7 @@ describe('jest CommonJS', () => {
26052614
const events = payloads.flatMap(({ payload }) => payload.events)
26062615

26072616
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2608-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2617+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
26092618

26102619
assert.equal(retriedTests.length, 1)
26112620
const [retriedTest] = retriedTests
@@ -2654,7 +2663,7 @@ describe('jest CommonJS', () => {
26542663
const events = payloads.flatMap(({ payload }) => payload.events)
26552664

26562665
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2657-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2666+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
26582667

26592668
assert.equal(retriedTests.length, 1)
26602669
const [retriedTest] = retriedTests
@@ -2738,7 +2747,7 @@ describe('jest CommonJS', () => {
27382747
const events = payloads.flatMap(({ payload }) => payload.events)
27392748

27402749
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2741-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2750+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
27422751

27432752
assert.equal(retriedTests.length, 1)
27442753
const [retriedTest] = retriedTests
@@ -2786,7 +2795,7 @@ describe('jest CommonJS', () => {
27862795
const events = payloads.flatMap(({ payload }) => payload.events)
27872796

27882797
const tests = events.filter(event => event.type === 'test').map(event => event.content)
2789-
const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true')
2798+
const retriedTests = tests.filter(test => test.meta[TEST_RETRY_REASON] === TEST_RETRY_REASON_TYPES.atr)
27902799

27912800
assert.equal(retriedTests.length, 1)
27922801
const [retriedTest] = retriedTests
@@ -3034,7 +3043,7 @@ describe('jest CommonJS', () => {
30343043
assert.notProperty(test.meta, TEST_RETRY_REASON)
30353044
} else {
30363045
assert.propertyVal(test.meta, TEST_IS_RETRY, 'true')
3037-
assert.propertyVal(test.meta, TEST_RETRY_REASON, 'attempt_to_fix')
3046+
assert.propertyVal(test.meta, TEST_RETRY_REASON, TEST_RETRY_REASON_TYPES.atf)
30383047
}
30393048

30403049
if (isLastAttempt) {

0 commit comments

Comments
 (0)