feat(ci-vis): add cursor-based pagination for known tests endpoint (#7866)

calvinbayer · claude · web-flow · commit bcae91f37665 · 2026-03-31T22:06:31.000+02:00
* feat(ci-vis): add cursor-based pagination for known tests endpoint

Implement pagination support for the known tests API endpoint
(api/v2/ci/libraries/tests) as part of SDTEST-2988. The client now
sends page_info in requests and follows cursor-based pagination when
the response includes page_info.has_next=true, merging tests across
all pages into a single result.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;

* fix(ci-vis): resolve lint errors in known tests pagination

- Use ternary + spread operator instead of if/else + concat
- Use printf-style log formatting instead of template literals

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;

* Address review feedback: remove unnecessary export and middleware

- Stop exporting mergeKnownTests (only used internally)
- Remove express.json() middleware from known tests route (req.body unused)
- Remove redundant knownTestsPageIndex reset at route registration (already reset in stop())

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;

---------

Co-authored-by: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/integration-tests/ci-visibility-intake.js b/integration-tests/ci-visibility-intake.js
@@ -52,6 +52,7 @@ let correlationId = DEFAULT_CORRELATION_ID
 let knownTests = DEFAULT_KNOWN_TESTS
 let knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS
 let waitingTime = 0
+let knownTestsPageIndex = 0
 let testManagementResponse = DEFAULT_TEST_MANAGEMENT_TESTS
 let testManagementResponseStatusCode = DEFAULT_TEST_MANAGEMENT_TESTS_RESPONSE_STATUS
 
@@ -247,18 +248,34 @@ class FakeCiVisIntake extends FakeAgent {
     ], (req, res) => {
       // The endpoint returns compressed data if 'accept-encoding' is set to 'gzip'
       const isGzip = req.headers['accept-encoding'] === 'gzip'
-      const data = JSON.stringify({
-        data: {
-          attributes: {
-            tests: knownTests,
+
+      let responseData
+      if (Array.isArray(knownTests)) {
+        // Paginated mode: knownTests is an array of page responses
+        const page = knownTestsPageIndex < knownTests.length ? knownTests[knownTestsPageIndex] : null
+        knownTestsPageIndex++
+        if (page) {
+          responseData = JSON.stringify(page)
+        } else {
+          res.status(404).send('')
+          return
+        }
+      } else {
+        // Legacy single-response mode
+        responseData = JSON.stringify({
+          data: {
+            attributes: {
+              tests: knownTests,
+            },
           },
-        },
-      })
+        })
+      }
+
       res.setHeader('content-type', 'application/json')
       if (isGzip) {
         res.setHeader('content-encoding', 'gzip')
       }
-      res.status(knownTestsStatusCode).send(isGzip ? zlib.gzipSync(data) : data)
+      res.status(knownTestsStatusCode).send(isGzip ? zlib.gzipSync(responseData) : responseData)
       this.emit('message', {
         headers: req.headers,
         url: req.url,
@@ -320,12 +337,17 @@ class FakeCiVisIntake extends FakeAgent {
     })
   }
 
+  resetKnownTestsPageIndex () {
+    knownTestsPageIndex = 0
+  }
+
   stop () {
     settings = DEFAULT_SETTINGS
     settingsResponseStatusCode = 200
     suitesToSkip = DEFAULT_SUITES_TO_SKIP
     gitUploadStatus = DEFAULT_GIT_UPLOAD_STATUS
     knownTestsStatusCode = DEFAULT_KNOWN_TESTS_RESPONSE_STATUS
+    knownTestsPageIndex = 0
     infoResponse = DEFAULT_INFO_RESPONSE
     testManagementResponseStatusCode = DEFAULT_TEST_MANAGEMENT_TESTS_RESPONSE_STATUS
     testManagementResponse = DEFAULT_TEST_MANAGEMENT_TESTS
diff --git a/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js b/packages/dd-trace/src/ci-visibility/early-flake-detection/get-known-tests.js
@@ -17,6 +17,36 @@ const {
 
 const { getNumFromKnownTests } = require('../../plugins/util/test')
 
+const MAX_KNOWN_TESTS_PAGES = 10_000
+
+/**
+ * Deep-merges page tests into aggregate.
+ * Structure: { module: { suite: [testName, ...] } }
+ */
+function mergeKnownTests (aggregate, page) {
+  if (!page) return aggregate
+  if (!aggregate) return page
+
+  for (const [moduleName, suites] of Object.entries(page)) {
+    if (!suites) continue
+
+    if (!aggregate[moduleName]) {
+      aggregate[moduleName] = suites
+      continue
+    }
+
+    for (const [suiteName, tests] of Object.entries(suites)) {
+      if (!tests || tests.length === 0) continue
+
+      aggregate[moduleName][suiteName] = aggregate[moduleName][suiteName]
+        ? [...aggregate[moduleName][suiteName], ...tests]
+        : tests
+    }
+  }
+
+  return aggregate
+}
+
 function getKnownTests ({
   url,
   isEvpProxy,
@@ -59,53 +89,94 @@ function getKnownTests ({
     options.headers['dd-api-key'] = apiKey
   }
 
-  const data = JSON.stringify({
-    data: {
-      id: id().toString(10),
-      type: 'ci_app_libraries_tests_request',
-      attributes: {
-        configurations: {
-          'os.platform': osPlatform,
-          'os.version': osVersion,
-          'os.architecture': osArchitecture,
-          'runtime.name': runtimeName,
-          'runtime.version': runtimeVersion,
-          custom,
-        },
-        service,
-        env,
-        repository_url: repositoryUrl,
-        sha,
-      },
-    },
-  })
+  const configurations = {
+    'os.platform': osPlatform,
+    'os.version': osVersion,
+    'os.architecture': osArchitecture,
+    'runtime.name': runtimeName,
+    'runtime.version': runtimeVersion,
+    custom,
+  }
 
   incrementCountMetric(TELEMETRY_KNOWN_TESTS)
 
   const startTime = Date.now()
+  let aggregateTests = null
+  let totalResponseBytes = 0
+  let pageNumber = 0
+
+  function fetchPage (pageState) {
+    pageNumber++
+
+    if (pageNumber > MAX_KNOWN_TESTS_PAGES) {
+      log.error('Known tests pagination exceeded maximum of %d pages. Aborting.', MAX_KNOWN_TESTS_PAGES)
+      distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+      return done(new Error(`Known tests pagination exceeded maximum of ${MAX_KNOWN_TESTS_PAGES} pages`))
+    }
+
+    const pageInfo = pageState ? { page_state: pageState } : {}
+
+    const data = JSON.stringify({
+      data: {
+        id: id().toString(10),
+        type: 'ci_app_libraries_tests_request',
+        attributes: {
+          configurations,
+          service,
+          env,
+          repository_url: repositoryUrl,
+          sha,
+          page_info: pageInfo,
+        },
+      },
+    })
+
+    request(data, options, (err, res, statusCode) => {
+      if (err) {
+        distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+        incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode })
+        return done(err)
+      }
 
-  request(data, options, (err, res, statusCode) => {
-    distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
-    if (err) {
-      incrementCountMetric(TELEMETRY_KNOWN_TESTS_ERRORS, { statusCode })
-      done(err)
-    } else {
       try {
-        const { data: { attributes: { tests: knownTests } } } = JSON.parse(res)
+        totalResponseBytes += res.length
+
+        const { data: { attributes } } = JSON.parse(res)
+        const { tests: pageTests, page_info: responsePageInfo } = attributes
 
-        const numTests = getNumFromKnownTests(knownTests)
+        aggregateTests = mergeKnownTests(aggregateTests, pageTests)
+
+        // Check if there are more pages
+        if (responsePageInfo && responsePageInfo.has_next) {
+          if (!responsePageInfo.cursor) {
+            log.error(
+              'Known tests response has has_next=true but no cursor on page %d. Aborting pagination.', pageNumber
+            )
+            distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+            return done(new Error('Known tests pagination: has_next=true but no cursor'))
+          }
+          return fetchPage(responsePageInfo.cursor)
+        }
+
+        // Done — no more pages
+        distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
+
+        const numTests = getNumFromKnownTests(aggregateTests)
 
         distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_TESTS, {}, numTests)
-        distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, res.length)
+        distributionMetric(TELEMETRY_KNOWN_TESTS_RESPONSE_BYTES, {}, totalResponseBytes)
 
         log.debug('Number of received known tests:', numTests)
 
-        done(null, knownTests)
+        done(null, aggregateTests)
       } catch (err) {
+        distributionMetric(TELEMETRY_KNOWN_TESTS_MS, {}, Date.now() - startTime)
         done(err)
       }
-    }
-  })
+    })
+  }
+
+  fetchPage(null)
 }
 
 module.exports = { getKnownTests }
diff --git a/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/ci-visibility-exporter.spec.js
@@ -849,6 +849,160 @@ describe('CI Visibility Exporter', () => {
           done()
         })
       })
+
+      it('should handle paginated responses and merge tests across pages', (done) => {
+        const scope = nock(url)
+          .post('/api/v2/ci/libraries/tests')
+          .reply(200, JSON.stringify({
+            data: {
+              attributes: {
+                tests: {
+                  jest: {
+                    suite1: ['test1'],
+                  },
+                },
+                page_info: {
+                  cursor: 'cursor_page2',
+                  has_next: true,
+                  size: 1,
+                },
+              },
+            },
+          }))
+          .post('/api/v2/ci/libraries/tests')
+          .reply(200, JSON.stringify({
+            data: {
+              attributes: {
+                tests: {
+                  jest: {
+                    suite1: ['test2'],
+                    suite2: ['test3'],
+                  },
+                },
+                page_info: {
+                  cursor: '',
+                  has_next: false,
+                  size: 2,
+                },
+              },
+            },
+          }))
+
+        const ciVisibilityExporter = new CiVisibilityExporter({ port })
+
+        ciVisibilityExporter._resolveCanUseCiVisProtocol(true)
+        ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true }
+        ciVisibilityExporter.getKnownTests({}, (err, knownTests) => {
+          assert.strictEqual(err, null)
+          assert.deepStrictEqual(knownTests, {
+            jest: {
+              suite1: ['test1', 'test2'],
+              suite2: ['test3'],
+            },
+          })
+          assert.strictEqual(scope.isDone(), true)
+          done()
+        })
+      })
+
+      it('should handle backward-compatible response without page_info', (done) => {
+        const scope = nock(url)
+          .post('/api/v2/ci/libraries/tests')
+          .reply(200, JSON.stringify({
+            data: {
+              attributes: {
+                tests: {
+                  jest: {
+                    suite1: ['test1'],
+                  },
+                },
+              },
+            },
+          }))
+
+        const ciVisibilityExporter = new CiVisibilityExporter({ port })
+
+        ciVisibilityExporter._resolveCanUseCiVisProtocol(true)
+        ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true }
+        ciVisibilityExporter.getKnownTests({}, (err, knownTests) => {
+          assert.strictEqual(err, null)
+          assert.deepStrictEqual(knownTests, {
+            jest: {
+              suite1: ['test1'],
+            },
+          })
+          assert.strictEqual(scope.isDone(), true)
+          done()
+        })
+      })
+
+      it('should send page_info in request payload', (done) => {
+        let firstRequestBody
+        let secondRequestBody
+        const scope = nock(url)
+          .post('/api/v2/ci/libraries/tests', (body) => {
+            firstRequestBody = body
+            return true
+          })
+          .reply(200, JSON.stringify({
+            data: {
+              attributes: {
+                tests: { jest: { suite1: ['test1'] } },
+                page_info: { cursor: 'abc123', has_next: true, size: 1 },
+              },
+            },
+          }))
+          .post('/api/v2/ci/libraries/tests', (body) => {
+            secondRequestBody = body
+            return true
+          })
+          .reply(200, JSON.stringify({
+            data: {
+              attributes: {
+                tests: { jest: { suite2: ['test2'] } },
+                page_info: { cursor: '', has_next: false, size: 1 },
+              },
+            },
+          }))
+
+        const ciVisibilityExporter = new CiVisibilityExporter({ port })
+
+        ciVisibilityExporter._resolveCanUseCiVisProtocol(true)
+        ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true }
+        ciVisibilityExporter.getKnownTests({}, (err) => {
+          assert.strictEqual(err, null)
+          assert.strictEqual(scope.isDone(), true)
+          // First request should have empty page_info
+          assert.deepStrictEqual(firstRequestBody.data.attributes.page_info, {})
+          // Second request should have page_state from cursor
+          assert.deepStrictEqual(secondRequestBody.data.attributes.page_info, { page_state: 'abc123' })
+          done()
+        })
+      })
+
+      it('should return error when has_next is true but cursor is missing', (done) => {
+        const scope = nock(url)
+          .post('/api/v2/ci/libraries/tests')
+          .reply(200, JSON.stringify({
+            data: {
+              attributes: {
+                tests: { jest: { suite1: ['test1'] } },
+                page_info: { has_next: true, size: 1 },
+              },
+            },
+          }))
+
+        const ciVisibilityExporter = new CiVisibilityExporter({ port })
+
+        ciVisibilityExporter._resolveCanUseCiVisProtocol(true)
+        ciVisibilityExporter._libraryConfig = { isKnownTestsEnabled: true }
+        ciVisibilityExporter.getKnownTests({}, (err) => {
+          assert.notStrictEqual(err, null)
+          assert.match(err.message, /has_next=true but no cursor/)
+          assert.strictEqual(scope.isDone(), true)
+          done()
+        })
+      })
     })
   })
 
