Changes for AI Integration Toolkit Project (#7178)

Changes to make it simpler to write integrations. Main changes are a test abstraction layer for setting up integration tests, which handles tracer / agent / module initialization. The other change being the ability to use assertFirstTraceSpan with the object API, while asserting basic types for tags that change between runs (mainly for remote ip tags and error stack traces).

Author: wconti27

.gitignore

@@ -135,3 +135,10 @@ packages/datadog-plugin-azure-functions/test/integration-test/fixtures/node_modu
 __azurite_db_queue__.json
 __azurite_db_queue_extent__.json
 __queuestorage__/AzuriteConfig
+
+
+# ignore apm-instrumentation-toolkit analysis results dirs
+.analysis/
+
+# ignore claude settings
+.claude/

eslint.config.mjs

@@ -48,6 +48,7 @@ export default [
       '**/versions', // This is effectively a node_modules tree.
       '**/acmeair-nodejs', // We don't own this.
       '**/vendor', // Generally, we didn't author this code.
+      '**/.analysis', // Ignore apm-instrumentation-toolkit analysis results
       'integration-tests/code-origin/typescript.js', // Generated
       'integration-tests/debugger/target-app/source-map-support/bundle.js', // Generated
       'integration-tests/debugger/target-app/source-map-support/hello/world.js', // Generated

integration-tests/helpers/index.js

@@ -24,6 +24,11 @@ const { DEBUG } = process.env
 // This is set by the setShouldKill function
 let shouldKill
 
+// Symbol constants for dynamic value matching in assertObjectContains
+const ANY_STRING = Symbol('test.ANY_STRING')
+const ANY_NUMBER = Symbol('test.ANY_NUMBER')
+const ANY_VALUE = Symbol('test.ANY_VALUE')
+
 /**
  * @param {string} filename
  * @param {string} cwd
@@ -635,9 +640,8 @@ function setShouldKill (value) {
   })
 }
 
-// @ts-expect-error assert.partialDeepStrictEqual does not exist on older Node.js versions
-// eslint-disable-next-line n/no-unsupported-features/node-builtins
-const assertObjectContains = assert.partialDeepStrictEqual || function assertObjectContains (actual, expected, msg) {
+// Implementation with optional matcher support (ANY_STRING, ANY_NUMBER, ANY_VALUE)
+function assertObjectContainsImpl (actual, expected, msg, useMatchers) {
   if (expected === null || typeof expected !== 'object') {
     assert.strictEqual(actual, expected, msg)
     return
@@ -652,7 +656,7 @@ const assertObjectContains = assert.partialDeepStrictEqual || function assertObj
         const actualItem = actual[i]
         try {
           if (expectedItem !== null && typeof expectedItem === 'object') {
-            assertObjectContains(actualItem, expectedItem, msg)
+            assertObjectContainsImpl(actualItem, expectedItem, msg, useMatchers)
           } else {
             assert.strictEqual(actualItem, expectedItem, msg)
           }
@@ -670,15 +674,40 @@ const assertObjectContains = assert.partialDeepStrictEqual || function assertObj
 
   for (const [key, val] of Object.entries(expected)) {
     assert.ok(Object.hasOwn(actual, key), msg)
-    if (val !== null && typeof val === 'object') {
-      assertObjectContains(actual[key], val, msg)
+    if (useMatchers && val === ANY_STRING) {
+      assert.strictEqual(typeof actual[key], 'string', `Expected ${key} to be a string but got ${typeof actual[key]}`)
+    } else if (useMatchers && val === ANY_NUMBER) {
+      assert.strictEqual(typeof actual[key], 'number', `Expected ${key} to be a number but got ${typeof actual[key]}`)
+    } else if (useMatchers && val === ANY_VALUE) {
+      assert.ok(actual[key] !== undefined, `Expected ${key} to be present but it was undefined`)
+    } else if (val !== null && typeof val === 'object') {
+      assertObjectContainsImpl(actual[key], val, msg, useMatchers)
     } else {
       assert.ok(actual, msg)
       assert.strictEqual(actual[key], expected[key], msg)
     }
   }
 }
 
+// Main assertObjectContains: tries partialDeepStrictEqual or strict first, falls back to matchers
+const assertObjectContains = function assertObjectContains (actual, expected, msg) {
+  // @ts-expect-error assert.partialDeepStrictEqual does not exist on older Node.js versions
+  // eslint-disable-next-line n/no-unsupported-features/node-builtins
+  const assertionFn = assert.partialDeepStrictEqual ||
+    ((actual, expected, msg) => assertObjectContainsImpl(actual, expected, msg, false))
+
+  try {
+    assertionFn(actual, expected, msg)
+  } catch (originalError) {
+    // First attempt failed, retry with matcher support
+    try {
+      assertObjectContainsImpl(actual, expected, msg, true)
+    } catch {
+      throw originalError
+    }
+  }
+}
+
 /**
  * @param {string} actual
  * @param {string} [msg]
@@ -688,6 +717,9 @@ function assertUUID (actual, msg = 'not a valid UUID') {
 }
 
 module.exports = {
+  ANY_NUMBER,
+  ANY_STRING,
+  ANY_VALUE,
   FakeAgent,
   hookFile,
   assertObjectContains,

packages/dd-trace/test/setup/helpers/plugin-test-helpers/index.js

@@ -0,0 +1,37 @@
+'use strict'
+
+const { describe, before, after } = require('mocha')
+const agent = require('../../../plugins/agent')
+const { withVersions } = require('../../mocha')
+
+function createIntegrationTestSuite (pluginName, packageName, options, testCallback) {
+  describe('Plugin', () => {
+    describe(pluginName, () => {
+      withVersions(pluginName, packageName, version => {
+        const meta = { agent, tracer: null, mod: null }
+
+        describe('without configuration', () => {
+          before(async () => {
+            const plugins = [pluginName, ...(options.additionalPlugins || [])]
+            const pluginConfigs = [options.pluginConfig || {}, ...(options.additionalPlugins || []).map(() => ({}))]
+            await agent.load(plugins, pluginConfigs)
+          })
+
+          after(async () => {
+            await agent.close({ ritmReset: false })
+          })
+
+          before(async () => {
+            meta.tracer = require('../../../../../dd-trace').init()
+            const mod = require(`../../../../../../versions/${packageName}@${version}`)
+            meta.mod = options.subModule ? mod.get(options.subModule) : mod.get()
+          })
+
+          testCallback(meta)
+        })
+      })
+    })
+  })
+}
+
+module.exports = { createIntegrationTestSuite }