[test optimization] allow overwriting the RUM flush wait time by setting environment variable DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS

* feat(test-optimization): allow overwriting the RUM flush wait time by setting environment variable DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS

* chore(test-optimization): add more error and debug logs for rum integrations

Author: calvinbayer

packages/datadog-instrumentations/src/playwright.js

@@ -11,6 +11,9 @@ const {
   getIsFaultyEarlyFlakeDetection
 } = require('../../dd-trace/src/plugins/util/test')
 const log = require('../../dd-trace/src/log')
+const {
+  getEnvironmentVariable
+} = require('../../dd-trace/src/config-helper')
 const { DD_MAJOR } = require('../../../version')
 
 const testStartCh = channel('ci:playwright:test:start')
@@ -38,7 +41,7 @@ const testSuiteToTestStatuses = new Map()
 const testSuiteToErrors = new Map()
 const testsToTestStatuses = new Map()
 
-const RUM_FLUSH_WAIT_TIME = 1000
+const RUM_FLUSH_WAIT_TIME = Number(getEnvironmentVariable('DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS')) || 1000
 
 let applyRepeatEachIndex = null
 
@@ -1078,9 +1081,19 @@ addHook({
 
     try {
       if (page) {
-        const isRumActive = await page.evaluate(() => {
-          return window.DD_RUM && window.DD_RUM.getInternalContext ? !!window.DD_RUM.getInternalContext() : false
+        const { isRumInstrumented, isRumActive, rumSamplingRate } = await page.evaluate(() => {
+          const isRumInstrumented = !!window.DD_RUM
+          const isRumActive = window.DD_RUM && window.DD_RUM.getInternalContext
+            ? !!window.DD_RUM.getInternalContext()
+            : false
+          const rumSamplingRate = window.DD_RUM && window.DD_RUM.getInitConfiguration
+            ? window.DD_RUM.getInitConfiguration().sessionSampleRate
+            : null
+          return { isRumInstrumented, isRumActive, rumSamplingRate }
         })
+        if (isRumInstrumented && rumSamplingRate < 100 && !isRumActive) {
+          log.debug("RUM was detected on the page, but it isn't active because the sampling rate is below 100%")
+        }
 
         if (isRumActive) {
           testPageGotoCh.publish({
@@ -1089,8 +1102,9 @@ addHook({
           })
         }
       }
-    } catch {
+    } catch (e) {
       // ignore errors such as redirects, context destroyed, etc
+      log.error('goto hook error', e)
     }
 
     return response
@@ -1182,8 +1196,9 @@ addHook({
                   }
                 }
               }
-            } catch {
+            } catch (e) {
               // ignore errors
+              log.error('afterEach hook error', e)
             }
           },
           title: 'afterEach hook',

packages/datadog-instrumentations/src/selenium.js

@@ -2,6 +2,7 @@
 
 const { addHook, channel } = require('./helpers/instrument')
 const shimmer = require('../../datadog-shimmer')
+const { getEnvironmentVariable } = require('../../dd-trace/src/config-helper')
 
 const ciSeleniumDriverGetStartCh = channel('ci:selenium:driver:get')
 
@@ -15,7 +16,8 @@ if (window.DD_RUM && window.DD_RUM.stopSession) {
 `
 const IS_RUM_ACTIVE_SCRIPT = 'return !!window.DD_RUM'
 
-const DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS = 500
+const DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS =
+Number(getEnvironmentVariable('DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS')) || 500
 const DD_CIVISIBILITY_TEST_EXECUTION_ID_COOKIE_NAME = 'datadog-ci-visibility-test-execution-id'
 
 // TODO: can we increase the supported version range?

packages/dd-trace/src/supported-configurations.json

@@ -50,6 +50,7 @@
     "DD_CIVISIBILITY_GIT_UPLOAD_ENABLED": ["A"],
     "DD_CIVISIBILITY_IMPACTED_TESTS_DETECTION_ENABLED": ["A"],
     "DD_CIVISIBILITY_ITR_ENABLED": ["A"],
+    "DD_CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS": ["A"],
     "DD_CIVISIBILITY_MANUAL_API_ENABLED": ["A"],
     "DD_CIVISIBILITY_TEST_COMMAND": ["A"],
     "DD_CIVISIBILITY_TEST_MODULE_ID": ["A"],