instrumentation telemetry: validate session id headers

docs/understand/weblogs/end-to-end_weblog.md

@@ -899,6 +899,21 @@ It supports the following body fields:
 
 This endpoint is OPTIONAL and not related to any test, but to the testing process. When called, it should flush any remaining data from the library to the respective outputs, usually the agent. See more in `docs/edit/flushing.md`.
 
+### GET /spawn_child
+
+This endpoint is used for telemetry session ID header tests (Stable Service Instance Identifier RFC). It must fork or exec a child process, pass in the required arguments, wait for the child, and return a response. Used to validate `DD-Session-ID`, `DD-Root-Session-ID`, and `DD-Parent-Session-ID` headers in instrumentation telemetry across process forks.
+RFC: https://docs.google.com/document/d/1ECKj9_NnwaKYtFqm3p3Rlpicx5d-OQcdj9kI2jvRqVU/edit?tab=t.0#heading=h.ojliy5oytqgg
+
+Required query parameters:
+
+- `sleep`: number of seconds the child process should sleep before exiting
+- `crash`: boolean (required) — `true` to kill the child with SIGSEGV after sleep, `false` to let it exit gracefully
+- `fork`: boolean (required) — `true` to use fork (parent-child), `false` to use exec. Runtimes that do not support fork (e.g. Java, C#) return 400 if `fork=true` is passed.
+
+Returns 200 status code on success. Response body may contain a message such as `Child process {pid} exited`. Returns 400 if `sleep`, `crash`, or `fork` is missing or invalid, or if `fork=true` is passed on a runtime that does not support forking.
+
+Note: `/fork_and_crash` exists only in lib-injection weblogs, not in end-to-end weblogs.
+
 ### \[GET,POST\] /rasp/lfi
 
 This endpoint is used to test for local file inclusion / path traversal attacks, consequently it must perform an operation on a file or directory, e.g. `open` with a relative path. The chosen operation must be injected with the `GET` or `POST` parameter.

manifests/cpp.yml

@@ -310,5 +310,7 @@ manifest:
   tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET)
   tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java)
   tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_required_headers: missing_feature

manifests/cpp_httpd.yml

@@ -205,6 +205,8 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once:
     - declaration: flaky (APMAPI-1876)
       component_version: ">1.0.4"
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.0.3'  # Modified by easy win activation script
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting::test_telemetry_enhanced_config_reporting_precedence: missing_feature  # Created by easy win activation script

manifests/cpp_kong.yml

@@ -36,5 +36,7 @@ manifest:
   tests/test_span_events.py: missing_feature
   tests/test_standard_tags.py: irrelevant
   tests/test_telemetry.py: missing_feature
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_v1_payloads.py: missing_feature

manifests/cpp_nginx.yml

@@ -427,6 +427,8 @@ manifest:
       component_version: <1.12.0
   tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.)
   tests/test_telemetry.py::Test_Telemetry::test_proxy_forwarding: missing_feature  # Created by easy win activation script
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: '>=1.12.0'
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: '>=1.12.0'
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_proxy_enrichment: missing_feature  # Created by easy win activation script
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.12.0'  # Modified by easy win activation script

manifests/dotnet.yml

@@ -1184,6 +1184,8 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_app_started_is_first_message:  # Easy win for poc, uds and version 3.36.0
     - declaration: bug (APMAPI-728)
       component_version: '>=3.4.0'
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (.NET does not support fork)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: v3.41.0
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0
   tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature

manifests/golang.yml

@@ -1511,6 +1511,11 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant
   tests/test_telemetry.py::Test_Telemetry::test_app_dependencies_loaded: irrelevant
   tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (Go does not support fork; use test_session_id_headers_across_spawned instead)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned:
+    - weblog_declaration:
+        '*': irrelevant
+        net-http: '>=2.8.0'
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature
   tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature

manifests/java.yml

@@ -4338,6 +4338,11 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_seq_id:  # Created by easy win activation script
     - weblog_declaration:
         spring-boot-3-native: missing_feature (GraalVM. Tracing support only)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (Java does not support fork)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned:
+    - weblog_declaration:
+        "*": irrelevant
+        spring-boot: '>=1.61.0'
   tests/test_telemetry.py::Test_Telemetry::test_status_ok:  # Created by easy win activation script
     - weblog_declaration:
         spring-boot-3-native: missing_feature (GraalVM. Tracing support only)

manifests/nodejs.yml

@@ -87,6 +87,7 @@ refs:
   - &ref_5_88_0 '>=5.88.0'
   - &ref_5_89_0 '>=5.89.0'
   - &ref_5_90_0 '>=5.90.0'
+  - &ref_5_93_0 '>=5.93.0'
   - &ref_6_0_0 '>=6.0.0-pre'
 manifest:
   tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts:
@@ -2489,6 +2490,14 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_proxy_forwarding:
     - weblog_declaration:
         nextjs: missing_feature
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks:
+    - weblog_declaration:
+        "*": missing_feature (spawn_child endpoint not implemented)
+        express4: *ref_5_93_0
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned:
+    - weblog_declaration:
+        "*": missing_feature (spawn_child endpoint not implemented)
+        express4: *ref_5_93_0
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_proxy_enrichment:
     - weblog_declaration:

manifests/php.yml

@@ -1470,6 +1470,8 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_app_started_client_configuration: missing_feature (Telemetry is not implemented yet.)
   tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: irrelevant (PHP registers 2 telemetry services)
   tests/test_telemetry.py::Test_Telemetry::test_seq_id: irrelevant (PHP registers 2 telemetry services)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature
   tests/test_telemetry.py::Test_TelemetrySCAEnvVar: v0.99.1

manifests/python.yml

@@ -2358,6 +2358,14 @@ manifest:
     - declaration: flaky (APMRP-360)
       component_version: <=1.20.2
     - declaration: bug (APMAPI-1858)
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks:
+    - weblog_declaration:
+        "*": missing_feature (not implemented yet)
+        flask-poc: '>=4.8.0'
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned:
+    - weblog_declaration:
+        flask-poc: missing_feature (not implemented yet)
+        "*": missing_feature (spawn_child endpoint not implemented)
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting:
     - weblog_declaration:

manifests/ruby.yml

@@ -2277,6 +2277,14 @@ manifest:
   tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once:
     - declaration: missing_feature (app-started not sent)
       component_version: <1.22.0
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks:
+    - weblog_declaration:
+        '*': missing_feature
+        rails72: '>=2.31.0'
+  tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned:
+    - weblog_declaration:
+        '*': missing_feature
+        rails72: '>=2.31.0'
   tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)"
   tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature (Temporarily disabled, will be re-enabled once dd-trace-rb#5483 is merged)
   tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature

tests/test_telemetry.py

@@ -580,6 +580,83 @@ def test_app_product_change(self):
         if app_product_change_event_found is False:
             raise Exception("app-product-change is not emitted when product change is enabled")
 
+    def setup_session_id_headers_across_forks(self):
+        """Trigger spawn_child endpoint to create a fork tree for session ID header validation."""
+        weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": True})
+
+    def setup_session_id_headers_across_spawned(self):
+        """Trigger spawn_child endpoint with exec (fork=false) for session ID header validation."""
+        weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": False})
+
+    def _validate_session_id_headers_across_processes(self) -> None:
+        """Validate DD-Session-ID, DD-Root-Session-ID, DD-Parent-Session-ID in telemetry.
+
+        Stable Service Instance Identifier RFC: each app instance has one root runtime_id.
+        DD-Session-ID (instance id) must equal runtime_id. When only DD-Session-ID is sent
+        (no DD-Root-Session-ID), the process is treated as the root. This test confirms
+        at least two different runtimes are captured (parent and child from spawn_child).
+        """
+        # Use lifecycle events only; metrics and log events from lib-datadog can contain
+        # runtime/session_ids that do not map to tracer-generated telemetry.
+        telemetry_data = list(interfaces.library.get_lifecycle_events())
+        if not telemetry_data:
+            raise ValueError("No telemetry data to validate on")
+
+        assert len(telemetry_data) > 1, (
+            f"Expected multiple telemetry events to verify consistency, got {len(telemetry_data)}"
+        )
+
+        runtime_ids = set[str]()
+        parent_runtime_ids = set[str]()
+        root_runtime_ids = set[str]()
+
+        for data in telemetry_data:
+            # Headers are not case sensitive
+            curr_sid = get_header(data, "request", "dd-session-id")
+            curr_rid = get_header(data, "request", "dd-root-session-id")
+            curr_pid = get_header(data, "request", "dd-parent-session-id")
+            curr_id = data["request"]["content"].get("runtime_id")
+
+            # Instance id (DD-Session-ID) must be present in all lifecycle events and equal to runtime_id
+            assert curr_sid is not None, f"DD-Session-ID is required in telemetry data: {data}"
+            assert curr_sid == curr_id, f"DD-Session-ID must match runtime_id: {curr_sid} != {curr_id}"
+
+            runtime_ids.add(curr_id)
+            if curr_pid is not None:
+                parent_runtime_ids.add(curr_pid)
+            if curr_rid is not None:
+                root_runtime_ids.add(curr_rid)
+            else:
+                # If dd-root-session-id is not set, dd-session-id is treated as root
+                root_runtime_ids.add(curr_id)
+
+        # One root per app instance: all processes share the same root session ID
+        assert len(root_runtime_ids) == 1, f"Expected 1 root runtime_id, got {root_runtime_ids}"
+
+        if len(runtime_ids) > 1:
+            # Multiple runtimes (per-process tracers): root must be consistent
+            # across all payloads from all processes
+            if parent_runtime_ids:
+                # DD-Parent-Session-ID is optional but must reference a known runtime if present
+                missing_parent_runtime_ids = parent_runtime_ids.difference(runtime_ids)
+                assert not missing_parent_runtime_ids, (
+                    f"Parent runtime_id with no telemetry data: {missing_parent_runtime_ids}"
+                )
+        else:
+            # Single runtime (e.g. nginx workers sharing one tracer): session ID
+            # must be consistent across all events
+            sole_rid = next(iter(runtime_ids))
+            sole_root = next(iter(root_runtime_ids))
+            assert sole_rid == sole_root, f"Single runtime_id {sole_rid} does not match root {sole_root}"
+
+    def test_session_id_headers_across_forks(self):
+        """Test session ID headers in telemetry (fork=true). Stable Service Instance Identifier RFC."""
+        self._validate_session_id_headers_across_processes()
+
+    def test_session_id_headers_across_spawned(self):
+        """Test session ID headers in telemetry (fork=false, exec). Stable Service Instance Identifier RFC."""
+        self._validate_session_id_headers_across_processes()
+
 
 @features.telemetry_app_started_event
 @scenarios.telemetry_enhanced_config_reporting

utils/build/docker/cpp_nginx/nginx/backend.c

@@ -6,6 +6,7 @@
 #include <signal.h>
 #include <stdio.h>
 #include <sys/stat.h>
+#include <sys/wait.h>
 #include <unistd.h>
 
 #define PORT 7778
@@ -310,6 +311,85 @@ static enum MHD_Result answer_to_connection(void *cls, struct MHD_Connection *co
         return ret;
     }
 
+    if (strcmp(url, "/spawn_child") == 0) {
+        const char *sleep_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "sleep");
+        const char *crash_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "crash");
+        const char *fork_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "fork");
+
+        if (!sleep_str || !crash_str || !fork_str) {
+            const char *msg = "sleep, crash, and fork parameters required";
+            struct MHD_Response *response = MHD_create_response_from_buffer(
+                strlen(msg), (void *)msg, MHD_RESPMEM_PERSISTENT);
+            int ret = MHD_queue_response(connection, 400, response);
+            MHD_destroy_response(response);
+            return ret;
+        }
+
+        int sleep_secs = atoi(sleep_str);
+        bool do_crash = strcmp(crash_str, "true") == 0;
+        bool use_fork = strcmp(fork_str, "true") == 0;
+
+        if (use_fork) {
+            pid_t pid = fork();
+            if (pid < 0) {
+                const char *msg = "fork failed";
+                struct MHD_Response *response = MHD_create_response_from_buffer(
+                    strlen(msg), (void *)msg, MHD_RESPMEM_PERSISTENT);
+                int ret = MHD_queue_response(connection, 500, response);
+                MHD_destroy_response(response);
+                return ret;
+            }
+            if (pid == 0) {
+                sleep(sleep_secs);
+                if (do_crash) {
+                    raise(SIGSEGV);
+                }
+                _exit(0);
+            }
+            int wstatus;
+            waitpid(pid, &wstatus, 0);
+            char buf[128];
+            snprintf(buf, sizeof(buf), "Child process %d exited with status %d", pid, WEXITSTATUS(wstatus));
+            struct MHD_Response *response = MHD_create_response_from_buffer(
+                strlen(buf), buf, MHD_RESPMEM_MUST_COPY);
+            int ret = MHD_queue_response(connection, 200, response);
+            MHD_destroy_response(response);
+            return ret;
+        }
+
+        /* exec path: fork + exec a child process */
+        {
+            pid_t pid = fork();
+            if (pid < 0) {
+                const char *msg = "fork failed";
+                struct MHD_Response *response = MHD_create_response_from_buffer(
+                    strlen(msg), (void *)msg, MHD_RESPMEM_PERSISTENT);
+                int ret = MHD_queue_response(connection, 500, response);
+                MHD_destroy_response(response);
+                return ret;
+            }
+            if (pid == 0) {
+                if (do_crash) {
+                    execlp("sh", "sh", "-c",
+                           sleep_str[0] ? "sleep $0 && kill -SEGV $$" : "kill -SEGV $$",
+                           sleep_str, (char *)NULL);
+                } else {
+                    execlp("sleep", "sleep", sleep_str, (char *)NULL);
+                }
+                _exit(1);
+            }
+            int wstatus;
+            waitpid(pid, &wstatus, 0);
+            char buf[128];
+            snprintf(buf, sizeof(buf), "Child process %d exited with status %d", pid, WEXITSTATUS(wstatus));
+            struct MHD_Response *response = MHD_create_response_from_buffer(
+                strlen(buf), buf, MHD_RESPMEM_MUST_COPY);
+            int ret = MHD_queue_response(connection, 200, response);
+            MHD_destroy_response(response);
+            return ret;
+        }
+    }
+
     if (strcmp(url, "/content") != 0 || !status_str || !value)
         return MHD_NO;  // Only respond to the correct URL and if all parameters are present
 

utils/build/docker/cpp_nginx/nginx/nginx-waf.conf

@@ -1,6 +1,7 @@
 error_log /var/log/nginx/error.log info;
 
 load_module modules/ngx_http_datadog_module.so;
+worker_processes 2;
 thread_pool waf_thread_pool threads=2 max_queue=1000;
 
 events {

utils/build/docker/cpp_nginx/nginx/nginx.conf

@@ -1,5 +1,7 @@
 load_module modules/ngx_http_datadog_module.so;
 
+worker_processes 2;
+
 events {
     worker_connections  1024;
 }
@@ -44,6 +46,10 @@ http {
                proxy_pass http://127.0.0.1:7778;
         }
 
+        location /spawn_child {
+          proxy_pass http://127.0.0.1:7778;
+        }
+
         location / {
           root /builds;
           try_files /hello.html =404;