From 8d9511f08725ccc1c807dc8f12a4ba39e59d6ba4 Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 16 Mar 2026 16:05:48 -0400 Subject: [PATCH 01/24] instrumentation telemetry: validate session id headers --- docs/weblog/end-to-end_weblog.md | 14 +++ manifests/dotnet.yml | 1 + manifests/golang.yml | 1 + manifests/java.yml | 1 + manifests/nodejs.yml | 6 ++ manifests/php.yml | 1 + manifests/python.yml | 8 ++ manifests/ruby.yml | 12 +++ tests/test_telemetry.py | 84 +++++++++++++++++ .../weblog/Endpoints/SpawnChildEndpoint.cs | 93 +++++++++++++++++++ utils/build/docker/dotnet/weblog/Program.cs | 18 ++++ .../golang/app/_shared/common/spawn_child.go | 51 ++++++++++ .../build/docker/golang/app/net-http/main.go | 2 + .../system_tests/springboot/App.java | 29 ++++++ utils/build/docker/nodejs/express/app.js | 37 ++++++++ .../build/docker/nodejs/express/fork_child.js | 11 +++ utils/build/docker/php/apache-mod/php.conf | 1 + utils/build/docker/php/common/spawn_child.php | 43 +++++++++ utils/build/docker/php/php-fpm/php-fpm.conf | 1 + utils/build/docker/python/flask/app.py | 39 ++++++++ .../app/controllers/system_test_controller.rb | 33 +++++++ .../docker/ruby/rails72/config/routes.rb | 1 + 22 files changed, 487 insertions(+) create mode 100644 utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs create mode 100644 utils/build/docker/golang/app/_shared/common/spawn_child.go create mode 100644 utils/build/docker/nodejs/express/fork_child.js create mode 100644 utils/build/docker/php/common/spawn_child.php diff --git a/docs/weblog/end-to-end_weblog.md b/docs/weblog/end-to-end_weblog.md index 0995e5e6492..cf98d4d9757 100644 --- a/docs/weblog/end-to-end_weblog.md +++ b/docs/weblog/end-to-end_weblog.md @@ -899,6 +899,20 @@ It supports the following body fields: This endpoint is OPTIONAL and not related to any test, but to the testing process. When called, it should flush any remaining data from the library to the respective outputs, usually the agent. See more in `docs/internals/flushing.md`. +### GET /spawn_child + +This endpoint is used for telemetry session ID header tests (Stable Service Instance Identifier RFC). It must fork or exec a child process, pass in the required arguments, wait for the child, and return a response. Used to validate `DD-Session-ID`, `DD-Root-Session-ID`, and `DD-Parent-Session-ID` headers in instrumentation telemetry across process forks. + +Required query parameters: + +- `sleep`: number of seconds the child process should sleep before exiting +- `crash`: boolean (required) — `true` to kill the child with SIGSEGV after sleep, `false` to let it exit gracefully +- `fork`: boolean (required) — `true` to use fork (parent-child), `false` to use exec. Runtimes that do not support fork (e.g. Java, C#) return 400 if `fork=true` is passed. + +Returns 200 status code on success. Response body may contain a message such as `Child process {pid} exited`. Returns 400 if `sleep`, `crash`, or `fork` is missing or invalid, or if `fork=true` is passed on a runtime that does not support forking. + +Note: `/fork_and_crash` exists only in lib-injection weblogs, not in end-to-end weblogs. + ### \[GET,POST\] /rasp/lfi This endpoint is used to test for local file inclusion / path traversal attacks, consequently it must perform an operation on a file or directory, e.g. `open` with a relative path. The chosen operation must be injected with the `GET` or `POST` parameter. diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 392c7380f9c..4ef119a2036 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1115,6 +1115,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_is_first_message: # Easy win for poc, uds and version 3.36.0 - declaration: bug (APMAPI-728) component_version: '>=3.4.0' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0 tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature tests/test_telemetry.py::Test_TelemetryV2: v2.35.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index f3b83b64ae3..6575c6a5c1c 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1399,6 +1399,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_dependencies_loaded: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature tests/test_telemetry.py::Test_TelemetryV2: v1.49.1 diff --git a/manifests/java.yml b/manifests/java.yml index 722606c5f38..16b919c402f 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4016,6 +4016,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_seq_id: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) tests/test_telemetry.py::Test_Telemetry::test_status_ok: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 9915271383b..e1afd3ba7db 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2192,6 +2192,12 @@ manifest: uds-express4: *ref_3_7_0 tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: + - weblog_declaration: + "*": v4.21.0 + nextjs: missing_feature (spawn_child endpoint not implemented) + fastify: missing_feature (spawn_child endpoint not implemented) + express4-typescript: missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: "*": irrelevant diff --git a/manifests/php.yml b/manifests/php.yml index 2a58aa79948..52b011c42e4 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -949,6 +949,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_client_configuration: missing_feature (Telemetry is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: irrelevant (PHP registers 2 telemetry services) tests/test_telemetry.py::Test_Telemetry::test_seq_id: irrelevant (PHP registers 2 telemetry services) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: v0.99.1 tests/test_telemetry.py::Test_TelemetryV2: v0.90 diff --git a/manifests/python.yml b/manifests/python.yml index 072bcdac329..6d674d0e610 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2088,6 +2088,14 @@ manifest: - declaration: flaky (APMRP-360) component_version: <=1.20.2 - declaration: bug (APMAPI-1858) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: + - weblog_declaration: + "*": v2.9.0 + tornado: missing_feature (spawn_child endpoint not implemented) + django-poc: missing_feature (spawn_child endpoint not implemented) + django-py3.13: missing_feature (spawn_child endpoint not implemented) + fastapi: missing_feature (spawn_child endpoint not implemented) + python3.12: missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: "*": irrelevant diff --git a/manifests/ruby.yml b/manifests/ruby.yml index c17a9cef81f..7f55a1048ef 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2111,6 +2111,18 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: - declaration: missing_feature (app-started not sent) component_version: <1.22.0 + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: + - weblog_declaration: + rack: missing_feature (spawn_child endpoint not implemented) + rails42: missing_feature (spawn_child endpoint not implemented) + rails52: missing_feature (spawn_child endpoint not implemented) + rails61: missing_feature (spawn_child endpoint not implemented) + rails80: missing_feature (spawn_child endpoint not implemented) + sinatra14: missing_feature (spawn_child endpoint not implemented) + sinatra22: missing_feature (spawn_child endpoint not implemented) + sinatra32: missing_feature (spawn_child endpoint not implemented) + sinatra41: missing_feature (spawn_child endpoint not implemented) + "*": v2.18.0 tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: # Modified by easy win activation script - weblog_declaration: '*': missing_feature diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 7c7f8e9519d..8f45a090485 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -560,6 +560,90 @@ def test_app_product_change(self): if app_product_change_event_found is False: raise Exception("app-product-change is not emitted when product change is enabled") + def setup_session_id_headers_across_forks(self): + """Trigger spawn_child endpoint to create a fork tree for session ID header validation.""" + weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": True}) + # Allow parent to flush telemetry after child (parent returns after waitpid) + time.sleep(3) + + def test_session_id_headers_across_forks(self): + """Test session ID headers in telemetry (Stable Service Instance Identifier RFC). + + setup_session_id_headers_across_forks triggers spawn_child, creating a parent-child pair. + Validates: DD-Session-ID matches runtime_id; root has no DD-Root/DD-Parent-Session-ID; + child has DD-Root-Session-ID pointing to root. + """ + telemetry_data = list(interfaces.library.get_telemetry_data(flatten_message_batches=False)) + if not telemetry_data: + raise ValueError("No telemetry data to validate on") + + # Group by DD-Session-ID (equals runtime_id) + by_session_id: dict[str, list[dict]] = defaultdict(list) + for data in telemetry_data: + sid = get_header(data, "request", "dd-session-id") + if sid: + by_session_id[sid].append(data) + + if not by_session_id: + raise ValueError("No telemetry with DD-Session-ID found") + + # Find child (has DD-Root-Session-ID) + child_sid = None + root_sid = None + for sid, requests in by_session_id.items(): + root_sid = get_header(requests[0], "request", "dd-root-session-id") + if root_sid: + child_sid = sid + break + + assert child_sid is not None, "No forked process found (DD-Root-Session-ID missing)" + assert root_sid is not None, "No forked process found (DD-Root-Session-ID missing)" + + # Validate DD-Session-ID matches runtime_id when present (root may omit header in some tracer versions) + for data in telemetry_data: + runtime_id = data["request"]["content"].get("runtime_id") + sid = get_header(data, "request", "dd-session-id") + if runtime_id and sid is not None: + assert sid == runtime_id, ( + f"DD-Session-ID '{sid}' != runtime_id '{runtime_id}' in {data['log_filename']}" + ) + + # Root: find by runtime_id (root may not have DD-Session-ID so may not be in by_session_id) + root_requests = by_session_id.get(root_sid, []) + root_data: dict | None = root_requests[0] if root_requests else None + if root_data is None: + root_data = next( + (d for d in telemetry_data if d["request"]["content"].get("runtime_id") == root_sid), + None, + ) + if root_data is None: + runtime_ids = {d["request"]["content"].get("runtime_id") for d in telemetry_data} + session_ids = set(by_session_id.keys()) + raise AssertionError( + f"Root session '{root_sid}' (from child's DD-Root-Session-ID) not in telemetry. " + f"Parent may not have flushed before validation. " + f"runtime_ids in logs: {runtime_ids}, session_ids: {session_ids}" + ) + + # Root: no DD-Root-Session-ID, no DD-Parent-Session-ID + assert get_header(root_data, "request", "dd-root-session-id") is None, "Root must not have DD-Root-Session-ID" + assert get_header(root_data, "request", "dd-parent-session-id") is None, ( + "Root must not have DD-Parent-Session-ID" + ) + + # Child: DD-Root-Session-ID points to root + child_data = by_session_id[child_sid][0] + assert get_header(child_data, "request", "dd-root-session-id") == root_sid, ( + "Child DD-Root-Session-ID must match root" + ) + + # DD-Parent-Session-ID if present must reference root or known session + parent_sid = get_header(child_data, "request", "dd-parent-session-id") + if parent_sid: + assert parent_sid == root_sid or parent_sid in by_session_id, ( + f"DD-Parent-Session-ID '{parent_sid}' must be root or in telemetry" + ) + @features.telemetry_app_started_event @scenarios.telemetry_enhanced_config_reporting diff --git a/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs b/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs new file mode 100644 index 00000000000..1ce5a40cb47 --- /dev/null +++ b/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs @@ -0,0 +1,93 @@ +using System.Diagnostics; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; + +namespace weblog +{ + /// + /// Spawn child for telemetry session ID header tests. Inspired by lib-injection fork_and_crash: + /// fork=true spawns same process with env vars; fork=false uses exec (shell). + /// + public class SpawnChildEndpoint : ISystemTestEndpoint + { + public void Register(Microsoft.AspNetCore.Routing.IEndpointRouteBuilder routeBuilder) + { + routeBuilder.MapGet("/spawn_child", async context => + { + var sleepStr = context.Request.Query["sleep"].ToString(); + var crashStr = (context.Request.Query["crash"].ToString() ?? "").ToLowerInvariant(); + var forkStr = (context.Request.Query["fork"].ToString() ?? "").ToLowerInvariant(); + + if (string.IsNullOrEmpty(sleepStr) || !int.TryParse(sleepStr, out int sleep) || sleep < 0) + { + context.Response.StatusCode = 400; + await context.Response.WriteAsync("sleep required"); + return; + } + if (crashStr != "true" && crashStr != "false") + { + context.Response.StatusCode = 400; + await context.Response.WriteAsync("crash required (boolean)"); + return; + } + if (forkStr != "true" && forkStr != "false") + { + context.Response.StatusCode = 400; + await context.Response.WriteAsync("fork required (boolean)"); + return; + } + + var crash = crashStr == "true"; + Process process; + + if (forkStr == "true") + { + // Fork path: spawn same process with env vars (lib-injection fork_and_crash pattern) + var cmdArgs = Environment.GetCommandLineArgs(); + var args = cmdArgs.Length > 1 ? string.Join(" ", cmdArgs.Skip(1)) : "app.dll"; + var startInfo = new ProcessStartInfo + { + FileName = Environment.ProcessPath ?? "/usr/share/dotnet/dotnet", + Arguments = args, + WorkingDirectory = Environment.CurrentDirectory, + }; + startInfo.Environment["SPAWN_CHILD_FORKED"] = "1"; + startInfo.Environment["SPAWN_CHILD_SLEEP"] = sleep.ToString(); + startInfo.Environment["SPAWN_CHILD_CRASH"] = crash ? "1" : "0"; + + process = Process.Start(startInfo); + } + else + { + // Exec path: shell script + var script = crash + ? $"sleep {sleep} && kill -SEGV $$$$" + : $"sleep {sleep} && exit 0"; + var startInfo = new ProcessStartInfo + { + FileName = "/bin/sh", + Arguments = $"-c \"{script}\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + process = Process.Start(startInfo); + } + + if (process == null) + { + context.Response.StatusCode = 500; + await context.Response.WriteAsync("Failed to start child process"); + return; + } + + using (process) + { + await process.WaitForExitAsync(); + context.Response.ContentType = "text/plain"; + await context.Response.WriteAsync($"Process {process.Id} has exited with code {process.ExitCode}"); + } + }); + } + } +} diff --git a/utils/build/docker/dotnet/weblog/Program.cs b/utils/build/docker/dotnet/weblog/Program.cs index d6b1cba06d0..b7308651313 100644 --- a/utils/build/docker/dotnet/weblog/Program.cs +++ b/utils/build/docker/dotnet/weblog/Program.cs @@ -11,6 +11,24 @@ public class Program { public static void Main(string[] args) { + // Spawn-child forked mode (inspired by lib-injection fork_and_crash): sleep then optionally crash + if (Environment.GetEnvironmentVariable("SPAWN_CHILD_FORKED") != null) + { + var sleepSec = int.TryParse(Environment.GetEnvironmentVariable("SPAWN_CHILD_SLEEP"), out var s) ? s : 0; + var doCrash = Environment.GetEnvironmentVariable("SPAWN_CHILD_CRASH") == "1"; + if (sleepSec > 0) + { + Thread.Sleep(sleepSec * 1000); + } + if (doCrash) + { + var t = new Thread(() => throw new BadImageFormatException("spawn_child crash")); + t.Start(); + t.Join(); + } + return; + } + // Enable Datadog log injection only if CONFIG_CHAINING_TEST is set to "true" if (Environment.GetEnvironmentVariable("CONFIG_CHAINING_TEST") == "true") { diff --git a/utils/build/docker/golang/app/_shared/common/spawn_child.go b/utils/build/docker/golang/app/_shared/common/spawn_child.go new file mode 100644 index 00000000000..abd0d6ac17d --- /dev/null +++ b/utils/build/docker/golang/app/_shared/common/spawn_child.go @@ -0,0 +1,51 @@ +package common + +import ( + "fmt" + "net/http" + "os/exec" + "strconv" + "strings" +) + +// SpawnChild handles GET /spawn_child for telemetry session ID header tests. +// Go does not support fork; returns 400 when fork=true. Otherwise uses exec. +func SpawnChild(w http.ResponseWriter, r *http.Request) { + sleepStr := r.URL.Query().Get("sleep") + crashStr := strings.ToLower(r.URL.Query().Get("crash")) + forkStr := strings.ToLower(r.URL.Query().Get("fork")) + + sleep, err := strconv.Atoi(sleepStr) + if err != nil || sleep < 0 { + http.Error(w, "sleep required", http.StatusBadRequest) + return + } + if crashStr != "true" && crashStr != "false" { + http.Error(w, "crash required (boolean)", http.StatusBadRequest) + return + } + if forkStr != "true" && forkStr != "false" { + http.Error(w, "fork required (boolean)", http.StatusBadRequest) + return + } + if forkStr == "true" { + http.Error(w, "fork not supported", http.StatusBadRequest) + return + } + + crash := crashStr == "true" + script := fmt.Sprintf("sleep %d", sleep) + if crash { + script += " && kill -SEGV $$" + } else { + script += " && exit 0" + } + cmd := exec.Command("sh", "-c", script) + _ = cmd.Run() + status := 0 + if cmd.ProcessState != nil { + status = cmd.ProcessState.ExitCode() + } + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(fmt.Sprintf("Child process exited with status %d", status))) +} diff --git a/utils/build/docker/golang/app/net-http/main.go b/utils/build/docker/golang/app/net-http/main.go index 85b8a5f35f1..31548e446b9 100644 --- a/utils/build/docker/golang/app/net-http/main.go +++ b/utils/build/docker/golang/app/net-http/main.go @@ -187,6 +187,8 @@ func main() { w.Write([]byte("OK")) }) + mux.HandleFunc("/spawn_child", common.SpawnChild) + mux.HandleFunc("/make_distant_call", func(w http.ResponseWriter, r *http.Request) { url := r.URL.Query().Get("url") if url == "" { diff --git a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java index 6b75783ae67..1f5102e77c0 100644 --- a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java +++ b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java @@ -318,6 +318,35 @@ ResponseEntity status(@RequestParam Integer code) { return new ResponseEntity<>(HttpStatus.valueOf(code)); } + @GetMapping("/spawn_child") + ResponseEntity spawnChild( + @RequestParam(required = false) Integer sleep, + @RequestParam(required = false) String crash, + @RequestParam(required = false) String fork) { + if (sleep == null || sleep < 0) { + return ResponseEntity.badRequest().body("sleep required"); + } + if (crash == null || (!crash.equalsIgnoreCase("true") && !crash.equalsIgnoreCase("false"))) { + return ResponseEntity.badRequest().body("crash required (boolean)"); + } + if (fork == null || (!fork.equalsIgnoreCase("true") && !fork.equalsIgnoreCase("false"))) { + return ResponseEntity.badRequest().body("fork required (boolean)"); + } + if (fork.equalsIgnoreCase("true")) { + return ResponseEntity.badRequest().body("fork not supported"); + } + try { + ProcessBuilder pb = new ProcessBuilder( + "sh", "-c", + String.format("sleep %d && %s", sleep, crash.equalsIgnoreCase("true") ? "kill -SEGV $$" : "exit 0")); + Process p = pb.start(); + int exitCode = p.waitFor(); + return ResponseEntity.ok("Process " + p.pid() + " has exited with code " + exitCode); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Failed: " + e.getMessage()); + } + } + @RequestMapping("/stats-unique") ResponseEntity statsUnique(@RequestParam(defaultValue = "200") Integer code) { return new ResponseEntity<>(HttpStatus.valueOf(code)); diff --git a/utils/build/docker/nodejs/express/app.js b/utils/build/docker/nodejs/express/app.js index fb79aebe5c7..5bd0b85a8d7 100644 --- a/utils/build/docker/nodejs/express/app.js +++ b/utils/build/docker/nodejs/express/app.js @@ -72,6 +72,43 @@ app.get('/', (req, res) => { res.send('Hello world!\n') }) +function subprocessAndExitHandler (req, res) { + const path = require('path') + const { spawn } = require('child_process') + const sleep = req.query.sleep != null ? String(req.query.sleep) : null + const crash = req.query.crash + if (sleep == null || sleep === '') { + res.status(400).send('sleep required') + return + } + const crashStr = String(crash || '').toLowerCase() + const forkStr = String(req.query.fork || '').toLowerCase() + if (crashStr !== 'true' && crashStr !== 'false') { + res.status(400).send('crash required (boolean)') + return + } + if (forkStr !== 'true' && forkStr !== 'false') { + res.status(400).send('fork required (boolean)') + return + } + const useFork = forkStr === 'true' + + if (useFork) { + const child = require('child_process').fork(path.join(__dirname, 'fork_child.js'), [sleep, crashStr]) + child.on('close', (code, signal) => { + res.send(`Child process ${child.pid} exited with code ${code}, signal ${signal}`) + }) + } else { + const child = spawn(process.execPath, [path.join(__dirname, 'fork_child.js'), sleep, crashStr], { + stdio: 'inherit' + }) + child.on('close', (code, signal) => { + res.send(`Child process ${child.pid} exited with code ${code}, signal ${signal}`) + }) + } +} +app.get('/spawn_child', subprocessAndExitHandler) + app.get('/healthcheck', (req, res) => { res.json({ status: 'ok', diff --git a/utils/build/docker/nodejs/express/fork_child.js b/utils/build/docker/nodejs/express/fork_child.js new file mode 100644 index 00000000000..ef612416c0f --- /dev/null +++ b/utils/build/docker/nodejs/express/fork_child.js @@ -0,0 +1,11 @@ +#!/usr/bin/env node +// Child process for spawn_child endpoint. Args: sleep (seconds), crash (true|false). +const sleepSec = parseInt(process.argv[2] || '2', 10) * 1000 +const crash = process.argv[3] === 'true' +setTimeout(() => { + if (crash) { + process.kill(process.pid, 'SIGSEGV') + } else { + process.exit(0) + } +}, sleepSec) diff --git a/utils/build/docker/php/apache-mod/php.conf b/utils/build/docker/php/apache-mod/php.conf index 1c56b062c20..7f1727e3cd2 100644 --- a/utils/build/docker/php/apache-mod/php.conf +++ b/utils/build/docker/php/apache-mod/php.conf @@ -8,6 +8,7 @@ RewriteRule "^/identify-propagate$" "/identify-propagate/" RewriteRule "^/headers$" "/headers/" RewriteRule "^/status$" "/status/" + RewriteRule "^/spawn_child$" "/spawn_child/" RewriteRule "^/read_file$" "/read_file/" RewriteRule "^/make_distant_call$" "/make_distant_call/" RewriteRule "^/log/library$" "/log-library/" diff --git a/utils/build/docker/php/common/spawn_child.php b/utils/build/docker/php/common/spawn_child.php new file mode 100644 index 00000000000..167a2580424 --- /dev/null +++ b/utils/build/docker/php/common/spawn_child.php @@ -0,0 +1,43 @@ +&1', $output, $returnVar); + +header('Content-Type: text/plain'); +echo 'Child process exited with status ' . $returnVar; diff --git a/utils/build/docker/php/php-fpm/php-fpm.conf b/utils/build/docker/php/php-fpm/php-fpm.conf index b5163539de7..b92287e6d43 100644 --- a/utils/build/docker/php/php-fpm/php-fpm.conf +++ b/utils/build/docker/php/php-fpm/php-fpm.conf @@ -23,6 +23,7 @@ RewriteRule "^/identify-propagate$" "/identify-propagate/" RewriteRule "^/headers$" "/headers/" RewriteRule "^/status$" "/status/" + RewriteRule "^/spawn_child$" "/spawn_child/" RewriteRule "^/read_file$" "/read_file/" RewriteRule "^/make_distant_call$" "/make_distant_call/" RewriteRule "^/log/library$" "/log-library/" diff --git a/utils/build/docker/python/flask/app.py b/utils/build/docker/python/flask/app.py index 6c0199f6c72..9f098296c82 100644 --- a/utils/build/docker/python/flask/app.py +++ b/utils/build/docker/python/flask/app.py @@ -14,6 +14,8 @@ import json import logging import os +import signal +import time import random import shlex import subprocess @@ -322,6 +324,43 @@ def healthcheck(): } +@app.route("/spawn_child") +def spawn_child(): + """Spawn child via fork or exec. Params: sleep, crash, fork. Used for telemetry session ID header tests.""" + sleep_arg = request.args.get("sleep", type=int) + crash_arg = request.args.get("crash", "").lower() + fork_arg = (request.args.get("fork") or "").lower() + if sleep_arg is None: + return "sleep required", 400 + if crash_arg not in ("true", "false"): + return "crash required (boolean)", 400 + if fork_arg not in ("true", "false"): + return "fork required (boolean)", 400 + crash = crash_arg == "true" + use_fork = fork_arg == "true" + + if use_fork: + pid = os.fork() + if pid > 0: + _, status = os.waitpid(pid, 0) + return f"Child process {pid} exited with status {status}" + time.sleep(sleep_arg) + if crash: + os.kill(os.getpid(), signal.SIGSEGV) + sys.exit(0) + + # exec path: spawn subprocess + proc = subprocess.run( + [ + sys.executable, + "-c", + f"import time, sys, os, signal; time.sleep({sleep_arg}); os.kill(os.getpid(), signal.SIGSEGV) if {crash} else sys.exit(0)", + ], + timeout=sleep_arg + 5, + ) + return f"Child process exited with status {proc.returncode}" + + @app.route("/sample_rate_route/") def sample_rate(i): return "OK" diff --git a/utils/build/docker/ruby/rails72/app/controllers/system_test_controller.rb b/utils/build/docker/ruby/rails72/app/controllers/system_test_controller.rb index 3ff404b3557..ad85152715c 100644 --- a/utils/build/docker/ruby/rails72/app/controllers/system_test_controller.rb +++ b/utils/build/docker/ruby/rails72/app/controllers/system_test_controller.rb @@ -9,6 +9,39 @@ def root render plain: "Hello world!\n" end + def spawn_child + sleep_sec = params[:sleep]&.to_i + crash = params[:crash].to_s.downcase + fork_param = (params[:fork] || '').to_s.downcase + if sleep_sec.nil? || sleep_sec.negative? + render plain: 'sleep required', status: 400 + return + end + unless %w[true false].include?(crash) + render plain: 'crash required (boolean)', status: 400 + return + end + unless %w[true false].include?(fork_param) + render plain: 'fork required (boolean)', status: 400 + return + end + do_crash = crash == 'true' + use_fork = fork_param == 'true' + + if use_fork + pid = Process.fork do + sleep(sleep_sec) + do_crash ? Process.kill('SEGV', Process.pid) : exit(0) + end + Process.wait(pid) + render plain: "Child process #{pid} exited" + else + pid = Process.spawn('ruby', '-e', "sleep(#{sleep_sec}); #{do_crash ? "Process.kill('SEGV', Process.pid)" : 'exit(0)'}") + Process.wait(pid) + render plain: "Child process #{pid} exited" + end + end + def waf render plain: 'Hello, world!' end diff --git a/utils/build/docker/ruby/rails72/config/routes.rb b/utils/build/docker/ruby/rails72/config/routes.rb index 3301f2cb357..31c45233a38 100644 --- a/utils/build/docker/ruby/rails72/config/routes.rb +++ b/utils/build/docker/ruby/rails72/config/routes.rb @@ -16,6 +16,7 @@ def call(_env) get '/healthcheck' => 'internal#healthcheck' get '/flush' => 'internal#flush' + get '/spawn_child' => 'system_test#spawn_child' get '/waf' => 'system_test#waf' post '/waf' => 'system_test#waf' From f387188d6736b2a1bf335ce1d051ca8bc957fa58 Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 16 Mar 2026 16:16:15 -0400 Subject: [PATCH 02/24] add spawn test and clean up manifest --- manifests/dotnet.yml | 3 ++- manifests/golang.yml | 3 ++- manifests/java.yml | 3 ++- manifests/nodejs.yml | 8 ++------ manifests/php.yml | 3 ++- manifests/python.yml | 12 ++++++------ manifests/ruby.yml | 14 ++------------ tests/test_telemetry.py | 24 ++++++++++++++++-------- 8 files changed, 34 insertions(+), 36 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 4ef119a2036..c983dc05e18 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1115,7 +1115,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_is_first_message: # Easy win for poc, uds and version 3.36.0 - declaration: bug (APMAPI-728) component_version: '>=3.4.0' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: poc, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: poc, not enabled yet)' tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0 tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature tests/test_telemetry.py::Test_TelemetryV2: v2.35.0 diff --git a/manifests/golang.yml b/manifests/golang.yml index 6575c6a5c1c..df91718490c 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1399,7 +1399,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_dependencies_loaded: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: net-http, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: net-http, not enabled yet)' tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature tests/test_telemetry.py::Test_TelemetryV2: v1.49.1 diff --git a/manifests/java.yml b/manifests/java.yml index 16b919c402f..c5bf487ca60 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4016,7 +4016,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_seq_id: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: spring-boot, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: spring-boot, not enabled yet)' tests/test_telemetry.py::Test_Telemetry::test_status_ok: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index e1afd3ba7db..7aa4b253827 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2192,12 +2192,8 @@ manifest: uds-express4: *ref_3_7_0 tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - - weblog_declaration: - "*": v4.21.0 - nextjs: missing_feature (spawn_child endpoint not implemented) - fastify: missing_feature (spawn_child endpoint not implemented) - express4-typescript: missing_feature (spawn_child endpoint not implemented) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: express4, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: express4, not enabled yet)' tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: "*": irrelevant diff --git a/manifests/php.yml b/manifests/php.yml index 52b011c42e4..24419f51399 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -949,7 +949,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_client_configuration: missing_feature (Telemetry is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: irrelevant (PHP registers 2 telemetry services) tests/test_telemetry.py::Test_Telemetry::test_seq_id: irrelevant (PHP registers 2 telemetry services) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (fork not supported (session ID test requires fork=true)) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: apache-mod, php-fpm, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: apache-mod, php-fpm, not enabled yet)' tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: v0.99.1 tests/test_telemetry.py::Test_TelemetryV2: v0.90 diff --git a/manifests/python.yml b/manifests/python.yml index 6d674d0e610..74f43b29e68 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2090,12 +2090,12 @@ manifest: - declaration: bug (APMAPI-1858) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: - "*": v2.9.0 - tornado: missing_feature (spawn_child endpoint not implemented) - django-poc: missing_feature (spawn_child endpoint not implemented) - django-py3.13: missing_feature (spawn_child endpoint not implemented) - fastapi: missing_feature (spawn_child endpoint not implemented) - python3.12: missing_feature (spawn_child endpoint not implemented) + flask-poc: v4.8.0 + "*": missing_feature (spawn_child endpoint not implemented) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: + - weblog_declaration: + flask-poc: v4.8.0 + "*": missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: "*": irrelevant diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 7f55a1048ef..06a09158ff2 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2111,18 +2111,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: - declaration: missing_feature (app-started not sent) component_version: <1.22.0 - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - - weblog_declaration: - rack: missing_feature (spawn_child endpoint not implemented) - rails42: missing_feature (spawn_child endpoint not implemented) - rails52: missing_feature (spawn_child endpoint not implemented) - rails61: missing_feature (spawn_child endpoint not implemented) - rails80: missing_feature (spawn_child endpoint not implemented) - sinatra14: missing_feature (spawn_child endpoint not implemented) - sinatra22: missing_feature (spawn_child endpoint not implemented) - sinatra32: missing_feature (spawn_child endpoint not implemented) - sinatra41: missing_feature (spawn_child endpoint not implemented) - "*": v2.18.0 + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: rails72, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: rails72, not enabled yet)' tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: # Modified by easy win activation script - weblog_declaration: '*': missing_feature diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 8f45a090485..a16b546f5a4 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -566,13 +566,13 @@ def setup_session_id_headers_across_forks(self): # Allow parent to flush telemetry after child (parent returns after waitpid) time.sleep(3) - def test_session_id_headers_across_forks(self): - """Test session ID headers in telemetry (Stable Service Instance Identifier RFC). + def setup_session_id_headers_across_spawned(self): + """Trigger spawn_child endpoint with exec (fork=false) for session ID header validation.""" + weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": False}) + time.sleep(3) - setup_session_id_headers_across_forks triggers spawn_child, creating a parent-child pair. - Validates: DD-Session-ID matches runtime_id; root has no DD-Root/DD-Parent-Session-ID; - child has DD-Root-Session-ID pointing to root. - """ + def _validate_session_id_headers_across_processes(self) -> None: + """Validate DD-Session-ID, DD-Root-Session-ID, DD-Parent-Session-ID in telemetry.""" telemetry_data = list(interfaces.library.get_telemetry_data(flatten_message_batches=False)) if not telemetry_data: raise ValueError("No telemetry data to validate on") @@ -596,8 +596,8 @@ def test_session_id_headers_across_forks(self): child_sid = sid break - assert child_sid is not None, "No forked process found (DD-Root-Session-ID missing)" - assert root_sid is not None, "No forked process found (DD-Root-Session-ID missing)" + assert child_sid is not None, "No child process found (DD-Root-Session-ID missing)" + assert root_sid is not None, "No child process found (DD-Root-Session-ID missing)" # Validate DD-Session-ID matches runtime_id when present (root may omit header in some tracer versions) for data in telemetry_data: @@ -644,6 +644,14 @@ def test_session_id_headers_across_forks(self): f"DD-Parent-Session-ID '{parent_sid}' must be root or in telemetry" ) + def test_session_id_headers_across_forks(self): + """Test session ID headers in telemetry (fork=true). Stable Service Instance Identifier RFC.""" + self._validate_session_id_headers_across_processes() + + def test_session_id_headers_across_spawned(self): + """Test session ID headers in telemetry (fork=false, exec). Stable Service Instance Identifier RFC.""" + self._validate_session_id_headers_across_processes() + @features.telemetry_app_started_event @scenarios.telemetry_enhanced_config_reporting From 4974b0f30e2939aadd410ac75ba178f15eff4178 Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 16 Mar 2026 17:09:50 -0400 Subject: [PATCH 03/24] enable ruby tests --- manifests/ruby.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 06a09158ff2..99cde86b2f3 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2111,8 +2111,14 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: - declaration: missing_feature (app-started not sent) component_version: <1.22.0 - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: rails72, not enabled yet)' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: rails72, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: + - weblog_declaration: + '*': missing_feature + rails72: v2.32.0 + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: + - weblog_declaration: + '*': missing_feature + rails72: v2.32.0 tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: # Modified by easy win activation script - weblog_declaration: '*': missing_feature From 116bfa7555625a0c78172757ef22e8dd4f00a6a4 Mon Sep 17 00:00:00 2001 From: Munir Date: Tue, 17 Mar 2026 14:14:16 -0400 Subject: [PATCH 04/24] update test --- manifests/python.yml | 4 +- manifests/ruby.yml | 4 +- tests/test_telemetry.py | 90 +++++++++++++---------------------------- 3 files changed, 31 insertions(+), 67 deletions(-) diff --git a/manifests/python.yml b/manifests/python.yml index 74f43b29e68..b817f57ee83 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2090,11 +2090,11 @@ manifest: - declaration: bug (APMAPI-1858) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: - flask-poc: v4.8.0 + flask-poc: v4.7.0 "*": missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: - flask-poc: v4.8.0 + flask-poc: v4.7.0 "*": missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 99cde86b2f3..39843b8a790 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2114,11 +2114,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: '*': missing_feature - rails72: v2.32.0 + rails72: v2.31.0 tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: '*': missing_feature - rails72: v2.32.0 + rails72: v2.31.0 tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: # Modified by easy win activation script - weblog_declaration: '*': missing_feature diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index a16b546f5a4..cbf19218be1 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -569,6 +569,7 @@ def setup_session_id_headers_across_forks(self): def setup_session_id_headers_across_spawned(self): """Trigger spawn_child endpoint with exec (fork=false) for session ID header validation.""" weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": False}) + # Allow parent to flush telemetry after child (parent returns after waitpid) time.sleep(3) def _validate_session_id_headers_across_processes(self) -> None: @@ -576,72 +577,35 @@ def _validate_session_id_headers_across_processes(self) -> None: telemetry_data = list(interfaces.library.get_telemetry_data(flatten_message_batches=False)) if not telemetry_data: raise ValueError("No telemetry data to validate on") + elif len(telemetry_data) == 1: + raise ValueError(f"Only one telemetry data to validate on. Expected at least 2: {telemetry_data}") - # Group by DD-Session-ID (equals runtime_id) - by_session_id: dict[str, list[dict]] = defaultdict(list) - for data in telemetry_data: - sid = get_header(data, "request", "dd-session-id") - if sid: - by_session_id[sid].append(data) - - if not by_session_id: - raise ValueError("No telemetry with DD-Session-ID found") - - # Find child (has DD-Root-Session-ID) - child_sid = None - root_sid = None - for sid, requests in by_session_id.items(): - root_sid = get_header(requests[0], "request", "dd-root-session-id") - if root_sid: - child_sid = sid - break - - assert child_sid is not None, "No child process found (DD-Root-Session-ID missing)" - assert root_sid is not None, "No child process found (DD-Root-Session-ID missing)" + runtime_ids = set[str]() + parent_runtime_ids = set[str]() + root_runtime_ids = set[str]() - # Validate DD-Session-ID matches runtime_id when present (root may omit header in some tracer versions) for data in telemetry_data: - runtime_id = data["request"]["content"].get("runtime_id") - sid = get_header(data, "request", "dd-session-id") - if runtime_id and sid is not None: - assert sid == runtime_id, ( - f"DD-Session-ID '{sid}' != runtime_id '{runtime_id}' in {data['log_filename']}" - ) - - # Root: find by runtime_id (root may not have DD-Session-ID so may not be in by_session_id) - root_requests = by_session_id.get(root_sid, []) - root_data: dict | None = root_requests[0] if root_requests else None - if root_data is None: - root_data = next( - (d for d in telemetry_data if d["request"]["content"].get("runtime_id") == root_sid), - None, - ) - if root_data is None: - runtime_ids = {d["request"]["content"].get("runtime_id") for d in telemetry_data} - session_ids = set(by_session_id.keys()) - raise AssertionError( - f"Root session '{root_sid}' (from child's DD-Root-Session-ID) not in telemetry. " - f"Parent may not have flushed before validation. " - f"runtime_ids in logs: {runtime_ids}, session_ids: {session_ids}" - ) - - # Root: no DD-Root-Session-ID, no DD-Parent-Session-ID - assert get_header(root_data, "request", "dd-root-session-id") is None, "Root must not have DD-Root-Session-ID" - assert get_header(root_data, "request", "dd-parent-session-id") is None, ( - "Root must not have DD-Parent-Session-ID" - ) - - # Child: DD-Root-Session-ID points to root - child_data = by_session_id[child_sid][0] - assert get_header(child_data, "request", "dd-root-session-id") == root_sid, ( - "Child DD-Root-Session-ID must match root" - ) - - # DD-Parent-Session-ID if present must reference root or known session - parent_sid = get_header(child_data, "request", "dd-parent-session-id") - if parent_sid: - assert parent_sid == root_sid or parent_sid in by_session_id, ( - f"DD-Parent-Session-ID '{parent_sid}' must be root or in telemetry" + curr_sid = get_header(data, "request", "dd-session-id") + curr_rid = get_header(data, "request", "dd-root-session-id") + curr_pid = get_header(data, "request", "dd-parent-session-id") + curr_id = data["request"]["content"].get("runtime_id") + + assert curr_sid is not None, f"DD-Session-ID is required in telemetry data: {data}" + assert curr_sid == curr_id, f"DD-Session-ID must match runtime_id: {curr_sid} != {curr_id}" + + runtime_ids.add(curr_id) + if curr_pid is not None: + parent_runtime_ids.add(curr_pid) + if curr_rid is not None: + root_runtime_ids.add(curr_rid) + + assert len(runtime_ids) > 1, f"Expected at least 2 runtime_ids, got {runtime_ids}" + assert len(root_runtime_ids) == 1, f"Expected 1 root runtime_id, got {root_runtime_ids}" + if parent_runtime_ids: + # Optional header: DD-Parent-Session-ID is not required by telemetry intake but is useful for debugging + missing_parent_runtime_ids = parent_runtime_ids.difference(runtime_ids) + assert not missing_parent_runtime_ids, ( + f"Parent runtime_id with no telemetry data: {missing_parent_runtime_ids}" ) def test_session_id_headers_across_forks(self): From ba3eca6f7a96aabc03c2cad93ce270198797d372 Mon Sep 17 00:00:00 2001 From: Munir Date: Wed, 18 Mar 2026 09:49:54 -0400 Subject: [PATCH 05/24] last touch ups --- manifests/python.yml | 4 ++-- manifests/ruby.yml | 4 ++-- tests/test_telemetry.py | 10 +++------- utils/interfaces/_library/core.py | 16 ++++++++++++++++ 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifests/python.yml b/manifests/python.yml index b817f57ee83..30fee0b16c6 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2090,11 +2090,11 @@ manifest: - declaration: bug (APMAPI-1858) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: - flask-poc: v4.7.0 + flask-poc: missing_feature (not implemented yet) "*": missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: - flask-poc: v4.7.0 + flask-poc: missing_feature (not implemented yet) "*": missing_feature (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 39843b8a790..870168ce667 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2114,11 +2114,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: '*': missing_feature - rails72: v2.31.0 + rails72: missing_feature (not implemented yet) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: '*': missing_feature - rails72: v2.31.0 + rails72: missing_feature (not implemented yet) tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: # Modified by easy win activation script - weblog_declaration: '*': missing_feature diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index cbf19218be1..0920556eb99 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -563,22 +563,18 @@ def test_app_product_change(self): def setup_session_id_headers_across_forks(self): """Trigger spawn_child endpoint to create a fork tree for session ID header validation.""" weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": True}) - # Allow parent to flush telemetry after child (parent returns after waitpid) - time.sleep(3) def setup_session_id_headers_across_spawned(self): """Trigger spawn_child endpoint with exec (fork=false) for session ID header validation.""" weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": False}) - # Allow parent to flush telemetry after child (parent returns after waitpid) - time.sleep(3) def _validate_session_id_headers_across_processes(self) -> None: """Validate DD-Session-ID, DD-Root-Session-ID, DD-Parent-Session-ID in telemetry.""" - telemetry_data = list(interfaces.library.get_telemetry_data(flatten_message_batches=False)) + # ignore metrics and log events which can be generated by lib-datadog. These events + # contain runtime/session_ids which do not map to telemetry generated by the tracer. + telemetry_data = list(interfaces.library.get_lifecycle_events()) if not telemetry_data: raise ValueError("No telemetry data to validate on") - elif len(telemetry_data) == 1: - raise ValueError(f"Only one telemetry data to validate on. Expected at least 2: {telemetry_data}") runtime_ids = set[str]() parent_runtime_ids = set[str]() diff --git a/utils/interfaces/_library/core.py b/utils/interfaces/_library/core.py index 23bbbdb33eb..b08d9a43f9d 100644 --- a/utils/interfaces/_library/core.py +++ b/utils/interfaces/_library/core.py @@ -22,6 +22,15 @@ from utils._weblog import HttpResponse, GrpcResponse from utils.interfaces._misc_validators import HeadersPresenceValidator +LIFECYCLE_EVENTS = [ + "app-started", + "app-closing", + "app-integrations-change", + "app-dependencies-loaded", + "app-client-configuration-change", + "app-product-change", +] + class LibraryInterfaceValidator(ProxyBasedInterfaceValidator): """Validate library/agent interface""" @@ -215,6 +224,13 @@ def get_telemetry_data(self, *, flatten_message_batches: bool = True): else: yield data + def get_lifecycle_events(self): + for data in self.get_telemetry_data(flatten_message_batches=True): + content = data["request"]["content"] + if content.get("request_type") not in LIFECYCLE_EVENTS: + continue + yield data + def get_telemetry_configurations(self) -> list[dict]: """Extract and sort configuration entries from telemetry events.""" configurations = [] From 1b42dfdc27223ce688d34fb7f7d79a6f87a4a933 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Wed, 18 Mar 2026 09:52:58 -0400 Subject: [PATCH 06/24] Apply suggestions from code review Co-authored-by: Munir Abdinur --- docs/weblog/end-to-end_weblog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/weblog/end-to-end_weblog.md b/docs/weblog/end-to-end_weblog.md index cf98d4d9757..94f15d51c14 100644 --- a/docs/weblog/end-to-end_weblog.md +++ b/docs/weblog/end-to-end_weblog.md @@ -902,6 +902,7 @@ This endpoint is OPTIONAL and not related to any test, but to the testing proces ### GET /spawn_child This endpoint is used for telemetry session ID header tests (Stable Service Instance Identifier RFC). It must fork or exec a child process, pass in the required arguments, wait for the child, and return a response. Used to validate `DD-Session-ID`, `DD-Root-Session-ID`, and `DD-Parent-Session-ID` headers in instrumentation telemetry across process forks. +RFC: https://docs.google.com/document/d/1ECKj9_NnwaKYtFqm3p3Rlpicx5d-OQcdj9kI2jvRqVU/edit?tab=t.0#heading=h.ojliy5oytqgg Required query parameters: From bc3d7a5a3e3d16f2a98d7216bc568c5bb704876d Mon Sep 17 00:00:00 2001 From: Munir Date: Wed, 18 Mar 2026 11:16:21 -0400 Subject: [PATCH 07/24] add better comments --- tests/test_telemetry.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index b2276dd232f..af5e4f6b5cb 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -577,9 +577,15 @@ def setup_session_id_headers_across_spawned(self): weblog.get("/spawn_child", params={"sleep": 2, "crash": False, "fork": False}) def _validate_session_id_headers_across_processes(self) -> None: - """Validate DD-Session-ID, DD-Root-Session-ID, DD-Parent-Session-ID in telemetry.""" - # ignore metrics and log events which can be generated by lib-datadog. These events - # contain runtime/session_ids which do not map to telemetry generated by the tracer. + """Validate DD-Session-ID, DD-Root-Session-ID, DD-Parent-Session-ID in telemetry. + + Stable Service Instance Identifier RFC: each app instance has one root runtime_id. + DD-Session-ID (instance id) must equal runtime_id. When only DD-Session-ID is sent + (no DD-Root-Session-ID), the process is treated as the root. This test confirms + at least two different runtimes are captured (parent and child from spawn_child). + """ + # Use lifecycle events only; metrics and log events from lib-datadog can contain + # runtime/session_ids that do not map to tracer-generated telemetry. telemetry_data = list(interfaces.library.get_lifecycle_events()) if not telemetry_data: raise ValueError("No telemetry data to validate on") @@ -589,11 +595,13 @@ def _validate_session_id_headers_across_processes(self) -> None: root_runtime_ids = set[str]() for data in telemetry_data: + # Headers are not case sensitive curr_sid = get_header(data, "request", "dd-session-id") curr_rid = get_header(data, "request", "dd-root-session-id") curr_pid = get_header(data, "request", "dd-parent-session-id") curr_id = data["request"]["content"].get("runtime_id") + # Instance id (DD-Session-ID) must be present in all lifecycle events and equal to runtime_id assert curr_sid is not None, f"DD-Session-ID is required in telemetry data: {data}" assert curr_sid == curr_id, f"DD-Session-ID must match runtime_id: {curr_sid} != {curr_id}" @@ -603,10 +611,12 @@ def _validate_session_id_headers_across_processes(self) -> None: if curr_rid is not None: root_runtime_ids.add(curr_rid) + # At least two runtimes: parent (root) and child from spawn_child assert len(runtime_ids) > 1, f"Expected at least 2 runtime_ids, got {runtime_ids}" + # One root per app instance: all children share the same DD-Root-Session-ID assert len(root_runtime_ids) == 1, f"Expected 1 root runtime_id, got {root_runtime_ids}" if parent_runtime_ids: - # Optional header: DD-Parent-Session-ID is not required by telemetry intake but is useful for debugging + # DD-Parent-Session-ID is optional but must reference a known runtime if present missing_parent_runtime_ids = parent_runtime_ids.difference(runtime_ids) assert not missing_parent_runtime_ids, ( f"Parent runtime_id with no telemetry data: {missing_parent_runtime_ids}" From f3c55eaff5811bd027786a637a59e952355dbb99 Mon Sep 17 00:00:00 2001 From: Munir Date: Thu, 19 Mar 2026 14:47:31 -0500 Subject: [PATCH 08/24] fix manifest skips --- manifests/cpp.yml | 2 ++ manifests/cpp_httpd.yml | 2 ++ manifests/cpp_kong.yml | 2 ++ manifests/cpp_nginx.yml | 2 ++ manifests/dotnet.yml | 4 ++-- manifests/golang.yml | 4 ++-- manifests/java.yml | 4 ++-- manifests/nodejs.yml | 4 ++-- manifests/php.yml | 2 ++ 9 files changed, 18 insertions(+), 8 deletions(-) diff --git a/manifests/cpp.yml b/manifests/cpp.yml index f9bc4141c20..1b1529e6c1b 100644 --- a/manifests/cpp.yml +++ b/manifests/cpp.yml @@ -309,5 +309,7 @@ manifest: tests/test_library_logs.py::Test_NoExceptions::test_dotnet: irrelevant (only for .NET) tests/test_library_logs.py::Test_NoExceptions::test_java_logs: irrelevant (only for Java) tests/test_library_logs.py::Test_NoExceptions::test_java_telemetry_logs: irrelevant (only for Java) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_required_headers: missing_feature diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 29b5d4058db..df06989afa4 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -196,6 +196,8 @@ manifest: component_version: <1.0.3 tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.0.3' # Modified by easy win activation script tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting::test_telemetry_enhanced_config_reporting_precedence: missing_feature # Created by easy win activation script tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/manifests/cpp_kong.yml b/manifests/cpp_kong.yml index 07c1d7bdfef..3d1372ca2fa 100644 --- a/manifests/cpp_kong.yml +++ b/manifests/cpp_kong.yml @@ -36,5 +36,7 @@ manifest: tests/test_span_events.py: missing_feature tests/test_standard_tags.py: irrelevant tests/test_telemetry.py: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_v1_payloads.py: missing_feature diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index 2204cbb4ae8..24d3ef3cf92 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -420,6 +420,8 @@ manifest: component_version: <1.12.0 tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_proxy_forwarding: missing_feature # Created by easy win activation script + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_Telemetry::test_telemetry_proxy_enrichment: missing_feature # Created by easy win activation script tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.12.0' # Modified by easy win activation script diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 0f4625bc0b9..4a01c604223 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1178,8 +1178,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_is_first_message: # Easy win for poc, uds and version 3.36.0 - declaration: bug (APMAPI-728) component_version: '>=3.4.0' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: poc, not enabled yet)' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: poc, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: poc, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: poc, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0 tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/manifests/golang.yml b/manifests/golang.yml index 744c47e8717..3a5c8d61622 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1496,8 +1496,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_dependencies_loaded: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: net-http, not enabled yet)' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: net-http, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: net-http, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: net-http, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index 6345fbc21a2..c6ab2abeb69 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4299,8 +4299,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_seq_id: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: spring-boot, not enabled yet)' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: spring-boot, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: spring-boot, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: spring-boot, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_status_ok: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index cda71e69596..b02307d9747 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2356,8 +2356,8 @@ manifest: uds-express4: *ref_3_7_0 tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: 'missing_feature (spawn_child: express4, not enabled yet)' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: 'missing_feature (spawn_child: express4, not enabled yet)' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: express4, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: express4, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: diff --git a/manifests/php.yml b/manifests/php.yml index 7162457e989..ee8a1a08af6 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -1052,6 +1052,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_client_configuration: missing_feature (Telemetry is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: irrelevant (PHP registers 2 telemetry services) tests/test_telemetry.py::Test_Telemetry::test_seq_id: irrelevant (PHP registers 2 telemetry services) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: v0.99.1 From 8143d57f3e6d0e9068577ec120789949b4a94aab Mon Sep 17 00:00:00 2001 From: Munir Date: Thu, 19 Mar 2026 15:16:10 -0500 Subject: [PATCH 09/24] fix manifest files and nodejs build --- manifests/dotnet.yml | 4 ++-- manifests/golang.yml | 4 ++-- manifests/java.yml | 4 ++-- manifests/nodejs.yml | 10 ++++++++-- utils/build/docker/nodejs/install_ddtrace.sh | 4 ++-- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 4a01c604223..40e59ec641e 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1178,8 +1178,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_is_first_message: # Easy win for poc, uds and version 3.36.0 - declaration: bug (APMAPI-728) component_version: '>=3.4.0' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: poc, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: poc, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child poc, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child poc, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0 tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/manifests/golang.yml b/manifests/golang.yml index 3a5c8d61622..b489ae6dced 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1496,8 +1496,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_dependencies_loaded: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: net-http, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: net-http, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child net-http, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child net-http, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index c6ab2abeb69..03640438f90 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4299,8 +4299,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_seq_id: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: spring-boot, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: spring-boot, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child spring-boot, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child spring-boot, not enabled yet) tests/test_telemetry.py::Test_Telemetry::test_status_ok: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index b02307d9747..f7b9cbd58ad 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2356,8 +2356,14 @@ manifest: uds-express4: *ref_3_7_0 tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child: express4, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child: express4, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: + - weblog_declaration: + "*": missing_feature (spawn_child endpoint not implemented) + express4: *ref_5_90_0 + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: + - weblog_declaration: + "*": missing_feature (spawn_child endpoint not implemented) + express4: *ref_5_90_0 tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: diff --git a/utils/build/docker/nodejs/install_ddtrace.sh b/utils/build/docker/nodejs/install_ddtrace.sh index 057f0a0b373..dc5ce480717 100755 --- a/utils/build/docker/nodejs/install_ddtrace.sh +++ b/utils/build/docker/nodejs/install_ddtrace.sh @@ -21,12 +21,12 @@ if [ -e /binaries/nodejs-load-from-local ]; then echo "using local version that will be mounted at runtime" else if [ -e /binaries/nodejs-load-from-npm ]; then - target=$( Date: Thu, 19 Mar 2026 15:28:51 -0500 Subject: [PATCH 10/24] clean up nodejs test case --- manifests/cpp_httpd.yml | 2 +- manifests/nodejs.yml | 4 ++-- tests/test_telemetry.py | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index df06989afa4..322a3a03f00 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -195,9 +195,9 @@ manifest: - declaration: missing_feature (DD_TELEMETRY_HEARTBEAT_INTERVAL not supported) component_version: <1.0.3 tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.0.3' # Modified by easy win activation script tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting::test_telemetry_enhanced_config_reporting_precedence: missing_feature # Created by easy win activation script tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index f7b9cbd58ad..7e181605cd3 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -2357,11 +2357,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - - weblog_declaration: + - weblog_declaration: "*": missing_feature (spawn_child endpoint not implemented) express4: *ref_5_90_0 tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - - weblog_declaration: + - weblog_declaration: "*": missing_feature (spawn_child endpoint not implemented) express4: *ref_5_90_0 tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index af5e4f6b5cb..28bdf76df6e 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -610,6 +610,9 @@ def _validate_session_id_headers_across_processes(self) -> None: parent_runtime_ids.add(curr_pid) if curr_rid is not None: root_runtime_ids.add(curr_rid) + else: + # If dd-root-session-id is not set, dd-session-id is treated as root + root_runtime_ids.add(curr_id) # At least two runtimes: parent (root) and child from spawn_child assert len(runtime_ids) > 1, f"Expected at least 2 runtime_ids, got {runtime_ids}" From a0b2978264522b47ce5a724518c70011a36faf09 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 10:21:06 -0400 Subject: [PATCH 11/24] enable nodejs system test and fix test app for node --- manifests/nodejs.yml | 5 +++-- utils/build/docker/nodejs/express/fork_child.js | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 7e181605cd3..70dbc4729a9 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -84,6 +84,7 @@ refs: - &ref_5_87_0 '>=5.87.0' # Debugger: Capture expressions support - &ref_5_89_0 '>=5.89.0' - &ref_5_90_0 '>=5.90.0' + - &ref_5_93_0 '>=5.92.0' manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: - weblog_declaration: @@ -2359,11 +2360,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: "*": missing_feature (spawn_child endpoint not implemented) - express4: *ref_5_90_0 + express4: *ref_5_93_0 tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: "*": missing_feature (spawn_child endpoint not implemented) - express4: *ref_5_90_0 + express4: *ref_5_93_0 tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: - weblog_declaration: diff --git a/utils/build/docker/nodejs/express/fork_child.js b/utils/build/docker/nodejs/express/fork_child.js index ef612416c0f..2543235155c 100644 --- a/utils/build/docker/nodejs/express/fork_child.js +++ b/utils/build/docker/nodejs/express/fork_child.js @@ -1,5 +1,6 @@ #!/usr/bin/env node // Child process for spawn_child endpoint. Args: sleep (seconds), crash (true|false). +require('dd-trace').init() const sleepSec = parseInt(process.argv[2] || '2', 10) * 1000 const crash = process.argv[3] === 'true' setTimeout(() => { From f307e4b528b5653e862fffdc924e8b681f80738c Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 10:23:21 -0400 Subject: [PATCH 12/24] fix version --- manifests/nodejs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml index 70dbc4729a9..073d09f914d 100644 --- a/manifests/nodejs.yml +++ b/manifests/nodejs.yml @@ -84,7 +84,7 @@ refs: - &ref_5_87_0 '>=5.87.0' # Debugger: Capture expressions support - &ref_5_89_0 '>=5.89.0' - &ref_5_90_0 '>=5.90.0' - - &ref_5_93_0 '>=5.92.0' + - &ref_5_93_0 '>=5.93.0' manifest: tests/ai_guard/test_ai_guard_sdk.py::Test_ContentParts: - weblog_declaration: From cd21b0c0aa49844e5a167cb6f00ca10431c8ee5f Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 11:48:11 -0400 Subject: [PATCH 13/24] fix(golang): add proper child process for session ID header tests Add a standalone child binary that initializes dd-trace-go and emits telemetry, replacing the previous shell-based approach. Simplify spawn_child.go to use plain os/exec since the SDK now auto-propagates DD_ROOT_GO_SESSION_ID. Update Dockerfile to build/copy the child binary. Mark fork test as irrelevant for Go (no fork support). Co-Authored-By: Claude Opus 4.6 --- manifests/golang.yml | 7 +++- .../golang/app/_shared/common/spawn_child.go | 18 ++++---- utils/build/docker/golang/app/child/main.go | 42 +++++++++++++++++++ utils/build/docker/golang/net-http.Dockerfile | 4 +- 4 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 utils/build/docker/golang/app/child/main.go diff --git a/manifests/golang.yml b/manifests/golang.yml index b489ae6dced..15bfd05ffbd 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1496,8 +1496,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_api_still_v1: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_dependencies_loaded: irrelevant tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child net-http, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child net-http, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (Go does not support fork; use test_session_id_headers_across_spawned instead) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: + - weblog_declaration: + '*': missing_feature + net-http: '>=2.5.0' tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/utils/build/docker/golang/app/_shared/common/spawn_child.go b/utils/build/docker/golang/app/_shared/common/spawn_child.go index abd0d6ac17d..61b00f80c9e 100644 --- a/utils/build/docker/golang/app/_shared/common/spawn_child.go +++ b/utils/build/docker/golang/app/_shared/common/spawn_child.go @@ -3,13 +3,17 @@ package common import ( "fmt" "net/http" + "os" "os/exec" "strconv" "strings" ) // SpawnChild handles GET /spawn_child for telemetry session ID header tests. -// Go does not support fork; returns 400 when fork=true. Otherwise uses exec. +// Go does not support fork; returns 400 when fork=true. Otherwise spawns a +// child Go process that initializes dd-trace-go and emits its own telemetry. +// The SDK is responsible for propagating DD_ROOT_GO_SESSION_ID via the +// process environment so that child processes inherit the root session ID. func SpawnChild(w http.ResponseWriter, r *http.Request) { sleepStr := r.URL.Query().Get("sleep") crashStr := strings.ToLower(r.URL.Query().Get("crash")) @@ -33,14 +37,10 @@ func SpawnChild(w http.ResponseWriter, r *http.Request) { return } - crash := crashStr == "true" - script := fmt.Sprintf("sleep %d", sleep) - if crash { - script += " && kill -SEGV $$" - } else { - script += " && exit 0" - } - cmd := exec.Command("sh", "-c", script) + cmd := exec.Command("/app/child", sleepStr, crashStr) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + _ = cmd.Run() status := 0 if cmd.ProcessState != nil { diff --git a/utils/build/docker/golang/app/child/main.go b/utils/build/docker/golang/app/child/main.go new file mode 100644 index 00000000000..1734a5c0f91 --- /dev/null +++ b/utils/build/docker/golang/app/child/main.go @@ -0,0 +1,42 @@ +// Child process for /spawn_child endpoint. Initializes dd-trace-go so that +// telemetry (app-started, heartbeats, app-closed) is emitted with its own +// runtime_id. If DD_ROOT_GO_SESSION_ID is set in the environment (injected by +// the parent), the SDK uses it for the DD-Root-Session-ID header. +// +// Usage: child +package main + +import ( + "fmt" + "os" + "strconv" + "syscall" + "time" + + "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" +) + +func main() { + if len(os.Args) < 3 { + fmt.Fprintf(os.Stderr, "usage: child \n") + os.Exit(1) + } + + sleep, err := strconv.Atoi(os.Args[1]) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid sleep: %v\n", err) + os.Exit(1) + } + crash := os.Args[2] == "true" + + tracer.Start() + + time.Sleep(time.Duration(sleep) * time.Second) + + if crash { + tracer.Stop() + syscall.Kill(syscall.Getpid(), syscall.SIGSEGV) + } + + tracer.Stop() +} diff --git a/utils/build/docker/golang/net-http.Dockerfile b/utils/build/docker/golang/net-http.Dockerfile index ec03f7ad6bf..35bcd3ed2e3 100644 --- a/utils/build/docker/golang/net-http.Dockerfile +++ b/utils/build/docker/golang/net-http.Dockerfile @@ -18,7 +18,8 @@ RUN --mount=type=cache,target=${GOMODCACHE} --mount=type=bind,source=binaries,target=/binaries \ go mod download && go mod verify && \ /utils/install_ddtrace.sh && \ - go build -v -tags=appsec -o=./weblog ./net-http + go build -v -tags=appsec -o=./weblog ./net-http && \ + go build -v -o=./child ./child # ============================================================================== @@ -27,6 +28,7 @@ FROM golang:1.25-alpine RUN apk add --no-cache curl bash gcc musl-dev COPY --from=build /app/weblog /app/weblog +COPY --from=build /app/child /app/child COPY --from=build /app/SYSTEM_TESTS_LIBRARY_VERSION /app/SYSTEM_TESTS_LIBRARY_VERSION WORKDIR /app From b590862f0d8f8209d65ca9dcc60e1bccd3790ffb Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 11:52:25 -0400 Subject: [PATCH 14/24] refactor: inline child process logic into weblog binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of a separate child binary, re-exec the weblog itself with DD_SYSTEM_TEST_CHILD_SLEEP env var. RunAsChildIfRequested() in common handles child mode — starts tracer, sleeps, stops, exits. This avoids a separate build target and Dockerfile changes. Co-Authored-By: Claude Opus 4.6 --- .../golang/app/_shared/common/spawn_child.go | 39 ++++++++++++++--- utils/build/docker/golang/app/child/main.go | 42 ------------------- .../build/docker/golang/app/net-http/main.go | 2 + utils/build/docker/golang/net-http.Dockerfile | 4 +- 4 files changed, 37 insertions(+), 50 deletions(-) delete mode 100644 utils/build/docker/golang/app/child/main.go diff --git a/utils/build/docker/golang/app/_shared/common/spawn_child.go b/utils/build/docker/golang/app/_shared/common/spawn_child.go index 61b00f80c9e..6bc99dee219 100644 --- a/utils/build/docker/golang/app/_shared/common/spawn_child.go +++ b/utils/build/docker/golang/app/_shared/common/spawn_child.go @@ -7,13 +7,38 @@ import ( "os/exec" "strconv" "strings" + "syscall" + "time" + + "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" ) +// RunAsChildIfRequested checks if the process was re-exec'd in child mode. +// If so, it initializes the tracer, sleeps, optionally crashes, then exits. +// Call this at the top of main() before any other initialization. +func RunAsChildIfRequested() { + sleepStr := os.Getenv("DD_SYSTEM_TEST_CHILD_SLEEP") + if sleepStr == "" { + return + } + sleep, _ := strconv.Atoi(sleepStr) + crash := os.Getenv("DD_SYSTEM_TEST_CHILD_CRASH") == "true" + + tracer.Start() + time.Sleep(time.Duration(sleep) * time.Second) + tracer.Stop() + + if crash { + syscall.Kill(syscall.Getpid(), syscall.SIGSEGV) + } + os.Exit(0) +} + // SpawnChild handles GET /spawn_child for telemetry session ID header tests. -// Go does not support fork; returns 400 when fork=true. Otherwise spawns a -// child Go process that initializes dd-trace-go and emits its own telemetry. -// The SDK is responsible for propagating DD_ROOT_GO_SESSION_ID via the -// process environment so that child processes inherit the root session ID. +// Go does not support fork; returns 400 when fork=true. Otherwise re-execs +// the current binary in child mode, which initializes dd-trace-go and emits +// its own telemetry. The SDK propagates DD_ROOT_GO_SESSION_ID via the process +// environment so that child processes inherit the root session ID automatically. func SpawnChild(w http.ResponseWriter, r *http.Request) { sleepStr := r.URL.Query().Get("sleep") crashStr := strings.ToLower(r.URL.Query().Get("crash")) @@ -37,7 +62,11 @@ func SpawnChild(w http.ResponseWriter, r *http.Request) { return } - cmd := exec.Command("/app/child", sleepStr, crashStr) + cmd := exec.Command(os.Args[0]) + cmd.Env = append(os.Environ(), + "DD_SYSTEM_TEST_CHILD_SLEEP="+sleepStr, + "DD_SYSTEM_TEST_CHILD_CRASH="+crashStr, + ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/utils/build/docker/golang/app/child/main.go b/utils/build/docker/golang/app/child/main.go deleted file mode 100644 index 1734a5c0f91..00000000000 --- a/utils/build/docker/golang/app/child/main.go +++ /dev/null @@ -1,42 +0,0 @@ -// Child process for /spawn_child endpoint. Initializes dd-trace-go so that -// telemetry (app-started, heartbeats, app-closed) is emitted with its own -// runtime_id. If DD_ROOT_GO_SESSION_ID is set in the environment (injected by -// the parent), the SDK uses it for the DD-Root-Session-ID header. -// -// Usage: child -package main - -import ( - "fmt" - "os" - "strconv" - "syscall" - "time" - - "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" -) - -func main() { - if len(os.Args) < 3 { - fmt.Fprintf(os.Stderr, "usage: child \n") - os.Exit(1) - } - - sleep, err := strconv.Atoi(os.Args[1]) - if err != nil { - fmt.Fprintf(os.Stderr, "invalid sleep: %v\n", err) - os.Exit(1) - } - crash := os.Args[2] == "true" - - tracer.Start() - - time.Sleep(time.Duration(sleep) * time.Second) - - if crash { - tracer.Stop() - syscall.Kill(syscall.Getpid(), syscall.SIGSEGV) - } - - tracer.Stop() -} diff --git a/utils/build/docker/golang/app/net-http/main.go b/utils/build/docker/golang/app/net-http/main.go index 31548e446b9..ce8038f2100 100644 --- a/utils/build/docker/golang/app/net-http/main.go +++ b/utils/build/docker/golang/app/net-http/main.go @@ -42,6 +42,8 @@ import ( ) func main() { + common.RunAsChildIfRequested() + logrus.SetFormatter(&logrus.JSONFormatter{}) logrus.SetOutput(os.Stdout) logrus.SetLevel(logrus.DebugLevel) diff --git a/utils/build/docker/golang/net-http.Dockerfile b/utils/build/docker/golang/net-http.Dockerfile index 35bcd3ed2e3..ec03f7ad6bf 100644 --- a/utils/build/docker/golang/net-http.Dockerfile +++ b/utils/build/docker/golang/net-http.Dockerfile @@ -18,8 +18,7 @@ RUN --mount=type=cache,target=${GOMODCACHE} --mount=type=bind,source=binaries,target=/binaries \ go mod download && go mod verify && \ /utils/install_ddtrace.sh && \ - go build -v -tags=appsec -o=./weblog ./net-http && \ - go build -v -o=./child ./child + go build -v -tags=appsec -o=./weblog ./net-http # ============================================================================== @@ -28,7 +27,6 @@ FROM golang:1.25-alpine RUN apk add --no-cache curl bash gcc musl-dev COPY --from=build /app/weblog /app/weblog -COPY --from=build /app/child /app/child COPY --from=build /app/SYSTEM_TESTS_LIBRARY_VERSION /app/SYSTEM_TESTS_LIBRARY_VERSION WORKDIR /app From d51f23d8072a5ab72bf6dc1db629a263bff465e0 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 12:00:11 -0400 Subject: [PATCH 15/24] chore: rename DD_ROOT_GO_SESSION_ID to _DD_ROOT_GO_SESSION_ID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match the SDK rename — underscore prefix signifies internal env var. Co-Authored-By: Claude Opus 4.6 --- utils/build/docker/golang/app/_shared/common/spawn_child.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build/docker/golang/app/_shared/common/spawn_child.go b/utils/build/docker/golang/app/_shared/common/spawn_child.go index 6bc99dee219..6c167073733 100644 --- a/utils/build/docker/golang/app/_shared/common/spawn_child.go +++ b/utils/build/docker/golang/app/_shared/common/spawn_child.go @@ -37,7 +37,7 @@ func RunAsChildIfRequested() { // SpawnChild handles GET /spawn_child for telemetry session ID header tests. // Go does not support fork; returns 400 when fork=true. Otherwise re-execs // the current binary in child mode, which initializes dd-trace-go and emits -// its own telemetry. The SDK propagates DD_ROOT_GO_SESSION_ID via the process +// its own telemetry. The SDK propagates _DD_ROOT_GO_SESSION_ID via the process // environment so that child processes inherit the root session ID automatically. func SpawnChild(w http.ResponseWriter, r *http.Request) { sleepStr := r.URL.Query().Get("sleep") From d0b1214697274623aece430d4e44cf3eea7356c9 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 12:06:19 -0400 Subject: [PATCH 16/24] update go manifest version to next release --- manifests/golang.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/golang.yml b/manifests/golang.yml index 15bfd05ffbd..0dc253db6d9 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1500,7 +1500,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: '*': missing_feature - net-http: '>=2.5.0' + net-http: '>=2.8.0' tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature From 74a841ef04c3a9f047e5464030b2b5649db18810 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 13:15:13 -0400 Subject: [PATCH 17/24] enable system tests for java --- manifests/java.yml | 7 +++++-- .../system_tests/springboot/App.java | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/manifests/java.yml b/manifests/java.yml index 03640438f90..1d4a3904d25 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4299,8 +4299,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_seq_id: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child spring-boot, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child spring-boot, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (Java does not support fork) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: + - weblog_declaration: + "*": irrelevant + spring-boot: v1.60.3 tests/test_telemetry.py::Test_Telemetry::test_status_ok: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) diff --git a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java index f40fd038a5c..d5e61813a90 100644 --- a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java +++ b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/App.java @@ -340,8 +340,12 @@ ResponseEntity spawnChild( } try { ProcessBuilder pb = new ProcessBuilder( - "sh", "-c", - String.format("sleep %d && %s", sleep, crash.equalsIgnoreCase("true") ? "kill -SEGV $$" : "exit 0")); + "java", "-Xmx128m", + "-javaagent:/app/dd-java-agent.jar", + "-jar", "/app/app.jar"); + pb.environment().put("DD_SYSTEM_TEST_CHILD_SLEEP", String.valueOf(sleep)); + pb.environment().put("DD_SYSTEM_TEST_CHILD_CRASH", crash.toLowerCase()); + pb.inheritIO(); Process p = pb.start(); int exitCode = p.waitFor(); return ResponseEntity.ok("Process " + p.pid() + " has exited with code " + exitCode); @@ -1499,7 +1503,16 @@ private void setRootSpanTag(final String key, final String value) { } } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { + String childSleep = System.getenv("DD_SYSTEM_TEST_CHILD_SLEEP"); + if (childSleep != null) { + int sleep = Integer.parseInt(childSleep); + Thread.sleep(sleep * 1000L); + if ("true".equals(System.getenv("DD_SYSTEM_TEST_CHILD_CRASH"))) { + Runtime.getRuntime().halt(139); + } + return; + } SpringApplication.run(App.class, args); } From 548a29c6e81a61499c32f0065cb7f97957d2180f Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Fri, 20 Mar 2026 13:22:28 -0400 Subject: [PATCH 18/24] fix(dotnet): fix spawn_child to re-exec traced process and update manifest - SpawnChildEndpoint now re-execs the dotnet weblog (with CLR profiler) instead of spawning a bare shell for the exec path - Returns 400 for fork=true since .NET doesn't support fork - Manifest: fork test marked irrelevant, spawn test enabled for >=v3.4.0 Co-Authored-By: Claude Opus 4.6 --- manifests/dotnet.yml | 4 +- .../weblog/Endpoints/SpawnChildEndpoint.cs | 52 +++++++------------ 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 40e59ec641e..eca098ac6bd 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1178,8 +1178,8 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_app_started_is_first_message: # Easy win for poc, uds and version 3.36.0 - declaration: bug (APMAPI-728) component_version: '>=3.4.0' - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature (spawn_child poc, not enabled yet) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature (spawn_child poc, not enabled yet) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (.NET does not support fork) + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: v3.4.0 tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0 tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs b/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs index 1ce5a40cb47..bb0d37de6bd 100644 --- a/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs +++ b/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs @@ -37,42 +37,30 @@ public void Register(Microsoft.AspNetCore.Routing.IEndpointRouteBuilder routeBui return; } - var crash = crashStr == "true"; - Process process; - if (forkStr == "true") { - // Fork path: spawn same process with env vars (lib-injection fork_and_crash pattern) - var cmdArgs = Environment.GetCommandLineArgs(); - var args = cmdArgs.Length > 1 ? string.Join(" ", cmdArgs.Skip(1)) : "app.dll"; - var startInfo = new ProcessStartInfo - { - FileName = Environment.ProcessPath ?? "/usr/share/dotnet/dotnet", - Arguments = args, - WorkingDirectory = Environment.CurrentDirectory, - }; - startInfo.Environment["SPAWN_CHILD_FORKED"] = "1"; - startInfo.Environment["SPAWN_CHILD_SLEEP"] = sleep.ToString(); - startInfo.Environment["SPAWN_CHILD_CRASH"] = crash ? "1" : "0"; - - process = Process.Start(startInfo); + context.Response.StatusCode = 400; + await context.Response.WriteAsync("fork not supported in .NET"); + return; } - else + + var crash = crashStr == "true"; + + // Re-exec the weblog binary as a child process. The CLR profiler + // auto-attaches dd-trace-dotnet, so the child emits its own telemetry. + var cmdArgs = Environment.GetCommandLineArgs(); + var args = cmdArgs.Length > 1 ? string.Join(" ", cmdArgs.Skip(1)) : "app.dll"; + var startInfo = new ProcessStartInfo { - // Exec path: shell script - var script = crash - ? $"sleep {sleep} && kill -SEGV $$$$" - : $"sleep {sleep} && exit 0"; - var startInfo = new ProcessStartInfo - { - FileName = "/bin/sh", - Arguments = $"-c \"{script}\"", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - }; - process = Process.Start(startInfo); - } + FileName = Environment.ProcessPath ?? "/usr/share/dotnet/dotnet", + Arguments = args, + WorkingDirectory = Environment.CurrentDirectory, + }; + startInfo.Environment["SPAWN_CHILD_FORKED"] = "1"; + startInfo.Environment["SPAWN_CHILD_SLEEP"] = sleep.ToString(); + startInfo.Environment["SPAWN_CHILD_CRASH"] = crash ? "1" : "0"; + + var process = Process.Start(startInfo); if (process == null) { From 5bf3eb767deccaf273eceedf3d07cbec28d2a791 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Thu, 26 Mar 2026 14:42:13 -0400 Subject: [PATCH 19/24] enable dotnet system tests --- manifests/dotnet.yml | 2 +- .../build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs | 2 ++ utils/build/docker/dotnet/weblog/Program.cs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index de1c7007e47..8e8f5d86e19 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -1182,7 +1182,7 @@ manifest: - declaration: bug (APMAPI-728) component_version: '>=3.4.0' tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (.NET does not support fork) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: v3.4.0 + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: v3.41.0 tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: v3.25.0 tests/test_telemetry.py::Test_TelemetrySCAEnvVar: missing_feature diff --git a/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs b/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs index bb0d37de6bd..afdd7d38ec8 100644 --- a/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs +++ b/utils/build/docker/dotnet/weblog/Endpoints/SpawnChildEndpoint.cs @@ -1,4 +1,6 @@ +using System; using System.Diagnostics; +using System.Linq; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; diff --git a/utils/build/docker/dotnet/weblog/Program.cs b/utils/build/docker/dotnet/weblog/Program.cs index b7308651313..684ffb681ab 100644 --- a/utils/build/docker/dotnet/weblog/Program.cs +++ b/utils/build/docker/dotnet/weblog/Program.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Http; From 75eb41d59f15e26952bcf6ace2c26edf2b392288 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Fri, 3 Apr 2026 17:15:54 -0400 Subject: [PATCH 20/24] clean up versions Co-authored-by: Munir Abdinur --- manifests/golang.yml | 2 +- manifests/java.yml | 2 +- manifests/python.yml | 2 +- manifests/ruby.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manifests/golang.yml b/manifests/golang.yml index 767b3a582ac..7e8801812bf 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1502,7 +1502,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: irrelevant (Go does not support fork; use test_session_id_headers_across_spawned instead) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: - '*': missing_feature + '*': irrelevant net-http: '>=2.8.0' tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: missing_feature diff --git a/manifests/java.yml b/manifests/java.yml index 9b024918bbd..f2415160216 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -4332,7 +4332,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: "*": irrelevant - spring-boot: v1.60.3 + spring-boot: '>=1.61.0' tests/test_telemetry.py::Test_Telemetry::test_status_ok: # Created by easy win activation script - weblog_declaration: spring-boot-3-native: missing_feature (GraalVM. Tracing support only) diff --git a/manifests/python.yml b/manifests/python.yml index 6b545aac7f6..4e2886224e1 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2209,7 +2209,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: flask-poc: missing_feature (not implemented yet) - "*": missing_feature (spawn_child endpoint not implemented) + "*": '>=4.8.0' (spawn_child endpoint not implemented) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: flask-poc: missing_feature (not implemented yet) diff --git a/manifests/ruby.yml b/manifests/ruby.yml index 5221d4d862b..ddf3f7c0be2 100644 --- a/manifests/ruby.yml +++ b/manifests/ruby.yml @@ -2271,11 +2271,11 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: '*': missing_feature - rails72: missing_feature (not implemented yet) + rails72: '>=2.31.0' tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: '*': missing_feature - rails72: missing_feature (not implemented yet) + rails72: '>=2.31.0' tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: # Modified by easy win activation script - weblog_declaration: From 441d7531a956c7920e734ef5323062946eafe147 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Tue, 7 Apr 2026 16:30:43 -0400 Subject: [PATCH 21/24] enable cpp tests --- manifests/cpp_nginx.yml | 4 +- manifests/python.yml | 2 +- tests/test_telemetry.py | 29 +++++-- utils/build/docker/cpp_nginx/nginx/backend.c | 80 +++++++++++++++++++ utils/build/docker/cpp_nginx/nginx/nginx.conf | 4 + 5 files changed, 108 insertions(+), 11 deletions(-) diff --git a/manifests/cpp_nginx.yml b/manifests/cpp_nginx.yml index d47f3ac52fe..49cd27ea6e2 100644 --- a/manifests/cpp_nginx.yml +++ b/manifests/cpp_nginx.yml @@ -425,8 +425,8 @@ manifest: component_version: <1.12.0 tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) tests/test_telemetry.py::Test_Telemetry::test_proxy_forwarding: missing_feature # Created by easy win activation script - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: '>=1.12.0' + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: '>=1.12.0' tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_Telemetry::test_telemetry_proxy_enrichment: missing_feature # Created by easy win activation script tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.12.0' # Modified by easy win activation script diff --git a/manifests/python.yml b/manifests/python.yml index c03180df266..24c619e0068 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2358,7 +2358,7 @@ manifest: tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: flask-poc: missing_feature (not implemented yet) - "*": '>=4.8.0' (spawn_child endpoint not implemented) + "*": '>=4.8.0' tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: flask-poc: missing_feature (not implemented yet) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 1e1cce01644..65f4d71bfd5 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -602,6 +602,10 @@ def _validate_session_id_headers_across_processes(self) -> None: if not telemetry_data: raise ValueError("No telemetry data to validate on") + assert len(telemetry_data) > 1, ( + f"Expected multiple telemetry events to verify consistency, got {len(telemetry_data)}" + ) + runtime_ids = set[str]() parent_runtime_ids = set[str]() root_runtime_ids = set[str]() @@ -626,15 +630,24 @@ def _validate_session_id_headers_across_processes(self) -> None: # If dd-root-session-id is not set, dd-session-id is treated as root root_runtime_ids.add(curr_id) - # At least two runtimes: parent (root) and child from spawn_child - assert len(runtime_ids) > 1, f"Expected at least 2 runtime_ids, got {runtime_ids}" - # One root per app instance: all children share the same DD-Root-Session-ID + # One root per app instance: all processes share the same root session ID assert len(root_runtime_ids) == 1, f"Expected 1 root runtime_id, got {root_runtime_ids}" - if parent_runtime_ids: - # DD-Parent-Session-ID is optional but must reference a known runtime if present - missing_parent_runtime_ids = parent_runtime_ids.difference(runtime_ids) - assert not missing_parent_runtime_ids, ( - f"Parent runtime_id with no telemetry data: {missing_parent_runtime_ids}" + + if len(runtime_ids) > 1: + # Multiple runtimes (per-process tracers): parent + child from spawn_child + if parent_runtime_ids: + # DD-Parent-Session-ID is optional but must reference a known runtime if present + missing_parent_runtime_ids = parent_runtime_ids.difference(runtime_ids) + assert not missing_parent_runtime_ids, ( + f"Parent runtime_id with no telemetry data: {missing_parent_runtime_ids}" + ) + else: + # Single runtime (shared tracer, e.g. nginx): all events must report + # the same session ID consistently + sole_rid = next(iter(runtime_ids)) + sole_root = next(iter(root_runtime_ids)) + assert sole_rid == sole_root, ( + f"Single runtime_id {sole_rid} does not match root {sole_root}" ) def test_session_id_headers_across_forks(self): diff --git a/utils/build/docker/cpp_nginx/nginx/backend.c b/utils/build/docker/cpp_nginx/nginx/backend.c index ad18e5a7c15..0c2b896d5c7 100644 --- a/utils/build/docker/cpp_nginx/nginx/backend.c +++ b/utils/build/docker/cpp_nginx/nginx/backend.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #define PORT 7778 @@ -310,6 +311,85 @@ static enum MHD_Result answer_to_connection(void *cls, struct MHD_Connection *co return ret; } + if (strcmp(url, "/spawn_child") == 0) { + const char *sleep_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "sleep"); + const char *crash_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "crash"); + const char *fork_str = MHD_lookup_connection_value(connection, MHD_GET_ARGUMENT_KIND, "fork"); + + if (!sleep_str || !crash_str || !fork_str) { + const char *msg = "sleep, crash, and fork parameters required"; + struct MHD_Response *response = MHD_create_response_from_buffer( + strlen(msg), (void *)msg, MHD_RESPMEM_PERSISTENT); + int ret = MHD_queue_response(connection, 400, response); + MHD_destroy_response(response); + return ret; + } + + int sleep_secs = atoi(sleep_str); + bool do_crash = strcmp(crash_str, "true") == 0; + bool use_fork = strcmp(fork_str, "true") == 0; + + if (use_fork) { + pid_t pid = fork(); + if (pid < 0) { + const char *msg = "fork failed"; + struct MHD_Response *response = MHD_create_response_from_buffer( + strlen(msg), (void *)msg, MHD_RESPMEM_PERSISTENT); + int ret = MHD_queue_response(connection, 500, response); + MHD_destroy_response(response); + return ret; + } + if (pid == 0) { + sleep(sleep_secs); + if (do_crash) { + raise(SIGSEGV); + } + _exit(0); + } + int wstatus; + waitpid(pid, &wstatus, 0); + char buf[128]; + snprintf(buf, sizeof(buf), "Child process %d exited with status %d", pid, WEXITSTATUS(wstatus)); + struct MHD_Response *response = MHD_create_response_from_buffer( + strlen(buf), buf, MHD_RESPMEM_MUST_COPY); + int ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + return ret; + } + + /* exec path: fork + exec a child process */ + { + pid_t pid = fork(); + if (pid < 0) { + const char *msg = "fork failed"; + struct MHD_Response *response = MHD_create_response_from_buffer( + strlen(msg), (void *)msg, MHD_RESPMEM_PERSISTENT); + int ret = MHD_queue_response(connection, 500, response); + MHD_destroy_response(response); + return ret; + } + if (pid == 0) { + if (do_crash) { + execlp("sh", "sh", "-c", + sleep_str[0] ? "sleep $0 && kill -SEGV $$" : "kill -SEGV $$", + sleep_str, (char *)NULL); + } else { + execlp("sleep", "sleep", sleep_str, (char *)NULL); + } + _exit(1); + } + int wstatus; + waitpid(pid, &wstatus, 0); + char buf[128]; + snprintf(buf, sizeof(buf), "Child process %d exited with status %d", pid, WEXITSTATUS(wstatus)); + struct MHD_Response *response = MHD_create_response_from_buffer( + strlen(buf), buf, MHD_RESPMEM_MUST_COPY); + int ret = MHD_queue_response(connection, 200, response); + MHD_destroy_response(response); + return ret; + } + } + if (strcmp(url, "/content") != 0 || !status_str || !value) return MHD_NO; // Only respond to the correct URL and if all parameters are present diff --git a/utils/build/docker/cpp_nginx/nginx/nginx.conf b/utils/build/docker/cpp_nginx/nginx/nginx.conf index bedb6d6d4b3..0b152f74ef5 100644 --- a/utils/build/docker/cpp_nginx/nginx/nginx.conf +++ b/utils/build/docker/cpp_nginx/nginx/nginx.conf @@ -44,6 +44,10 @@ http { proxy_pass http://127.0.0.1:7778; } + location /spawn_child { + proxy_pass http://127.0.0.1:7778; + } + location / { root /builds; try_files /hello.html =404; From f6ea416b4a20e76289912c7026ad001a69c45b46 Mon Sep 17 00:00:00 2001 From: Ayan Khan Date: Tue, 7 Apr 2026 16:52:51 -0400 Subject: [PATCH 22/24] increase number of owrker processes --- tests/test_telemetry.py | 7 ++++--- utils/build/docker/cpp_nginx/nginx/nginx-waf.conf | 1 + utils/build/docker/cpp_nginx/nginx/nginx.conf | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 65f4d71bfd5..68bb6c27e58 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -634,7 +634,8 @@ def _validate_session_id_headers_across_processes(self) -> None: assert len(root_runtime_ids) == 1, f"Expected 1 root runtime_id, got {root_runtime_ids}" if len(runtime_ids) > 1: - # Multiple runtimes (per-process tracers): parent + child from spawn_child + # Multiple runtimes (per-process tracers): root must be consistent + # across all payloads from all processes if parent_runtime_ids: # DD-Parent-Session-ID is optional but must reference a known runtime if present missing_parent_runtime_ids = parent_runtime_ids.difference(runtime_ids) @@ -642,8 +643,8 @@ def _validate_session_id_headers_across_processes(self) -> None: f"Parent runtime_id with no telemetry data: {missing_parent_runtime_ids}" ) else: - # Single runtime (shared tracer, e.g. nginx): all events must report - # the same session ID consistently + # Single runtime (e.g. nginx workers sharing one tracer): session ID + # must be consistent across all events sole_rid = next(iter(runtime_ids)) sole_root = next(iter(root_runtime_ids)) assert sole_rid == sole_root, ( diff --git a/utils/build/docker/cpp_nginx/nginx/nginx-waf.conf b/utils/build/docker/cpp_nginx/nginx/nginx-waf.conf index 3de9e6b716b..0cbea98a35a 100644 --- a/utils/build/docker/cpp_nginx/nginx/nginx-waf.conf +++ b/utils/build/docker/cpp_nginx/nginx/nginx-waf.conf @@ -1,6 +1,7 @@ error_log /var/log/nginx/error.log info; load_module modules/ngx_http_datadog_module.so; +worker_processes 2; thread_pool waf_thread_pool threads=2 max_queue=1000; events { diff --git a/utils/build/docker/cpp_nginx/nginx/nginx.conf b/utils/build/docker/cpp_nginx/nginx/nginx.conf index 0b152f74ef5..3f5d6c3fe2e 100644 --- a/utils/build/docker/cpp_nginx/nginx/nginx.conf +++ b/utils/build/docker/cpp_nginx/nginx/nginx.conf @@ -1,5 +1,7 @@ load_module modules/ngx_http_datadog_module.so; +worker_processes 2; + events { worker_connections 1024; } From 34709510b9078aa2d2baad131bd73c02bafda3f3 Mon Sep 17 00:00:00 2001 From: Munir Date: Mon, 13 Apr 2026 11:24:05 -0400 Subject: [PATCH 23/24] linting --- manifests/cpp_httpd.yml | 4 ++-- tests/test_telemetry.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/manifests/cpp_httpd.yml b/manifests/cpp_httpd.yml index 61cb97e83de..82dc9c2fbb6 100644 --- a/manifests/cpp_httpd.yml +++ b/manifests/cpp_httpd.yml @@ -201,11 +201,11 @@ manifest: - declaration: flaky (APMAPI-1876) component_version: ">1.0.4" tests/test_telemetry.py::Test_Telemetry::test_app_product_change: missing_feature (Weblog GET/enable_product and app-product-change event is not implemented yet.) - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature - tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_Telemetry::test_app_started_sent_exactly_once: - declaration: flaky (APMAPI-1876) component_version: ">1.0.4" + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: missing_feature + tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: missing_feature tests/test_telemetry.py::Test_Telemetry::test_telemetry_message_has_datadog_container_id: "irrelevant (cgroup in weblog is 0::/, so this test can't work)" tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting: '>=1.0.3' # Modified by easy win activation script tests/test_telemetry.py::Test_TelemetryEnhancedConfigReporting::test_telemetry_enhanced_config_reporting_precedence: missing_feature # Created by easy win activation script diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 68bb6c27e58..578834e5680 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -647,9 +647,7 @@ def _validate_session_id_headers_across_processes(self) -> None: # must be consistent across all events sole_rid = next(iter(runtime_ids)) sole_root = next(iter(root_runtime_ids)) - assert sole_rid == sole_root, ( - f"Single runtime_id {sole_rid} does not match root {sole_root}" - ) + assert sole_rid == sole_root, f"Single runtime_id {sole_rid} does not match root {sole_root}" def test_session_id_headers_across_forks(self): """Test session ID headers in telemetry (fork=true). Stable Service Instance Identifier RFC.""" From 7e72a422cc2c0d86c72d0674ce0a79cf3fe8e212 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Mon, 13 Apr 2026 12:33:54 -0400 Subject: [PATCH 24/24] fix python manifest --- manifests/python.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/python.yml b/manifests/python.yml index a5bcfb32550..88d216cbf35 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -2360,8 +2360,8 @@ manifest: - declaration: bug (APMAPI-1858) tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_forks: - weblog_declaration: - flask-poc: missing_feature (not implemented yet) - "*": '>=4.8.0' + "*": missing_feature (not implemented yet) + flask-poc: '>=4.8.0' tests/test_telemetry.py::Test_Telemetry::test_session_id_headers_across_spawned: - weblog_declaration: flask-poc: missing_feature (not implemented yet)