From 77de13793b5f9891b1e62bafe400a947f413f7fa Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 11:27:22 +0100 Subject: [PATCH 1/9] test(crashtracker): increase crash-report timeout to 20s Timeout was hitting in CI; tests may just need more time. Bumping from 10s to 20s to reduce flakiness. If the root cause is a race where a test hangs, a higher timeout alone won't fix it. --- test/crashtracker/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index d017c46..5effa3e 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -24,7 +24,7 @@ const timeout = setTimeout(() => { execSync('cat stderr.log', opts) throw new Error('No crash report received before timing out.') -}, 10_000) +}, 20_000) let currentTest From 93f3d64be0a817dbbac2c3554ed8670c283e78ca Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 12:01:53 +0100 Subject: [PATCH 2/9] wip --- test/crashtracker/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index 5effa3e..73bc342 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -24,7 +24,7 @@ const timeout = setTimeout(() => { execSync('cat stderr.log', opts) throw new Error('No crash report received before timing out.') -}, 20_000) +}, 120_000) let currentTest From e56f33c50d46c31242ec2631f1b8c650abfb223c Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 13:22:56 +0100 Subject: [PATCH 3/9] fix(crashtracker): guard log file reads in timeout handler The crashtracker-receiver creates stdout.log and stderr.log, but if it never starts, these files won't exist. The timeout handler previously called `cat` unconditionally, which threw an error that masked the actual timeout message. --- test/crashtracker/index.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index 73bc342..f6170f2 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -20,8 +20,18 @@ rmSync(path.join(cwd, 'stdout.log'), { force: true }) rmSync(path.join(cwd, 'stderr.log'), { force: true }) const timeout = setTimeout(() => { - execSync('cat stdout.log', opts) - execSync('cat stderr.log', opts) + const stdoutLog = path.join(cwd, 'stdout.log') + const stderrLog = path.join(cwd, 'stderr.log') + if (existsSync(stdoutLog)) { + execSync(`cat ${stdoutLog}`, opts) + } else { + console.error('stdout.log not found (crashtracker-receiver may not have started)') + } + if (existsSync(stderrLog)) { + execSync(`cat ${stderrLog}`, opts) + } else { + console.error('stderr.log not found (crashtracker-receiver may not have started)') + } throw new Error('No crash report received before timing out.') }, 120_000) From 44a9f6e989d0338164d33836b02731d5071e523e Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 14:57:35 +0100 Subject: [PATCH 4/9] test(crashtracker): add error handling to child process exec Previously, `runApp` ignored child process failures, causing the test to silently hang until the timeout. Now the promise rejects on spawn errors and on child exit when no crash report is received (after a 5s grace period). --- test/crashtracker/index.js | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index f6170f2..456a49d 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -61,16 +61,41 @@ app.post('/telemetry/proxy/api/v2/apmtelemetry', (req, res) => { let PORT function runApp (script) { - return new Promise((resolve) => { - exec(`node ${script}`, { + return new Promise((resolve, reject) => { + let closeTimer + let done = false + + const child = exec(`node ${script}`, { ...opts, env: { ...process.env, PORT }, }) + child.on('error', (err) => { + cleanup() + reject(new Error(`Child process for "${script}" failed to start`, { cause: err })) + }) + + child.on('close', (code, signal) => { + if (done) return + // Allow a grace period for the crash report HTTP request to arrive + // after the child process exits (e.g. segfault sends report then dies). + closeTimer = setTimeout(() => { + cleanup() + const reason = signal ? `signal ${signal}` : `exit code ${code}` + reject(new Error(`Child process for "${script}" exited with ${reason} before sending a crash report`)) + }, 5000) + }) + currentTest = (logPayload, tags) => { + cleanup() currentTest = undefined resolve({ logPayload, tags }) } + + function cleanup () { + clearTimeout(closeTimer) + done = true + } }) } From d765b8514f62a811542d9d512eb52065cb74bbc1 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 17:25:47 +0100 Subject: [PATCH 5/9] test(crashtracker): increase handler timeout to 15s The crashtracker signal handler's 3s timeout budget was too short for in-process symbol resolution (blazesym) on Alpine under CI resource pressure, causing the handler to silently abandon crash reports before they could be sent. --- test/crashtracker/test-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/crashtracker/test-utils.js b/test/crashtracker/test-utils.js index 5dd68e4..cc62da4 100644 --- a/test/crashtracker/test-utils.js +++ b/test/crashtracker/test-utils.js @@ -16,7 +16,7 @@ function initTestCrashtracker () { }, timeout_ms: 3000, }, - timeout: { secs: 3, nanos: 0 }, + timeout: { secs: 15, nanos: 0 }, resolve_frames: 'EnabledWithInprocessSymbols', wait_for_receiver: true, demangle_names: false, From 2d51efc4f2fa5e9f3c61ad11180e17185c54e172 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 18:23:42 +0100 Subject: [PATCH 6/9] test(crashtracker): reduce crash-report timeout to 10s --- test/crashtracker/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index 456a49d..a10e92b 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -34,7 +34,7 @@ const timeout = setTimeout(() => { } throw new Error('No crash report received before timing out.') -}, 120_000) +}, 10_000) let currentTest From 85fde90100193f2e5f9ce75a23d6002a166a9fef Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 18:43:20 +0100 Subject: [PATCH 7/9] cleanup --- test/crashtracker/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index a10e92b..a8852c4 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -80,7 +80,6 @@ function runApp (script) { // Allow a grace period for the crash report HTTP request to arrive // after the child process exits (e.g. segfault sends report then dies). closeTimer = setTimeout(() => { - cleanup() const reason = signal ? `signal ${signal}` : `exit code ${code}` reject(new Error(`Child process for "${script}" exited with ${reason} before sending a crash report`)) }, 5000) From f562880ef47cdde21c07944a5acbb510f85e895c Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 18:51:45 +0100 Subject: [PATCH 8/9] Improve logging --- test/crashtracker/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index a8852c4..7eb4c23 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -99,6 +99,8 @@ function runApp (script) { } async function testSegfault () { + console.log('Running test: testSegfault') + const { logPayload, tags } = await runApp('app-seg-fault') const stackTrace = JSON.parse(logPayload.message).error.stack.frames const boomFrame = stackTrace.find(frame => frame.function?.toLowerCase().includes('segfaultify')) @@ -113,6 +115,8 @@ async function testSegfault () { } async function testUnhandledError (label, script, { expectedType, expectedMessage, expectedFrame }) { + console.log('Running test: testUnhandledError', label) + const { logPayload } = await runApp(script) const crashReport = JSON.parse(logPayload.message) @@ -125,6 +129,8 @@ async function testUnhandledError (label, script, { expectedType, expectedMessag } async function testUnhandledNonError (label, script, { expectedFallbackType, expectedValue }) { + console.log('Running test: testUnhandledNonError', label) + const { logPayload } = await runApp(script) const crashReport = JSON.parse(logPayload.message) From 28eb001bdcbc703c90eca8fb13f700200f5b5217 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Tue, 3 Mar 2026 18:59:26 +0100 Subject: [PATCH 9/9] Increase timeout again It needs to be higher than 15 sec --- test/crashtracker/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/crashtracker/index.js b/test/crashtracker/index.js index 7eb4c23..ac3d149 100644 --- a/test/crashtracker/index.js +++ b/test/crashtracker/index.js @@ -34,7 +34,7 @@ const timeout = setTimeout(() => { } throw new Error('No crash report received before timing out.') -}, 10_000) +}, 20_000) let currentTest