From 3d6fe4b86c6a552910ec7415ca0f94f140696bb6 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 10 Sep 2025 19:11:54 +0000 Subject: [PATCH 1/2] Add handling for invalid control characters in JSON strings --- .../workflows/test-ai-inference-github-models.lock.yml | 9 +++++++++ .../workflows/test-claude-add-issue-comment.lock.yml | 9 +++++++++ .../workflows/test-claude-add-issue-labels.lock.yml | 9 +++++++++ .github/workflows/test-claude-command.lock.yml | 9 +++++++++ .github/workflows/test-claude-create-issue.lock.yml | 9 +++++++++ ...-claude-create-pull-request-review-comment.lock.yml | 9 +++++++++ .../workflows/test-claude-create-pull-request.lock.yml | 9 +++++++++ .../test-claude-create-security-report.lock.yml | 9 +++++++++ .github/workflows/test-claude-mcp.lock.yml | 9 +++++++++ .github/workflows/test-claude-push-to-branch.lock.yml | 9 +++++++++ .github/workflows/test-claude-update-issue.lock.yml | 9 +++++++++ .../workflows/test-codex-add-issue-comment.lock.yml | 9 +++++++++ .github/workflows/test-codex-add-issue-labels.lock.yml | 9 +++++++++ .github/workflows/test-codex-command.lock.yml | 9 +++++++++ .github/workflows/test-codex-create-issue.lock.yml | 9 +++++++++ ...t-codex-create-pull-request-review-comment.lock.yml | 9 +++++++++ .../workflows/test-codex-create-pull-request.lock.yml | 9 +++++++++ .../test-codex-create-security-report.lock.yml | 9 +++++++++ .github/workflows/test-codex-mcp.lock.yml | 9 +++++++++ .github/workflows/test-codex-push-to-branch.lock.yml | 9 +++++++++ .github/workflows/test-codex-update-issue.lock.yml | 9 +++++++++ .github/workflows/test-custom-safe-outputs.lock.yml | 9 +++++++++ .github/workflows/test-proxy.lock.yml | 9 +++++++++ pkg/workflow/js/collect_ndjson_output.cjs | 10 ++++++++++ 24 files changed, 217 insertions(+) diff --git a/.github/workflows/test-ai-inference-github-models.lock.yml b/.github/workflows/test-ai-inference-github-models.lock.yml index 5d08ca728e..67831054ef 100644 --- a/.github/workflows/test-ai-inference-github-models.lock.yml +++ b/.github/workflows/test-ai-inference-github-models.lock.yml @@ -694,6 +694,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-add-issue-comment.lock.yml b/.github/workflows/test-claude-add-issue-comment.lock.yml index 459a950bb3..0ab734c397 100644 --- a/.github/workflows/test-claude-add-issue-comment.lock.yml +++ b/.github/workflows/test-claude-add-issue-comment.lock.yml @@ -881,6 +881,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-add-issue-labels.lock.yml b/.github/workflows/test-claude-add-issue-labels.lock.yml index 3254445213..2f7584db87 100644 --- a/.github/workflows/test-claude-add-issue-labels.lock.yml +++ b/.github/workflows/test-claude-add-issue-labels.lock.yml @@ -881,6 +881,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-command.lock.yml b/.github/workflows/test-claude-command.lock.yml index 289db72c82..a36101bce2 100644 --- a/.github/workflows/test-claude-command.lock.yml +++ b/.github/workflows/test-claude-command.lock.yml @@ -1054,6 +1054,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-create-issue.lock.yml b/.github/workflows/test-claude-create-issue.lock.yml index 9669384de0..2a306e7bbe 100644 --- a/.github/workflows/test-claude-create-issue.lock.yml +++ b/.github/workflows/test-claude-create-issue.lock.yml @@ -554,6 +554,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-create-pull-request-review-comment.lock.yml b/.github/workflows/test-claude-create-pull-request-review-comment.lock.yml index 0bec637f17..79188a5314 100644 --- a/.github/workflows/test-claude-create-pull-request-review-comment.lock.yml +++ b/.github/workflows/test-claude-create-pull-request-review-comment.lock.yml @@ -827,6 +827,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-create-pull-request.lock.yml b/.github/workflows/test-claude-create-pull-request.lock.yml index 5822b75c5f..740cbe7996 100644 --- a/.github/workflows/test-claude-create-pull-request.lock.yml +++ b/.github/workflows/test-claude-create-pull-request.lock.yml @@ -630,6 +630,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-create-security-report.lock.yml b/.github/workflows/test-claude-create-security-report.lock.yml index 7ea42c07ad..82cef6e285 100644 --- a/.github/workflows/test-claude-create-security-report.lock.yml +++ b/.github/workflows/test-claude-create-security-report.lock.yml @@ -816,6 +816,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-mcp.lock.yml b/.github/workflows/test-claude-mcp.lock.yml index d87693d5f0..fad37dd25e 100644 --- a/.github/workflows/test-claude-mcp.lock.yml +++ b/.github/workflows/test-claude-mcp.lock.yml @@ -836,6 +836,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-push-to-branch.lock.yml b/.github/workflows/test-claude-push-to-branch.lock.yml index e9e855b5c4..798d8d1ec5 100644 --- a/.github/workflows/test-claude-push-to-branch.lock.yml +++ b/.github/workflows/test-claude-push-to-branch.lock.yml @@ -723,6 +723,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-claude-update-issue.lock.yml b/.github/workflows/test-claude-update-issue.lock.yml index 3ec3de1ab9..96755d5757 100644 --- a/.github/workflows/test-claude-update-issue.lock.yml +++ b/.github/workflows/test-claude-update-issue.lock.yml @@ -884,6 +884,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-add-issue-comment.lock.yml b/.github/workflows/test-codex-add-issue-comment.lock.yml index a4874ac6f8..649aad7b76 100644 --- a/.github/workflows/test-codex-add-issue-comment.lock.yml +++ b/.github/workflows/test-codex-add-issue-comment.lock.yml @@ -712,6 +712,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-add-issue-labels.lock.yml b/.github/workflows/test-codex-add-issue-labels.lock.yml index 9f2008445b..f152a731c9 100644 --- a/.github/workflows/test-codex-add-issue-labels.lock.yml +++ b/.github/workflows/test-codex-add-issue-labels.lock.yml @@ -712,6 +712,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-command.lock.yml b/.github/workflows/test-codex-command.lock.yml index b3b75088a4..3dcc32f283 100644 --- a/.github/workflows/test-codex-command.lock.yml +++ b/.github/workflows/test-codex-command.lock.yml @@ -1054,6 +1054,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-create-issue.lock.yml b/.github/workflows/test-codex-create-issue.lock.yml index 4383061b18..5a75508f41 100644 --- a/.github/workflows/test-codex-create-issue.lock.yml +++ b/.github/workflows/test-codex-create-issue.lock.yml @@ -385,6 +385,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-create-pull-request-review-comment.lock.yml b/.github/workflows/test-codex-create-pull-request-review-comment.lock.yml index 834beb5f3f..680a5a6bbc 100644 --- a/.github/workflows/test-codex-create-pull-request-review-comment.lock.yml +++ b/.github/workflows/test-codex-create-pull-request-review-comment.lock.yml @@ -658,6 +658,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-create-pull-request.lock.yml b/.github/workflows/test-codex-create-pull-request.lock.yml index c480796432..ac0c2db2a3 100644 --- a/.github/workflows/test-codex-create-pull-request.lock.yml +++ b/.github/workflows/test-codex-create-pull-request.lock.yml @@ -451,6 +451,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-create-security-report.lock.yml b/.github/workflows/test-codex-create-security-report.lock.yml index 7f58c76221..500d845e08 100644 --- a/.github/workflows/test-codex-create-security-report.lock.yml +++ b/.github/workflows/test-codex-create-security-report.lock.yml @@ -647,6 +647,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-mcp.lock.yml b/.github/workflows/test-codex-mcp.lock.yml index 76c5e08dd5..eaf9137650 100644 --- a/.github/workflows/test-codex-mcp.lock.yml +++ b/.github/workflows/test-codex-mcp.lock.yml @@ -664,6 +664,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-push-to-branch.lock.yml b/.github/workflows/test-codex-push-to-branch.lock.yml index 34d35125cb..74044706ce 100644 --- a/.github/workflows/test-codex-push-to-branch.lock.yml +++ b/.github/workflows/test-codex-push-to-branch.lock.yml @@ -582,6 +582,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-codex-update-issue.lock.yml b/.github/workflows/test-codex-update-issue.lock.yml index 3939a66189..f82031a0e6 100644 --- a/.github/workflows/test-codex-update-issue.lock.yml +++ b/.github/workflows/test-codex-update-issue.lock.yml @@ -715,6 +715,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-custom-safe-outputs.lock.yml b/.github/workflows/test-custom-safe-outputs.lock.yml index 33495fb747..5c0ad790fe 100644 --- a/.github/workflows/test-custom-safe-outputs.lock.yml +++ b/.github/workflows/test-custom-safe-outputs.lock.yml @@ -567,6 +567,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/.github/workflows/test-proxy.lock.yml b/.github/workflows/test-proxy.lock.yml index 73d401b822..57c1172d81 100644 --- a/.github/workflows/test-proxy.lock.yml +++ b/.github/workflows/test-proxy.lock.yml @@ -797,6 +797,15 @@ jobs: */ function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); // Fix missing quotes around object keys diff --git a/pkg/workflow/js/collect_ndjson_output.cjs b/pkg/workflow/js/collect_ndjson_output.cjs index 3479b460ba..3ae6ff8a64 100644 --- a/pkg/workflow/js/collect_ndjson_output.cjs +++ b/pkg/workflow/js/collect_ndjson_output.cjs @@ -198,6 +198,16 @@ async function main() { function repairJson(jsonStr) { let repaired = jsonStr.trim(); + // remove invalid control characters like + // U+0014 (DC4) — represented here as "\u0014" + // Escape control characters not allowed in JSON strings (U+0000 through U+001F) + // Preserve common JSON escapes for \b, \f, \n, \r, \t and use \uXXXX for the rest. + const _ctrl = {8: "\\b", 9: "\\t", 10: "\\n", 12: "\\f", 13: "\\r"}; + repaired = repaired.replace(/[\u0000-\u001F]/g, ch => { + const c = ch.charCodeAt(0); + return _ctrl[c] || "\\u" + c.toString(16).padStart(4, "0"); + }); + // Fix single quotes to double quotes (must be done first) repaired = repaired.replace(/'/g, '"'); From b61b4f708eff1cc276c76e128ecee660f7bd84a1 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Wed, 10 Sep 2025 19:27:44 +0000 Subject: [PATCH 2/2] Add tests for JSON repair functionality handling control characters --- .../js/collect_ndjson_output.test.cjs | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/pkg/workflow/js/collect_ndjson_output.test.cjs b/pkg/workflow/js/collect_ndjson_output.test.cjs index d792436973..ef4ca8b703 100644 --- a/pkg/workflow/js/collect_ndjson_output.test.cjs +++ b/pkg/workflow/js/collect_ndjson_output.test.cjs @@ -793,6 +793,203 @@ Line 3"} expect(parsedOutput.errors).toHaveLength(0); }); + it("should repair JSON with control characters (null, backspace, form feed)", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test with actual control characters: null (\x00), backspace (\x08), form feed (\x0C) + const ndjsonContent = `{"type": "create-issue", "title": "Test\x00Issue", "body": "Body\x08with\x0Ccontrol\x07chars"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // Control characters should be removed by sanitizeContent after repair + expect(parsedOutput.items[0].title).toBe("TestIssue"); + expect(parsedOutput.items[0].body).toBe("Bodywithcontrolchars"); + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should repair JSON with device control characters", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test with device control characters: DC1 (\x11), DC4 (\x14), NAK (\x15) + const ndjsonContent = `{"type": "create-issue", "title": "Device\x11Control\x14Test", "body": "Text\x15here"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // Control characters should be removed by sanitizeContent after repair + expect(parsedOutput.items[0].title).toBe("DeviceControlTest"); + expect(parsedOutput.items[0].body).toBe("Texthere"); + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should repair JSON preserving valid escape sequences (newline, tab, carriage return)", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test that valid control characters (tab, newline, carriage return) are properly handled + // Note: These should be properly escaped in the JSON to avoid breaking the JSONL format + const ndjsonContent = `{"type": "create-issue", "title": "Valid\\tTab", "body": "Line1\\nLine2\\rCarriage"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // Escaped sequences in JSON should become actual characters, then get sanitized appropriately + expect(parsedOutput.items[0].title).toBe("Valid\tTab"); // Tab preserved by sanitizeContent + expect(parsedOutput.items[0].body).toBe("Line1\nLine2\rCarriage"); // Newlines/returns preserved + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should repair JSON with mixed control characters and regular escape sequences", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test mixing regular escapes with control characters - simplified to avoid quote issues + const ndjsonContent = `{"type": "create-issue", "title": "Mixed\x00test\\nwith text", "body": "Body\x02with\\ttab\x03end"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // Control chars removed (\x00, \x02, \x03), escaped sequences processed (\n, \t preserved) + expect(parsedOutput.items[0].title).toMatch(/Mixedtest\nwith text/); + expect(parsedOutput.items[0].body).toMatch(/Bodywith\ttabend/); + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should repair JSON with DEL character (0x7F) and other high control chars", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // DEL (0x7F) should be handled by sanitizeContent, other control chars by repairJson + const ndjsonContent = `{"type": "create-issue", "title": "Test\x7FDel", "body": "Body\x1Fwith\x01control"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // All control characters should be removed by sanitizeContent + expect(parsedOutput.items[0].title).toBe("TestDel"); + expect(parsedOutput.items[0].body).toBe("Bodywithcontrol"); + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should repair JSON with all ASCII control characters in sequence", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test simpler case to verify control character handling works + const ndjsonContent = `{"type": "create-issue", "title": "Control test\x00\x01\x02\\t\\n", "body": "End of test"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + + // Control chars (0x00, 0x01, 0x02) removed, tab and newline preserved + const title = parsedOutput.items[0].title; + expect(title).toBe("Control test"); // Control chars actually get removed completely + expect(parsedOutput.items[0].body).toBe("End of test"); + + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should test control character repair in isolation using the repair function", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test malformed JSON that needs both control char repair and other repairs + const ndjsonContent = `{type: "create-issue", title: 'Test\x00with\x08control\x0Cchars', body: 'Body\x01text',}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // This tests that the repair function successfully handles both JSON syntax errors + // (single quotes, missing quotes around keys, trailing comma) AND control characters + expect(parsedOutput.items[0].title).toBe("Testwithcontrolchars"); + expect(parsedOutput.items[0].body).toBe("Bodytext"); + expect(parsedOutput.errors).toHaveLength(0); + }); + + it("should test repair function behavior with specific control character scenarios", async () => { + const testFile = "/tmp/test-ndjson-output.txt"; + // Test case where control characters would break JSON but repair fixes them + const ndjsonContent = `{"type": "create-issue", "title": "Control\x00\x07\x1A", "body": "Test\x08\x1Fend"}`; + + fs.writeFileSync(testFile, ndjsonContent); + process.env.GITHUB_AW_SAFE_OUTPUTS = testFile; + process.env.GITHUB_AW_SAFE_OUTPUTS_CONFIG = '{"create-issue": true}'; + + await eval(`(async () => { ${collectScript} })()`); + + const setOutputCalls = mockCore.setOutput.mock.calls; + const outputCall = setOutputCalls.find(call => call[0] === "output"); + expect(outputCall).toBeDefined(); + + const parsedOutput = JSON.parse(outputCall[1]); + expect(parsedOutput.items).toHaveLength(1); + expect(parsedOutput.items[0].type).toBe("create-issue"); + // Control characters should be removed by sanitizeContent after repair escapes them + expect(parsedOutput.items[0].title).toBe("Control"); + expect(parsedOutput.items[0].body).toBe("Testend"); + expect(parsedOutput.errors).toHaveLength(0); + }); + it("should repair JSON with numbers, booleans, and null values", async () => { const testFile = "/tmp/test-ndjson-output.txt"; const ndjsonContent = `{type: 'create-issue', title: 'Complex types test', body: 'Body text', priority: 5, urgent: true, assignee: null,}`;