From cf6b93b7db4a79f2af534fcc28b26012efc909c3 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 18 May 2026 17:52:09 +0300 Subject: [PATCH 01/24] feat(ci): add e2e duration charts Add per-spec timing propagation, Chart.js report rendering, Loop file uploads, and local report artifacts so E2E failures can include duration context. Signed-off-by: Nikita Korolev --- .../scripts/js/e2e/report/cluster-report.js | 57 ++++- .../js/e2e/report/cluster-report.test.js | 92 +++++--- .../scripts/js/e2e/report/messenger-report.js | 27 ++- .../js/e2e/report/messenger-report.test.js | 173 ++++++++++++-- .../__snapshots__/chart-config.test.js.snap | 220 +++++++++++++++++ .../report/messenger/charts/chart-config.js | 222 ++++++++++++++++++ .../messenger/charts/chart-config.test.js | 43 ++++ .../report/messenger/charts/chart-renderer.js | 62 +++++ .../messenger/charts/chart-renderer.test.js | 57 +++++ .../js/e2e/report/messenger/loop-client.js | 87 ++++++- .../e2e/report/messenger/loop-client.test.js | 56 +++++ .../js/e2e/report/messenger/markdown.js | 157 ++++++++++++- .../e2e/report/shared/ginkgo-report-utils.js | 49 +++- .../js/e2e/report/shared/report-model.js | 7 +- .github/scripts/js/package.json | 4 + .github/workflows/e2e-matrix.yml | 9 + 16 files changed, 1220 insertions(+), 102 deletions(-) create mode 100644 .github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap create mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-config.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-config.test.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-renderer.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js create mode 100644 .github/scripts/js/e2e/report/messenger/loop-client.test.js diff --git a/.github/scripts/js/e2e/report/cluster-report.js b/.github/scripts/js/e2e/report/cluster-report.js index e1df27c7d3..5cd550470c 100644 --- a/.github/scripts/js/e2e/report/cluster-report.js +++ b/.github/scripts/js/e2e/report/cluster-report.js @@ -69,11 +69,27 @@ const { */ const workflowStages = [ - { name: "bootstrap", displayName: "Bootstrap cluster", needsJobId: "bootstrap" }, - { name: "configure-sdn", displayName: "Configure SDN", needsJobId: "configure-sdn" }, - { name: "storage-setup", displayName: "Configure storage", needsJobId: "configure-storage" }, - { name: "virtualization-setup", displayName: "Configure Virtualization", needsJobId: "configure-virtualization" }, - { name: "e2e-test", displayName: "E2E test", needsJobId: "e2e-test" }, + { + name: "bootstrap", + displayName: "Bootstrap cluster", + needsJobId: "bootstrap", + }, + { + name: "configure-sdn", + displayName: "Configure SDN", + needsJobId: "configure-sdn", + }, + { + name: "storage-setup", + displayName: "Configure storage", + needsJobId: "configure-storage", + }, + { + name: "virtualization-setup", + displayName: "Configure Virtualization", + needsJobId: "configure-virtualization", + }, + { name: "e2e-test", displayName: "E2E test", needsJobId: "e2e-test" }, ]; function readClusterReportConfigFromEnv(env = process.env) { @@ -87,7 +103,11 @@ function readClusterReportConfigFromEnv(env = process.env) { }; } -const requiredClusterReportConfigKeys = ["storageType", "reportsDir", "reportFile"]; +const requiredClusterReportConfigKeys = [ + "storageType", + "reportsDir", + "reportFile", +]; function requireClusterReportConfig(config) { for (const key of requiredClusterReportConfigKeys) { @@ -128,7 +148,9 @@ async function listWorkflowRunJobs(github, context) { } function findWorkflowJob(jobs, pipelineJobName, jobName) { - const nestedJobName = pipelineJobName ? `${pipelineJobName} / ${jobName}` : ""; + const nestedJobName = pipelineJobName + ? `${pipelineJobName} / ${jobName}` + : ""; return ( jobs.find((job) => job.name === nestedJobName) || @@ -147,7 +169,8 @@ function readStageResultsFromEnv(env = process.env) { const stageResults = {}; for (const { name, needsJobId } of workflowStages) { - stageResults[name] = String((needs[needsJobId] || {}).result || "").trim() || "skipped"; + stageResults[name] = + String((needs[needsJobId] || {}).result || "").trim() || "skipped"; } return stageResults; } @@ -161,7 +184,9 @@ async function readStageJobUrlsFromApi(github, context, config, core) { if (job) { stageJobUrls[name] = job.html_url || ""; } else { - core.warning(`Unable to find workflow job "${displayName}" for E2E report`); + core.warning( + `Unable to find workflow job "${displayName}" for E2E report` + ); } } @@ -178,6 +203,8 @@ async function readStageJobUrlsFromApi(github, context, config, core) { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: null, * source: string, * }} Empty parsed-report payload. @@ -187,6 +214,8 @@ function emptyParsedReport(source) { metrics: zeroMetrics(), failedTests: [], failedTestDetails: [], + specTimings: [], + suiteTotalMs: 0, startedAt: null, source, }; @@ -217,6 +246,8 @@ const ginkgoOutputSource = { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: string|null, * }} parse Parser function for the source content. * @property {function(string): RegExp} pattern Builds the file-name regex for the source. @@ -252,6 +283,8 @@ function findGinkgoSource(config, source) { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: string|null, * source: string, * }} Parsed report payload with a source tag. @@ -320,6 +353,8 @@ function buildReportPayload({ metrics: parsedReport.metrics, failedTests: parsedReport.failedTests, failedTestDetails: parsedReport.failedTestDetails, + specTimings: parsedReport.specTimings || [], + suiteTotalMs: parsedReport.suiteTotalMs || 0, sourceReport: sourcePath, reportSource: parsedReport.source, }; @@ -391,7 +426,9 @@ async function buildClusterReport({ core, context, github, config } = {}) { ? null : findGinkgoSource(resolvedConfig, ginkgoOutputSource); const sourcePath = rawReportPath || outputPath; - const sourceDescriptor = rawReportPath ? ginkgoJsonSource : ginkgoOutputSource; + const sourceDescriptor = rawReportPath + ? ginkgoJsonSource + : ginkgoOutputSource; if (!rawReportPath) { core.warning( diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index c8bbd2b34f..dc8486be1d 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -47,14 +47,12 @@ function createContext() { * @returns {Record} Mocked GitHub client. */ function createGithub(jobNames) { - const jobs = jobNames.map( - (name, index) => ({ - name, - html_url: `https://github.com/test/repo/actions/runs/12345/job/${ - index + 1 - }`, - }) - ); + const jobs = jobNames.map((name, index) => ({ + name, + html_url: `https://github.com/test/repo/actions/runs/12345/job/${ + index + 1 + }`, + })); return { rest: { @@ -238,11 +236,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "success" }, - "configure-storage": { result: "success" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "success" }, + "configure-storage": { result: "success" }, "configure-virtualization": { result: "success" }, - "e2e-test": { result: "success" }, + "e2e-test": { result: "success" }, }); expect(readClusterReportConfigFromEnv()).toMatchObject({ @@ -275,11 +273,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "failure" }, - "configure-storage": { result: "skipped" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "failure" }, + "configure-storage": { result: "skipped" }, "configure-virtualization": { result: "skipped" }, - "e2e-test": { result: "skipped" }, + "e2e-test": { result: "skipped" }, }); const report = await buildClusterReport({ @@ -305,11 +303,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "failure" }, - "configure-storage": { result: "skipped" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "failure" }, + "configure-storage": { result: "skipped" }, "configure-virtualization": { result: "skipped" }, - "e2e-test": { result: "skipped" }, + "e2e-test": { result: "skipped" }, }); const report = await buildClusterReport({ @@ -341,11 +339,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "success" }, - "configure-storage": { result: "success" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "success" }, + "configure-storage": { result: "success" }, "configure-virtualization": { result: "success" }, - "e2e-test": { result: "success" }, + "e2e-test": { result: "success" }, }); const report = await buildClusterReport({ @@ -454,11 +452,43 @@ describe("cluster-report", () => { reason: "timed out waiting for VM to become ready", }, ]); + expect(report.suiteTotalMs).toBe(1800000); + expect(report.specTimings).toEqual([ + { + name: "passes", + group: "Suite", + state: "passed", + runtimeMs: 60000, + labels: [], + }, + { + name: "fails & burns", + group: "Suite", + state: "failed", + runtimeMs: 60000, + labels: ["Slow"], + }, + { + name: "errors ", + group: "Other", + state: "errors", + runtimeMs: 60000, + labels: [], + }, + { + name: "skipped", + group: "skipped", + state: "skipped", + runtimeMs: 60000, + labels: [], + }, + ]); expect(report.reportSource).toBe("ginkgo-json"); expect(report.sourceReport).toBe(rawReportPath); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).reportKind).toBe( - "tests" - ); + const persistedReport = JSON.parse(fs.readFileSync(reportFile, "utf8")); + expect(persistedReport.reportKind).toBe("tests"); + expect(persistedReport.specTimings).toEqual(report.specTimings); + expect(persistedReport.suiteTotalMs).toBe(1800000); expect(core.setOutput).toHaveBeenCalledWith("report_file", reportFile); expect(core.setOutput).toHaveBeenCalledWith("report_kind", "tests"); expect(core.setOutput).toHaveBeenCalledWith("status", "failure"); @@ -590,7 +620,7 @@ describe("cluster-report", () => { "/home/runner/work/virtualization/virtualization/test/e2e/e2e_test.go:44", "PASS: all 94 specs have precheck labels", " STEP: Ensuring 12 precreated CVIs are available @ 05/14/26 13:57:36.142", - " CVI \"v12n-e2e-testdata-iso\" exists but not ready (phase: Pending), waiting...", + ' CVI "v12n-e2e-testdata-iso" exists but not ready (phase: Pending), waiting...', " [FAILED] in [SynchronizedBeforeSuite] - /home/runner/work/.../until.go:207 @ 05/14/26 14:02:37.61", "[SynchronizedBeforeSuite] [FAILED] [307.964 seconds]", "[SynchronizedBeforeSuite]", @@ -636,7 +666,9 @@ describe("cluster-report", () => { ); // The plain "[SynchronizedBeforeSuite]" header that follows the // "[FAILED] [307.964 seconds]" line must not leak into the reason. - expect(detail.reason.split("\n")[0]).not.toBe("[SynchronizedBeforeSuite]"); + expect(detail.reason.split("\n")[0]).not.toBe( + "[SynchronizedBeforeSuite]" + ); })); test("fails when multiple matching Ginkgo JSON reports exist", async () => @@ -828,6 +860,8 @@ describe("cluster-report", () => { successRate: 92.78, }); expect(parsed.startedAt).toBe("2026-04-28T03:11:27.708387575Z"); + expect(parsed.specTimings).toHaveLength(131); + expect(parsed.suiteTotalMs).toBe(1800000); expect(parsed.failedTests).toHaveLength(7); expect(parsed.failedTests).toContain( "[It] VirtualMachineOperationRestore restores a virtual machine from a snapshot BestEffort restore mode; automatic restart approval mode; manual run policy [Slow]" diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index fb2c6906b2..1838a3d5ff 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -14,6 +14,7 @@ const fs = require("fs"); const { listMatchingFiles } = require("./shared/fs-utils"); const { REPORT_FILE_PATTERN } = require("./shared/report-model"); +const { renderClusterCharts } = require("./messenger/charts/chart-renderer"); const { makeThreadedReportInLoop } = require("./messenger/loop-client"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { @@ -103,12 +104,18 @@ function readReports(reportsDir, configuredClusters, core) { * @param {MessengerMessagesParams} params Message rendering inputs. * @returns {{ * message: string, - * threadMessages: string[] + * threadMessages: Array<{message: string, files: Array>}> * }} Rendered markdown payloads. */ -function buildMessengerMessages({ reportsDir, configuredClusters, core }) { +async function buildMessengerMessages({ + reportsDir, + configuredClusters, + core, +}) { const orderedReports = readReports(reportsDir, configuredClusters, core); - const threadMessages = buildThreadMessages(orderedReports); + const threadMessages = await buildThreadMessages(orderedReports, { + renderClusterCharts, + }); return { message: buildMainMessage(orderedReports), threadMessages, @@ -122,12 +129,12 @@ function buildMessengerMessages({ reportsDir, configuredClusters, core }) { * @param {RenderMessengerReportParams} params GitHub script dependencies. * @returns {Promise<{ * message: string, - * threadMessages: string[] + * threadMessages: Array<{message: string, files: Array>}> * }>} Rendered messages. */ async function renderMessengerReport({ core, reportsDir }) { const config = readMessengerConfigFromEnv(); - const { message, threadMessages } = buildMessengerMessages({ + const { message, threadMessages } = await buildMessengerMessages({ reportsDir: reportsDir || config.reportsDir, configuredClusters: config.configuredClusters, core, @@ -135,11 +142,17 @@ async function renderMessengerReport({ core, reportsDir }) { core.info(message); core.setOutput("message", message); - core.setOutput("thread_messages", JSON.stringify(threadMessages)); + core.setOutput( + "thread_messages", + JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message)) + ); if (config.loop) { try { - await makeThreadedReportInLoop({ message, threadMessages, loop: config.loop }, core); + await makeThreadedReportInLoop( + { message, threadMessages, loop: config.loop }, + core + ); } catch (error) { core.warning(`Unable to deliver report to Loop API: ${error.message}`); } diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 429a7c4f58..bed041f69b 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -13,7 +13,12 @@ const fs = require("fs"); const path = require("path"); +jest.mock("./messenger/charts/chart-renderer", () => ({ + renderClusterCharts: jest.fn().mockResolvedValue([]), +})); + const renderMessengerReport = require("./messenger-report"); +const { renderClusterCharts } = require("./messenger/charts/chart-renderer"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { createCore, withTempDir } = require("./shared/test-utils"); @@ -27,6 +32,8 @@ describe("messenger-report", () => { delete process.env.LOOP_CHANNEL_ID; delete process.env.LOOP_TOKEN; delete global.fetch; + renderClusterCharts.mockReset(); + renderClusterCharts.mockResolvedValue([]); }); test("reads normalized messenger config from env", () => { @@ -136,17 +143,21 @@ describe("messenger-report", () => { expect(result.message).toContain( "- [nfs](https://example.invalid/nfs): CONFIGURE SDN" ); + expect(result.message).not.toContain("### Top slowest tests"); expect(result.message).not.toContain("### Failed tests"); expect(result.threadMessages).toEqual([ - [ - "### Failed tests", - "", - "**[replicated](https://example.invalid/replicated)**", - "", - "| Tests | Reason |", - "|---|---|", - "| fails | Unexpected error: command timed out occurred |", - ].join("\n"), + { + message: [ + "### Failed tests", + "", + "**[replicated](https://example.invalid/replicated)**", + "", + "| Tests | Reason |", + "|---|---|", + "| fails | Unexpected error: command timed out occurred |", + ].join("\n"), + files: [], + }, ]); })); @@ -164,6 +175,67 @@ describe("messenger-report", () => { expect(result.threadMessages).toEqual([]); })); + test("renders top slowest tests and duration chart thread payloads", async () => + inTempDir(async (tempDir) => { + const chartFile = { + name: "replicated-top-slowest.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }; + renderClusterCharts.mockResolvedValue([chartFile]); + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 3, + skipped: 0, + failed: 0, + errors: 0, + total: 3, + successRate: 100, + }, + failedTests: [], + specTimings: [ + { name: "fast", group: "VM", state: "passed", runtimeMs: 1000 }, + { + name: "slow | pipe", + group: "Disk", + state: "passed", + runtimeMs: 90000, + }, + { name: "medium", group: "VM", state: "passed", runtimeMs: 30000 }, + ], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + + const core = createCore(); + const result = await renderMessengerReport({ core }); + + expect(result.message).toContain("### Top slowest tests"); + expect(result.message).toContain( + "| [replicated](https://example.invalid/replicated) | slow \\| pipe | 1m 30s |" + ); + expect(result.threadMessages).toEqual([ + { + message: expect.stringContaining("### Test durations"), + files: [chartFile], + }, + ]); + expect(core.setOutput).toHaveBeenCalledWith( + "thread_messages", + JSON.stringify([result.threadMessages[0].message]) + ); + })); + test("warns and skips report files that are missing storageType/cluster fields", async () => inTempDir(async (tempDir) => { fs.writeFileSync( @@ -264,8 +336,16 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.threadMessages).toEqual([ - "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| replicated | — |", - "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + { + message: + "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| replicated | — |", + files: [], + }, + { + message: + "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + files: [], + }, ]); })); @@ -306,16 +386,19 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.threadMessages).toEqual([ - [ - "### Failed tests", - "", - "**[nfs](https://example.invalid/nfs)**", - "", - "| Tests | Reason |", - "|---|---|", - "| VirtualMachineOperationRestore | — |", - "| VirtualMachineAdditionalNetworkInterfaces | — |", - ].join("\n"), + { + message: [ + "### Failed tests", + "", + "**[nfs](https://example.invalid/nfs)**", + "", + "| Tests | Reason |", + "|---|---|", + "| VirtualMachineOperationRestore | — |", + "| VirtualMachineAdditionalNetworkInterfaces | — |", + ].join("\n"), + files: [], + }, ]); })); @@ -448,6 +531,12 @@ describe("messenger-report", () => { test("posts main report and per-cluster failed tests thread via Loop API", async () => inTempDir(async (tempDir) => { + const chartFile = { + name: "replicated-top-slowest.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }; + renderClusterCharts.mockResolvedValue([chartFile]); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ @@ -466,6 +555,9 @@ describe("messenger-report", () => { successRate: 83.33, }, failedTests: ["[It] fails"], + specTimings: [ + { name: "slow", group: "VM", state: "failed", runtimeMs: 90000 }, + ], }) ); @@ -482,6 +574,11 @@ describe("messenger-report", () => { status: 201, text: async () => JSON.stringify({ id: "root-post-id" }), }) + .mockResolvedValueOnce({ + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-id" }] }), + }) .mockResolvedValueOnce({ ok: true, status: 201, @@ -490,7 +587,7 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); - expect(global.fetch).toHaveBeenCalledTimes(2); + expect(global.fetch).toHaveBeenCalledTimes(3); expect(global.fetch).toHaveBeenNthCalledWith( 1, "https://loop.example.invalid/api/v4/posts", @@ -506,11 +603,37 @@ describe("messenger-report", () => { channel_id: "channel-id", message: result.message, }); - expect(JSON.parse(global.fetch.mock.calls[1][1].body)).toEqual({ + expect(global.fetch).toHaveBeenNthCalledWith( + 2, + "https://loop.example.invalid/api/v4/files", + expect.objectContaining({ + method: "POST", + headers: { + Authorization: "Bearer loop-token", + }, + }) + ); + expect(JSON.parse(global.fetch.mock.calls[2][1].body)).toEqual({ channel_id: "channel-id", - message: - "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| fails | — |", + message: [ + "### Failed tests", + "", + "**[replicated](https://example.invalid/replicated)**", + "", + "| Tests | Reason |", + "|---|---|", + "| fails | — |", + "", + "### Test durations", + "", + "Attached charts:", + "- Top slowest specs", + "- Duration distribution", + "- Total duration by feature", + "- Duration by feature and status", + ].join("\n"), root_id: "root-post-id", + file_ids: ["file-id"], }); })); diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap new file mode 100644 index 0000000000..2a9e9877e6 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -0,0 +1,220 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`chart-config builds deterministic duration histogram config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#a371f7", + "data": Array [ + 1, + 1, + 0, + 1, + 1, + ], + "label": "Specs", + }, + ], + "labels": Array [ + "0-30s", + "30-60s", + "60-300s", + "300-600s", + ">600s", + ], + }, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E spec duration distribution", + }, + }, + "responsive": false, + "scales": Object { + "y": Object { + "beginAtZero": true, + "ticks": Object { + "precision": 0, + }, + }, + }, + }, + "type": "bar", + }, + "name": "duration-histogram", +} +`; + +exports[`chart-config builds deterministic feature totals config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#f0883e", + "data": Array [ + 601, + 311, + 60, + ], + "label": "Total duration, seconds", + }, + ], + "labels": Array [ + "Network", + "VM", + "Disk", + ], + }, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E duration by feature", + }, + }, + "responsive": false, + "scales": Object { + "y": Object { + "beginAtZero": true, + }, + }, + }, + "type": "bar", + }, + "name": "feature-totals", +} +`; + +exports[`chart-config builds deterministic status stacked config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#3fb950", + "data": Array [ + 0, + 0, + 10, + ], + "label": "passed", + }, + Object { + "backgroundColor": "#f85149", + "data": Array [ + 0, + 0, + 301, + ], + "label": "failed", + }, + Object { + "backgroundColor": "#d29922", + "data": Array [ + 0, + 601, + 0, + ], + "label": "errors", + }, + Object { + "backgroundColor": "#8b949e", + "data": Array [ + 60, + 0, + 0, + ], + "label": "skipped", + }, + ], + "labels": Array [ + "Disk", + "Network", + "VM", + ], + }, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E duration by feature and status", + }, + }, + "responsive": false, + "scales": Object { + "x": Object { + "stacked": true, + }, + "y": Object { + "beginAtZero": true, + "stacked": true, + }, + }, + }, + "type": "bar", + }, + "name": "status-stacked", +} +`; + +exports[`chart-config builds deterministic top-N config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#58a6ff", + "data": Array [ + 601, + 301, + 60, + ], + "label": "Duration, seconds", + }, + ], + "labels": Array [ + "error", + "slow fail", + "medium skip", + ], + }, + "options": Object { + "animation": false, + "indexAxis": "y", + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "Top slowest E2E specs", + }, + }, + "responsive": false, + "scales": Object { + "x": Object { + "beginAtZero": true, + }, + }, + }, + "type": "bar", + }, + "name": "top-slowest", +} +`; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js new file mode 100644 index 0000000000..7072697c05 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -0,0 +1,222 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const statusColors = { + passed: "#3fb950", + failed: "#f85149", + errors: "#d29922", + skipped: "#8b949e", +}; + +function normalizeTiming(timing) { + return { + name: String(timing.name || "Unnamed spec"), + group: String(timing.group || timing.name || "Ungrouped"), + state: String(timing.state || "errors"), + runtimeMs: Math.max(0, Number(timing.runtimeMs || 0)), + }; +} + +function seconds(ms) { + return Number((ms / 1000).toFixed(2)); +} + +function baseOptions(title, extra = {}) { + return { + responsive: false, + animation: false, + plugins: { + title: { + display: true, + text: title, + }, + legend: { + display: true, + }, + }, + ...extra, + }; +} + +function sortByRuntimeDesc(left, right) { + return ( + right.runtimeMs - left.runtimeMs || left.name.localeCompare(right.name) + ); +} + +function groupTotals(specTimings) { + const totals = new Map(); + for (const rawTiming of specTimings || []) { + const timing = normalizeTiming(rawTiming); + if (!totals.has(timing.group)) { + totals.set(timing.group, 0); + } + totals.set(timing.group, totals.get(timing.group) + timing.runtimeMs); + } + + return Array.from(totals, ([group, runtimeMs]) => ({ + group, + runtimeMs, + })).sort( + (left, right) => + right.runtimeMs - left.runtimeMs || left.group.localeCompare(right.group) + ); +} + +function buildTopNConfig(specTimings, n = 15) { + const timings = (specTimings || []) + .map(normalizeTiming) + .sort(sortByRuntimeDesc) + .slice(0, n); + + return { + name: "top-slowest", + config: { + type: "bar", + data: { + labels: timings.map((timing) => timing.name), + datasets: [ + { + label: "Duration, seconds", + data: timings.map((timing) => seconds(timing.runtimeMs)), + backgroundColor: "#58a6ff", + }, + ], + }, + options: baseOptions("Top slowest E2E specs", { + indexAxis: "y", + scales: { + x: { beginAtZero: true }, + }, + }), + }, + }; +} + +function buildDurationHistogramConfig( + specTimings, + buckets = [30, 60, 300, 600, Infinity] +) { + const counts = buckets.map(() => 0); + for (const rawTiming of specTimings || []) { + const durationSeconds = + Number(normalizeTiming(rawTiming).runtimeMs || 0) / 1000; + const bucketIndex = buckets.findIndex( + (bucket) => durationSeconds <= bucket + ); + counts[bucketIndex >= 0 ? bucketIndex : buckets.length - 1] += 1; + } + + let previous = 0; + const labels = buckets.map((bucket) => { + const label = + bucket === Infinity ? `>${previous}s` : `${previous}-${bucket}s`; + previous = bucket; + return label; + }); + + return { + name: "duration-histogram", + config: { + type: "bar", + data: { + labels, + datasets: [ + { + label: "Specs", + data: counts, + backgroundColor: "#a371f7", + }, + ], + }, + options: baseOptions("E2E spec duration distribution", { + scales: { + y: { beginAtZero: true, ticks: { precision: 0 } }, + }, + }), + }, + }; +} + +function buildFeatureTotalsConfig(specTimings) { + const totals = groupTotals(specTimings); + + return { + name: "feature-totals", + config: { + type: "bar", + data: { + labels: totals.map((entry) => entry.group), + datasets: [ + { + label: "Total duration, seconds", + data: totals.map((entry) => seconds(entry.runtimeMs)), + backgroundColor: "#f0883e", + }, + ], + }, + options: baseOptions("E2E duration by feature", { + scales: { + y: { beginAtZero: true }, + }, + }), + }, + }; +} + +function buildStatusStackedConfig(specTimings) { + const groups = new Map(); + for (const rawTiming of specTimings || []) { + const timing = normalizeTiming(rawTiming); + if (!groups.has(timing.group)) { + groups.set(timing.group, { passed: 0, failed: 0, errors: 0, skipped: 0 }); + } + const status = Object.prototype.hasOwnProperty.call( + statusColors, + timing.state + ) + ? timing.state + : "errors"; + groups.get(timing.group)[status] += timing.runtimeMs; + } + + const labels = Array.from(groups.keys()).sort(); + const statuses = ["passed", "failed", "errors", "skipped"]; + + return { + name: "status-stacked", + config: { + type: "bar", + data: { + labels, + datasets: statuses.map((status) => ({ + label: status, + data: labels.map((label) => seconds(groups.get(label)[status])), + backgroundColor: statusColors[status], + })), + }, + options: baseOptions("E2E duration by feature and status", { + scales: { + x: { stacked: true }, + y: { beginAtZero: true, stacked: true }, + }, + }), + }, + }; +} + +module.exports = { + buildDurationHistogramConfig, + buildFeatureTotalsConfig, + buildStatusStackedConfig, + buildTopNConfig, +}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js new file mode 100644 index 0000000000..adc26ca8db --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js @@ -0,0 +1,43 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + buildDurationHistogramConfig, + buildFeatureTotalsConfig, + buildStatusStackedConfig, + buildTopNConfig, +} = require("./chart-config"); + +const specTimings = [ + { name: "fast pass", group: "VM", state: "passed", runtimeMs: 10_000 }, + { name: "medium skip", group: "Disk", state: "skipped", runtimeMs: 60_000 }, + { name: "slow fail", group: "VM", state: "failed", runtimeMs: 301_000 }, + { name: "error", group: "Network", state: "errors", runtimeMs: 601_000 }, +]; + +describe("chart-config", () => { + test("builds deterministic top-N config", () => { + expect(buildTopNConfig(specTimings, 3)).toMatchSnapshot(); + }); + + test("builds deterministic duration histogram config", () => { + expect(buildDurationHistogramConfig(specTimings)).toMatchSnapshot(); + }); + + test("builds deterministic feature totals config", () => { + expect(buildFeatureTotalsConfig(specTimings)).toMatchSnapshot(); + }); + + test("builds deterministic status stacked config", () => { + expect(buildStatusStackedConfig(specTimings)).toMatchSnapshot(); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js new file mode 100644 index 0000000000..e944546321 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -0,0 +1,62 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + buildDurationHistogramConfig, + buildFeatureTotalsConfig, + buildStatusStackedConfig, + buildTopNConfig, +} = require("./chart-config"); + +let ChartJSNodeCanvas; + +function loadChartRenderer() { + if (!ChartJSNodeCanvas) { + ({ ChartJSNodeCanvas } = require("chartjs-node-canvas")); + } + + return new ChartJSNodeCanvas({ + width: 1280, + height: 720, + backgroundColour: "#ffffff", + }); +} + +async function renderClusterCharts(report) { + if ( + !Array.isArray(report && report.specTimings) || + report.specTimings.length === 0 + ) { + return []; + } + + const renderer = loadChartRenderer(); + const configs = [ + buildTopNConfig(report.specTimings), + buildDurationHistogramConfig(report.specTimings), + buildFeatureTotalsConfig(report.specTimings), + buildStatusStackedConfig(report.specTimings), + ]; + const clusterName = String(report.cluster || report.storageType || "cluster"); + + return Promise.all( + configs.map(async ({ name, config }) => ({ + name: `${clusterName}-${name}.png`, + buffer: await renderer.renderToBuffer(config, "image/png"), + mimeType: "image/png", + })) + ); +} + +module.exports = { + renderClusterCharts, +}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js new file mode 100644 index 0000000000..3551db51ec --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js @@ -0,0 +1,57 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +jest.mock("chartjs-node-canvas", () => ({ + ChartJSNodeCanvas: jest.fn().mockImplementation(() => ({ + renderToBuffer: jest.fn().mockResolvedValue(Buffer.from("png")), + })), +})); + +const { renderClusterCharts } = require("./chart-renderer"); + +describe("chart-renderer", () => { + test("returns no files when spec timings are empty", async () => { + await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); + }); + + test("renders four cluster chart images", async () => { + const files = await renderClusterCharts({ + cluster: "replicated", + specTimings: [ + { name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }, + ], + }); + + expect(files).toEqual([ + { + name: "replicated-top-slowest.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + { + name: "replicated-duration-histogram.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + { + name: "replicated-feature-totals.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + { + name: "replicated-status-stacked.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + ]); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 647651bea0..81ee8e7f50 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -26,7 +26,7 @@ /** * @typedef {Object} LoopPublishParams * @property {string} message - * @property {string[]} threadMessages + * @property {Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>} threadMessages * @property {LoopCredentials} loop */ @@ -60,20 +60,24 @@ function parseLoopApiPayload(responseText, core) { * @param {string} message Post body. * @param {string} [rootId] Optional thread root id for reply posts. * @param {LoopClientCore} core GitHub core API. + * @param {string[]} [fileIds] Uploaded Loop file ids to attach. * @returns {Promise>} Parsed Loop API response. */ -async function postToLoopApi(loop, message, rootId, core) { +async function postToLoopApi(loop, message, rootId, core, fileIds = []) { + const body = { + channel_id: loop.channelId, + message, + ...(rootId ? { root_id: rootId } : {}), + ...(fileIds.length > 0 ? { file_ids: fileIds } : {}), + }; + const response = await fetch(loop.apiUrl, { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, "Content-Type": "application/json", }, - body: JSON.stringify({ - channel_id: loop.channelId, - message, - ...(rootId ? { root_id: rootId } : {}), - }), + body: JSON.stringify(body), }); const responseText = await response.text(); @@ -88,6 +92,59 @@ async function postToLoopApi(loop, message, rootId, core) { return payload; } +function getFilesApiUrl(apiUrl) { + return String(apiUrl || "").replace(/\/posts$/, "/files"); +} + +/** + * Uploads a single file to Loop and returns the created file id. + * + * @param {LoopCredentials} loop Loop API credentials. + * @param {string} fileName File name shown in Loop. + * @param {Buffer} buffer File content. + * @param {LoopClientCore} core GitHub core API. + * @param {string} [mimeType] File MIME type. + * @returns {Promise} Uploaded Loop file id. + */ +async function uploadFileToLoop( + loop, + fileName, + buffer, + core, + mimeType = "image/png" +) { + const formData = new FormData(); + formData.append("channel_id", loop.channelId); + formData.append("files", new Blob([buffer], { type: mimeType }), fileName); + + const response = await fetch(getFilesApiUrl(loop.apiUrl), { + method: "POST", + headers: { + Authorization: `Bearer ${loop.token}`, + }, + body: formData, + }); + const responseText = await response.text(); + + if (!response.ok) { + throw new Error( + `Loop file upload failed with status ${response.status}: ${responseText}` + ); + } + + const payload = parseLoopApiPayload(responseText, core); + const fileId = + payload.file_infos && payload.file_infos[0] && payload.file_infos[0].id; + if (!fileId) { + throw new Error("Loop API did not return uploaded file id"); + } + + core.info( + `Loop API accepted file ${fileName} with status ${response.status}` + ); + return fileId; +} + /** * Publishes the main report and optional failed-tests thread to Loop. * @@ -95,7 +152,10 @@ async function postToLoopApi(loop, message, rootId, core) { * @param {LoopClientCore} core GitHub core API. * @returns {Promise} */ -async function makeThreadedReportInLoop({ message, threadMessages, loop }, core) { +async function makeThreadedReportInLoop( + { message, threadMessages, loop }, + core +) { const rootPost = await postToLoopApi(loop, message, undefined, core); if (!rootPost.id) { @@ -104,11 +164,18 @@ async function makeThreadedReportInLoop({ message, threadMessages, loop }, core) ); } - for (const replyMessage of threadMessages) { - await postToLoopApi(loop, replyMessage, rootPost.id, core); + for (const reply of threadMessages) { + const files = Array.isArray(reply.files) ? reply.files : []; + const fileIds = await Promise.all( + files.map((file) => + uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType) + ) + ); + await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds); } } module.exports = { makeThreadedReportInLoop, + uploadFileToLoop, }; diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js new file mode 100644 index 0000000000..1701e06a66 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -0,0 +1,56 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { uploadFileToLoop } = require("./loop-client"); +const { createCore } = require("../shared/test-utils"); + +describe("loop-client", () => { + afterEach(() => { + delete global.fetch; + }); + + test("uploads files to Loop multipart endpoint", async () => { + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-id" }] }), + }); + + const fileId = await uploadFileToLoop( + { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }, + "chart.png", + Buffer.from("image-bytes"), + createCore(), + "image/png" + ); + + expect(fileId).toBe("file-id"); + expect(global.fetch).toHaveBeenCalledWith( + "https://loop.example.invalid/api/v4/files", + expect.objectContaining({ + method: "POST", + headers: { + Authorization: "Bearer loop-token", + }, + }) + ); + + const body = global.fetch.mock.calls[0][1].body; + expect(body.get("channel_id")).toBe("channel-id"); + expect(body.get("files").name).toBe("chart.png"); + await expect(body.get("files").text()).resolves.toBe("image-bytes"); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index 893b15e45a..f44a38b171 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -36,6 +36,17 @@ function formatRate(value) { return `${Number.isFinite(rate) ? rate.toFixed(2) : "0.00"}%`; } +function formatDuration(runtimeMs) { + const durationSeconds = Number(runtimeMs || 0) / 1000; + if (durationSeconds < 60) { + return `${durationSeconds.toFixed(1)}s`; + } + + const minutes = Math.floor(durationSeconds / 60); + const seconds = Math.round(durationSeconds % 60); + return `${minutes}m ${seconds}s`; +} + function formatClusterLink(report) { const clusterName = sanitizeCell(report.cluster || report.storageType); return report.workflowRunUrl @@ -165,6 +176,65 @@ function renderTestResultsSection(testsReports) { return ["### Test results", "", ...rows, ""]; } +function renderDurationBar(runtimeMs, maxRuntimeMs, width = 10) { + if (!maxRuntimeMs) { + return ""; + } + + const filled = Math.max( + 1, + Math.round((Number(runtimeMs || 0) / maxRuntimeMs) * width) + ); + return `${"█".repeat(Math.min(width, filled))}${"░".repeat( + Math.max(0, width - filled) + )}`; +} + +function renderTopSlowestSection(testsReports) { + const rows = []; + + for (const report of testsReports) { + const timings = Array.isArray(report.specTimings) ? report.specTimings : []; + const topTimings = timings + .slice() + .sort( + (left, right) => + Number(right.runtimeMs || 0) - Number(left.runtimeMs || 0) || + String(left.name || "").localeCompare(String(right.name || "")) + ) + .slice(0, 3); + + for (const timing of topTimings) { + rows.push({ report, timing }); + } + } + + if (rows.length === 0) { + return []; + } + + const maxRuntimeMs = Math.max( + ...rows.map(({ timing }) => Number(timing.runtimeMs || 0)) + ); + const tableRows = [ + "| Cluster | Test | Duration | Bar |", + "|---|---|---:|---|", + ]; + + for (const { report, timing } of rows) { + tableRows.push( + buildMarkdownRow([ + formatClusterLink(report), + sanitizeCell(timing.name), + formatDuration(timing.runtimeMs), + renderDurationBar(timing.runtimeMs, maxRuntimeMs), + ]) + ); + } + + return ["### Top slowest tests", "", ...tableRows, ""]; +} + /** * Renders a `### ` section followed by a bullet list of * `- <cluster link>: <message>` rows, one per report. Returns an empty @@ -233,6 +303,7 @@ function buildMainMessage(orderedReports) { getMissingReportMessage ), ...renderTestResultsSection(testsReports), + ...renderTopSlowestSection(testsReports), ]; return lines.join("\n").trim(); @@ -337,28 +408,90 @@ function renderFailedTestsThreadMessage(report) { return lines.join("\n"); } +function hasSpecTimings(report) { + return Array.isArray(report.specTimings) && report.specTimings.length > 0; +} + +function renderChartCaption(files, chartsUnavailable) { + if (files.length === 0 && !chartsUnavailable) { + return ""; + } + + const lines = ["### Test durations"]; + if (files.length > 0) { + lines.push(""); + lines.push("Attached charts:"); + lines.push("- Top slowest specs"); + lines.push("- Duration distribution"); + lines.push("- Total duration by feature"); + lines.push("- Duration by feature and status"); + } + if (chartsUnavailable) { + lines.push(""); + lines.push("Charts unavailable."); + } + + return lines.join("\n"); +} + /** - * Builds optional failed-tests thread messages for clusters with failed tests. + * Builds optional per-cluster thread messages for failed tests and duration charts. * * @param {Array<Record<string, any>>} orderedReports Cluster reports in display order. - * @returns {string[]} Markdown thread message bodies. + * @param {{renderClusterCharts?: function(Record<string, any>): Promise<Array<{name: string, buffer: Buffer, mimeType: string}>>}} [options] + * @returns {Promise<Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>>} Markdown thread payloads. */ -function buildThreadMessages(orderedReports) { +async function buildThreadMessages( + orderedReports, + { renderClusterCharts } = {} +) { const testsReports = orderedReports.filter((report) => isTestResultReport(report) ); - const failedTestReports = testsReports.filter(hasFailedTests); + const threadMessages = []; + let renderedFailedTestsHeading = false; - if (failedTestReports.length === 0) { - return []; + for (const report of testsReports) { + const messageParts = []; + let files = []; + let chartsUnavailable = false; + + if (renderClusterCharts && hasSpecTimings(report)) { + try { + files = await renderClusterCharts(report); + } catch { + chartsUnavailable = true; + } + } + + if (!hasFailedTests(report) && files.length === 0 && !chartsUnavailable) { + continue; + } + + if (hasFailedTests(report)) { + const clusterMessage = renderFailedTestsThreadMessage(report); + messageParts.push( + renderedFailedTestsHeading + ? clusterMessage + : ["### Failed tests", clusterMessage].join("\n\n") + ); + renderedFailedTestsHeading = true; + } else { + messageParts.push(`**${formatClusterLink(report)}**`); + } + + const chartCaption = renderChartCaption(files, chartsUnavailable); + if (chartCaption) { + messageParts.push(chartCaption); + } + + threadMessages.push({ + message: messageParts.join("\n\n"), + files, + }); } - return failedTestReports.map((report, index) => { - const clusterMessage = renderFailedTestsThreadMessage(report); - return index === 0 - ? ["### Failed tests", clusterMessage].join("\n\n") - : clusterMessage; - }); + return threadMessages; } module.exports = { diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index 3d3f5b2940..eca0925605 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -72,10 +72,12 @@ function formatSpecName(specReport) { .map((part) => String(part || "").trim()) .filter(Boolean); const leafText = String(specReport.LeafNodeText || "").trim(); - const labels = [...new Set([ - ...flattenLabels(specReport.ContainerHierarchyLabels), - ...flattenLabels(specReport.LeafNodeLabels), - ])]; + const labels = [ + ...new Set([ + ...flattenLabels(specReport.ContainerHierarchyLabels), + ...flattenLabels(specReport.LeafNodeLabels), + ]), + ]; const labelSuffix = labels.map((label) => `[${label}]`).join(" "); const body = [...hierarchyParts, leafText].filter(Boolean).join(" "); @@ -86,6 +88,11 @@ function formatSpecName(specReport) { .trim(); } +function runtimeMs(value) { + const runtime = Number(value || 0); + return Number.isFinite(runtime) ? Math.round(runtime / 1_000_000) : 0; +} + /** * Maps a raw Ginkgo spec state into the metrics bucket used by the final * messenger report. @@ -124,12 +131,16 @@ function formatFailureReason(specReport) { const failureStates = new Set(["failed", "errors"]); function isSuiteNodeFailure(specReport) { - const leafNodeType = String((specReport && specReport.LeafNodeType) || "").trim(); + const leafNodeType = String( + (specReport && specReport.LeafNodeType) || "" + ).trim(); if (!leafNodeType || leafNodeType === "It") { return false; } - return failureStates.has(getMetricKeyForState(specReport && specReport.State)); + return failureStates.has( + getMetricKeyForState(specReport && specReport.State) + ); } function buildFailureDetail(specReport) { @@ -153,6 +164,8 @@ function buildFailureDetail(specReport) { * metrics: GinkgoMetrics, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array<{name: string, group: string, state: string, runtimeMs: number, labels: string[]}>, + * suiteTotalMs: number, * startedAt: string|null * }} Parsed report payload. */ @@ -161,10 +174,14 @@ function parseGinkgoReport(jsonContent) { const metrics = zeroMetrics(); const failedTests = []; const failedTestDetails = []; + const specTimings = []; const startedAt = suites.find((suite) => suite && suite.StartTime)?.StartTime || null; + let suiteTotalMs = 0; for (const suite of suites) { + suiteTotalMs += runtimeMs(suite && suite.RunTime); + for (const specReport of toArray(suite && suite.SpecReports)) { if (isSuiteNodeFailure(specReport)) { const failureDetail = buildFailureDetail(specReport); @@ -186,6 +203,20 @@ function parseGinkgoReport(jsonContent) { metrics.total += 1; const metricKey = getMetricKeyForState(specReport.State); metrics[metricKey] += 1; + const hierarchyParts = toArray(specReport.ContainerHierarchyTexts) + .map((part) => String(part || "").trim()) + .filter(Boolean); + const leafText = String(specReport.LeafNodeText || "").trim(); + specTimings.push({ + name: leafText, + group: hierarchyParts[0] || leafText, + state: metricKey, + runtimeMs: runtimeMs(specReport.RunTime), + labels: flattenLabels([ + ...toArray(specReport.ContainerHierarchyLabels), + ...toArray(specReport.LeafNodeLabels), + ]), + }); if (failureStates.has(metricKey)) { const failureDetail = buildFailureDetail(specReport); @@ -214,6 +245,8 @@ function parseGinkgoReport(jsonContent) { ]) ).values() ), + specTimings, + suiteTotalMs, startedAt, }; } @@ -330,6 +363,8 @@ function extractFailureReasonFromOutput(output, suiteNodeType) { * metrics: GinkgoMetrics, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: [], + * suiteTotalMs: 0, * startedAt: null, * }} Parsed fallback payload. */ @@ -340,6 +375,8 @@ function parseGinkgoOutput(outputContent) { metrics: zeroMetrics(), failedTests: [], failedTestDetails: [], + specTimings: [], + suiteTotalMs: 0, startedAt: null, }; diff --git a/.github/scripts/js/e2e/report/shared/report-model.js b/.github/scripts/js/e2e/report/shared/report-model.js index 3e8523e9e0..19fedf5cf2 100644 --- a/.github/scripts/js/e2e/report/shared/report-model.js +++ b/.github/scripts/js/e2e/report/shared/report-model.js @@ -54,12 +54,12 @@ function ginkgoOutputPattern(storageType) { } const stageMessage = { - "bootstrap": "BOOTSTRAP CLUSTER", + bootstrap: "BOOTSTRAP CLUSTER", "configure-sdn": "CONFIGURE SDN", "storage-setup": "STORAGE SETUP", "virtualization-setup": "VIRTUALIZATION SETUP", "e2e-test": "E2E TEST", - "ready": "CLUSTER READY", + ready: "CLUSTER READY", "artifact-missing": "TEST REPORTS NOT FOUND", }; @@ -94,7 +94,8 @@ const statusMessageTemplates = { }; function buildStatusMessage(status, stageLabel) { - const template = statusMessageTemplates[status] || statusMessageTemplates.failure; + const template = + statusMessageTemplates[status] || statusMessageTemplates.failure; return template.replace("%s", stageLabel); } diff --git a/.github/scripts/js/package.json b/.github/scripts/js/package.json index 7d57cc285c..b038118441 100644 --- a/.github/scripts/js/package.json +++ b/.github/scripts/js/package.json @@ -19,5 +19,9 @@ "eslint": "^10.2.1", "jest": "28.1.2", "prettier": "^2.5.0" + }, + "dependencies": { + "chart.js": "^4.5.1", + "chartjs-node-canvas": "^5.0.0" } } diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 9ba40daf57..feb0e895e5 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -488,6 +488,15 @@ jobs: path: downloaded-artifacts/ merge-multiple: false + - name: Set up Node.js for report rendering + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install E2E report dependencies + run: npm install + working-directory: .github/scripts/js + - name: Send results to channel id: render-report uses: actions/github-script@v7 From 4e194aaf714430b562da36e29398482a59548128 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Mon, 18 May 2026 18:21:47 +0300 Subject: [PATCH 02/24] fix(ci): log chart render errors and tighten e2e report install - Propagate core.warning from messenger-report.js into buildThreadMessages so duration-chart rendering failures are visible in CI logs instead of being silently masked by the "Charts unavailable." placeholder. - Cache ChartJSNodeCanvas as a module-level singleton; chart rendering is reused across clusters instead of allocating a fresh canvas each time. - Sanitize cluster name when composing chart file names so unexpected characters cannot leak into uploaded Loop attachment names. - Switch e2e-matrix workflow to npm ci with cache and document why node-version stays pinned to 20 (actions/github-script@v7 ABI). - Add jest case asserting core.warning is emitted when chart rendering rejects. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../scripts/js/e2e/report/messenger-report.js | 1 + .../js/e2e/report/messenger-report.test.js | 46 +++++++++++++++++++ .../report/messenger/charts/chart-renderer.js | 31 +++++++++---- .../js/e2e/report/messenger/markdown.js | 16 +++++-- .github/workflows/e2e-matrix.yml | 8 +++- 5 files changed, 89 insertions(+), 13 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index 1838a3d5ff..bb61b063fd 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -115,6 +115,7 @@ async function buildMessengerMessages({ const orderedReports = readReports(reportsDir, configuredClusters, core); const threadMessages = await buildThreadMessages(orderedReports, { renderClusterCharts, + core, }); return { message: buildMainMessage(orderedReports), diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index bed041f69b..da9cc00233 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -236,6 +236,52 @@ describe("messenger-report", () => { ); })); + test("warns and surfaces a placeholder when chart rendering fails", async () => + inTempDir(async (tempDir) => { + renderClusterCharts.mockRejectedValue(new Error("canvas unavailable")); + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 1, + skipped: 0, + failed: 0, + errors: 0, + total: 1, + successRate: 100, + }, + failedTests: [], + specTimings: [ + { name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }, + ], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + + const core = createCore(); + const result = await renderMessengerReport({ core }); + + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining( + "Unable to render duration charts for cluster replicated" + ) + ); + expect(result.threadMessages).toEqual([ + { + message: expect.stringContaining("Charts unavailable."), + files: [], + }, + ]); + })); + test("warns and skips report files that are missing storageType/cluster fields", async () => inTempDir(async (tempDir) => { fs.writeFileSync( diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js index e944546321..919a4d0db6 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -17,18 +17,29 @@ const { buildTopNConfig, } = require("./chart-config"); -let ChartJSNodeCanvas; +let canvasInstance; +// Module-level singleton: ChartJSNodeCanvas startup (loading chart.js + setting +// up the cairo-backed canvas) is non-trivial, and the renderer is stateless +// between renderToBuffer calls. Reusing it across clusters keeps memory usage +// flat when the messenger report grows. function loadChartRenderer() { - if (!ChartJSNodeCanvas) { - ({ ChartJSNodeCanvas } = require("chartjs-node-canvas")); + if (!canvasInstance) { + const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); + canvasInstance = new ChartJSNodeCanvas({ + width: 1280, + height: 720, + backgroundColour: "#ffffff", + }); } - return new ChartJSNodeCanvas({ - width: 1280, - height: 720, - backgroundColour: "#ffffff", - }); + return canvasInstance; +} + +function sanitizeFilenamePart(value) { + const fallback = "cluster"; + const safe = String(value || fallback).replace(/[^a-zA-Z0-9_-]+/g, "_"); + return safe || fallback; } async function renderClusterCharts(report) { @@ -46,7 +57,9 @@ async function renderClusterCharts(report) { buildFeatureTotalsConfig(report.specTimings), buildStatusStackedConfig(report.specTimings), ]; - const clusterName = String(report.cluster || report.storageType || "cluster"); + const clusterName = sanitizeFilenamePart( + report.cluster || report.storageType || "cluster" + ); return Promise.all( configs.map(async ({ name, config }) => ({ diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index f44a38b171..4cca867913 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -438,12 +438,15 @@ function renderChartCaption(files, chartsUnavailable) { * Builds optional per-cluster thread messages for failed tests and duration charts. * * @param {Array<Record<string, any>>} orderedReports Cluster reports in display order. - * @param {{renderClusterCharts?: function(Record<string, any>): Promise<Array<{name: string, buffer: Buffer, mimeType: string}>>}} [options] + * @param {{ + * renderClusterCharts?: function(Record<string, any>): Promise<Array<{name: string, buffer: Buffer, mimeType: string}>>, + * core?: {warning?: function(string): void} + * }} [options] * @returns {Promise<Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>>} Markdown thread payloads. */ async function buildThreadMessages( orderedReports, - { renderClusterCharts } = {} + { renderClusterCharts, core } = {} ) { const testsReports = orderedReports.filter((report) => isTestResultReport(report) @@ -459,8 +462,15 @@ async function buildThreadMessages( if (renderClusterCharts && hasSpecTimings(report)) { try { files = await renderClusterCharts(report); - } catch { + } catch (error) { chartsUnavailable = true; + if (core && typeof core.warning === "function") { + core.warning( + `Unable to render duration charts for cluster ${ + getReportClusterKey(report) || "unknown" + }: ${error.message}` + ); + } } } diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index feb0e895e5..814a70dda8 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -488,13 +488,19 @@ jobs: path: downloaded-artifacts/ merge-multiple: false + # Node 20 is required because actions/github-script@v7 ships with Node 20 + # and chartjs-node-canvas pulls a native `canvas` binding whose ABI must + # match the runtime that ultimately requires it. Bump together with + # actions/github-script when v8 (Node 22) is available. - name: Set up Node.js for report rendering uses: actions/setup-node@v4 with: node-version: 20 + cache: npm + cache-dependency-path: .github/scripts/js/package-lock.json - name: Install E2E report dependencies - run: npm install + run: npm ci working-directory: .github/scripts/js - name: Send results to channel From a40e567857aa967df9e7f51c45991e5b32490ed0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Mon, 18 May 2026 18:54:39 +0300 Subject: [PATCH 03/24] fix(ci, observability): keep Loop thread reply when chart upload fails If the Loop bot token lacks the upload_file permission, /api/v4/files returns HTTP 403 and the per-file uploadFileToLoop call throws. The exception used to propagate out of makeThreadedReportInLoop, aborting the whole thread and dropping the failed-tests reply that we delivered before charts were introduced. Wrap the per-reply upload in try/catch, log a warning explaining that attachments are skipped, and still post the reply text. The behaviour is now strictly additive to the pre-charts implementation: chart attachments are best-effort, the failed-tests table is guaranteed. Add a jest case that mocks /api/v4/files with a 403 and asserts the thread reply is posted without file_ids. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/messenger/loop-client.js | 24 ++++-- .../e2e/report/messenger/loop-client.test.js | 74 ++++++++++++++++++- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 81ee8e7f50..b8eaa4db07 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -166,11 +166,25 @@ async function makeThreadedReportInLoop( for (const reply of threadMessages) { const files = Array.isArray(reply.files) ? reply.files : []; - const fileIds = await Promise.all( - files.map((file) => - uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType) - ) - ); + let fileIds = []; + if (files.length > 0) { + try { + fileIds = await Promise.all( + files.map((file) => + uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType) + ) + ); + } catch (error) { + // Posting the reply without attachments is preferable to losing the + // whole thread (e.g. failed-tests table) when Loop rejects file + // uploads, typically with HTTP 403 when the bot token lacks the + // upload_file permission. + core.warning( + `Loop file upload failed; posting reply without attachments: ${error.message}` + ); + fileIds = []; + } + } await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds); } } diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 1701e06a66..58610acd29 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -10,7 +10,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { uploadFileToLoop } = require("./loop-client"); +const { + uploadFileToLoop, + makeThreadedReportInLoop, +} = require("./loop-client"); const { createCore } = require("../shared/test-utils"); describe("loop-client", () => { @@ -53,4 +56,73 @@ describe("loop-client", () => { expect(body.get("files").name).toBe("chart.png"); await expect(body.get("files").text()).resolves.toBe("image-bytes"); }); + + test("posts the reply without attachments when file upload fails", async () => { + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }; + const core = createCore(); + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: false, + status: 403, + text: async () => + JSON.stringify({ + id: "api.context.permissions.app_error", + message: "permission denied", + }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + global.fetch = jest.fn().mockImplementation(() => + Promise.resolve(responses.shift()) + ); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "chart.png", + buffer: Buffer.from("image-bytes"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + core + ); + + expect(global.fetch).toHaveBeenCalledTimes(3); + expect(global.fetch.mock.calls[0][0]).toBe(loop.apiUrl); + expect(global.fetch.mock.calls[1][0]).toBe( + "https://loop.example.invalid/api/v4/files" + ); + expect(global.fetch.mock.calls[2][0]).toBe(loop.apiUrl); + + const replyBody = JSON.parse(global.fetch.mock.calls[2][1].body); + expect(replyBody.root_id).toBe("root-post-id"); + expect(replyBody.message).toBe("reply"); + expect(replyBody).not.toHaveProperty("file_ids"); + + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining("Loop file upload failed; posting reply without attachments") + ); + }); }); From d93883b37a1de0c0264e08c51d0c8cb663202eab Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Mon, 18 May 2026 19:10:27 +0300 Subject: [PATCH 04/24] refactor(ci, observability): align messenger report with main + charts only Per user feedback after a local run review: the main message has to match the pre-charts main-branch layout, and the per-cluster thread reply should attach only the chart PNGs without the extra "### Test durations / Attached charts: ..." caption block. - markdown.js: drop renderTopSlowestSection (and the formatDuration/renderDurationBar helpers it owned) and the call from buildMainMessage so "Top slowest tests" no longer appears in the main message. - markdown.js: shrink renderChartCaption to "Charts unavailable." only on chart-render error and an empty string otherwise, so a cluster with successful charts gets a clean **[cluster](url)** header plus file attachments. - messenger-report.test.js: refresh the chart and Loop scenarios to assert the new minimal thread layout and the absence of the Top-3 section in the main message. Failed-tests thread reply behavior is preserved: clusters with failures still get the "### Failed tests" table, just without the chart caption block underneath it. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/messenger-report.test.js | 23 ++--- .../js/e2e/report/messenger/markdown.js | 93 +------------------ 2 files changed, 11 insertions(+), 105 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index da9cc00233..2ea0124a6c 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -175,7 +175,7 @@ describe("messenger-report", () => { expect(result.threadMessages).toEqual([]); })); - test("renders top slowest tests and duration chart thread payloads", async () => + test("attaches duration chart files to thread reply without a text caption", async () => inTempDir(async (tempDir) => { const chartFile = { name: "replicated-top-slowest.png", @@ -220,16 +220,19 @@ describe("messenger-report", () => { const core = createCore(); const result = await renderMessengerReport({ core }); - expect(result.message).toContain("### Top slowest tests"); - expect(result.message).toContain( - "| [replicated](https://example.invalid/replicated) | slow \\| pipe | 1m 30s |" - ); + expect(result.message).not.toContain("### Top slowest tests"); expect(result.threadMessages).toEqual([ { - message: expect.stringContaining("### Test durations"), + message: "**[replicated](https://example.invalid/replicated)**", files: [chartFile], }, ]); + expect(result.threadMessages[0].message).not.toContain( + "### Test durations" + ); + expect(result.threadMessages[0].message).not.toContain( + "Attached charts:" + ); expect(core.setOutput).toHaveBeenCalledWith( "thread_messages", JSON.stringify([result.threadMessages[0].message]) @@ -669,14 +672,6 @@ describe("messenger-report", () => { "| Tests | Reason |", "|---|---|", "| fails | — |", - "", - "### Test durations", - "", - "Attached charts:", - "- Top slowest specs", - "- Duration distribution", - "- Total duration by feature", - "- Duration by feature and status", ].join("\n"), root_id: "root-post-id", file_ids: ["file-id"], diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index 4cca867913..29cc6c9948 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -36,17 +36,6 @@ function formatRate(value) { return `${Number.isFinite(rate) ? rate.toFixed(2) : "0.00"}%`; } -function formatDuration(runtimeMs) { - const durationSeconds = Number(runtimeMs || 0) / 1000; - if (durationSeconds < 60) { - return `${durationSeconds.toFixed(1)}s`; - } - - const minutes = Math.floor(durationSeconds / 60); - const seconds = Math.round(durationSeconds % 60); - return `${minutes}m ${seconds}s`; -} - function formatClusterLink(report) { const clusterName = sanitizeCell(report.cluster || report.storageType); return report.workflowRunUrl @@ -176,65 +165,6 @@ function renderTestResultsSection(testsReports) { return ["### Test results", "", ...rows, ""]; } -function renderDurationBar(runtimeMs, maxRuntimeMs, width = 10) { - if (!maxRuntimeMs) { - return ""; - } - - const filled = Math.max( - 1, - Math.round((Number(runtimeMs || 0) / maxRuntimeMs) * width) - ); - return `${"█".repeat(Math.min(width, filled))}${"░".repeat( - Math.max(0, width - filled) - )}`; -} - -function renderTopSlowestSection(testsReports) { - const rows = []; - - for (const report of testsReports) { - const timings = Array.isArray(report.specTimings) ? report.specTimings : []; - const topTimings = timings - .slice() - .sort( - (left, right) => - Number(right.runtimeMs || 0) - Number(left.runtimeMs || 0) || - String(left.name || "").localeCompare(String(right.name || "")) - ) - .slice(0, 3); - - for (const timing of topTimings) { - rows.push({ report, timing }); - } - } - - if (rows.length === 0) { - return []; - } - - const maxRuntimeMs = Math.max( - ...rows.map(({ timing }) => Number(timing.runtimeMs || 0)) - ); - const tableRows = [ - "| Cluster | Test | Duration | Bar |", - "|---|---|---:|---|", - ]; - - for (const { report, timing } of rows) { - tableRows.push( - buildMarkdownRow([ - formatClusterLink(report), - sanitizeCell(timing.name), - formatDuration(timing.runtimeMs), - renderDurationBar(timing.runtimeMs, maxRuntimeMs), - ]) - ); - } - - return ["### Top slowest tests", "", ...tableRows, ""]; -} - /** * Renders a `### <title>` section followed by a bullet list of * `- <cluster link>: <message>` rows, one per report. Returns an empty @@ -303,7 +233,6 @@ function buildMainMessage(orderedReports) { getMissingReportMessage ), ...renderTestResultsSection(testsReports), - ...renderTopSlowestSection(testsReports), ]; return lines.join("\n").trim(); @@ -412,26 +341,8 @@ function hasSpecTimings(report) { return Array.isArray(report.specTimings) && report.specTimings.length > 0; } -function renderChartCaption(files, chartsUnavailable) { - if (files.length === 0 && !chartsUnavailable) { - return ""; - } - - const lines = ["### Test durations"]; - if (files.length > 0) { - lines.push(""); - lines.push("Attached charts:"); - lines.push("- Top slowest specs"); - lines.push("- Duration distribution"); - lines.push("- Total duration by feature"); - lines.push("- Duration by feature and status"); - } - if (chartsUnavailable) { - lines.push(""); - lines.push("Charts unavailable."); - } - - return lines.join("\n"); +function renderChartCaption(_files, chartsUnavailable) { + return chartsUnavailable ? "Charts unavailable." : ""; } /** From ea29baa2824bab5d644c804fe5dc38816f5fd117 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Tue, 19 May 2026 11:03:19 +0300 Subject: [PATCH 05/24] fix adding graph for report Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../scripts/js/e2e/report/messenger-report.js | 3 + .../js/e2e/report/messenger-report.test.js | 47 +++++++ .../scripts/js/e2e/report/messenger/config.js | 16 ++- .../js/e2e/report/messenger/loop-client.js | 6 +- .../e2e/report/messenger/loop-client.test.js | 130 ++++++++++++++++-- 5 files changed, 190 insertions(+), 12 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index bb61b063fd..20eec223be 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -156,6 +156,9 @@ async function renderMessengerReport({ core, reportsDir }) { ); } catch (error) { core.warning(`Unable to deliver report to Loop API: ${error.message}`); + if (config.loop.strictDelivery) { + throw error; + } } } diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 2ea0124a6c..1d96e047b7 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -31,6 +31,8 @@ describe("messenger-report", () => { delete process.env.LOOP_API_BASE_URL; delete process.env.LOOP_CHANNEL_ID; delete process.env.LOOP_TOKEN; + delete process.env.LOOP_STRICT_DELIVERY; + delete process.env.LOOP_STRICT_FILE_UPLOAD; delete global.fetch; renderClusterCharts.mockReset(); renderClusterCharts.mockResolvedValue([]); @@ -51,6 +53,8 @@ describe("messenger-report", () => { apiUrl: "https://loop.example.invalid/api/v4/posts", channelId: "channel-id", token: "token", + strictDelivery: false, + strictFileUploads: false, }, }); }); @@ -818,4 +822,47 @@ describe("messenger-report", () => { "Unable to deliver report to Loop API: Loop API request failed with status 500: server exploded" ); })); + + test("fails local delivery when strict Loop delivery mode is enabled", async () => + inTempDir(async (tempDir) => { + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 11, + skipped: 0, + failed: 0, + errors: 0, + total: 11, + successRate: 100, + }, + failedTests: [], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + process.env.LOOP_API_BASE_URL = "https://loop.example.invalid"; + process.env.LOOP_CHANNEL_ID = "channel-id"; + process.env.LOOP_TOKEN = "loop-token"; + process.env.LOOP_STRICT_DELIVERY = "1"; + + global.fetch = jest.fn().mockResolvedValue({ + ok: false, + status: 500, + text: async () => "server exploded", + }); + + await expect( + renderMessengerReport({ core: createCore() }) + ).rejects.toThrow( + "Loop API request failed with status 500: server exploded" + ); + })); }); diff --git a/.github/scripts/js/e2e/report/messenger/config.js b/.github/scripts/js/e2e/report/messenger/config.js index 12fa7dc659..cb31abde04 100644 --- a/.github/scripts/js/e2e/report/messenger/config.js +++ b/.github/scripts/js/e2e/report/messenger/config.js @@ -57,6 +57,10 @@ function parseConfiguredClusters(value) { } } +function parseBooleanEnv(value) { + return ["1", "true", "yes"].includes(String(value || "").toLowerCase()); +} + /** * Reads Loop credentials from the environment. * @@ -66,7 +70,7 @@ function parseConfiguredClusters(value) { * mistake and should surface as an error rather than a silent no-op. * * @param {NodeJS.ProcessEnv} [env=process.env] Environment variables source. - * @returns {{ apiUrl: string, channelId: string, token: string } | null} + * @returns {{ apiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null} */ function readLoopConfig(env = process.env) { const apiUrl = normalizeLoopApiBaseUrl(env.LOOP_API_BASE_URL); @@ -81,7 +85,13 @@ function readLoopConfig(env = process.env) { "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" ); } - return { apiUrl, channelId, token }; + return { + apiUrl, + channelId, + token, + strictDelivery: parseBooleanEnv(env.LOOP_STRICT_DELIVERY), + strictFileUploads: parseBooleanEnv(env.LOOP_STRICT_FILE_UPLOAD), + }; } /** @@ -91,7 +101,7 @@ function readLoopConfig(env = process.env) { * @returns {{ * reportsDir: string, * configuredClusters: string[], - * loop: { apiUrl: string, channelId: string, token: string } | null + * loop: { apiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null * }} Normalized messenger configuration. */ function readMessengerConfigFromEnv(env = process.env) { diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index b8eaa4db07..c6e369d131 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -27,7 +27,7 @@ * @typedef {Object} LoopPublishParams * @property {string} message * @property {Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>} threadMessages - * @property {LoopCredentials} loop + * @property {LoopCredentials & {strictFileUploads?: boolean}} loop */ /** @@ -175,6 +175,10 @@ async function makeThreadedReportInLoop( ) ); } catch (error) { + if (loop.strictFileUploads) { + throw error; + } + // Posting the reply without attachments is preferable to losing the // whole thread (e.g. failed-tests table) when Loop rejects file // uploads, typically with HTTP 403 when the bot token lacks the diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 58610acd29..de64cf72a0 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -10,10 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { - uploadFileToLoop, - makeThreadedReportInLoop, -} = require("./loop-client"); +const { uploadFileToLoop, makeThreadedReportInLoop } = require("./loop-client"); const { createCore } = require("../shared/test-utils"); describe("loop-client", () => { @@ -57,6 +54,72 @@ describe("loop-client", () => { await expect(body.get("files").text()).resolves.toBe("image-bytes"); }); + test("posts the reply with uploaded chart file ids", async () => { + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }; + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-one" }] }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-two" }] }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "top-slowest.png", + buffer: Buffer.from("one"), + mimeType: "image/png", + }, + { + name: "status-stacked.png", + buffer: Buffer.from("two"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + createCore() + ); + + expect(global.fetch).toHaveBeenCalledTimes(4); + expect(JSON.parse(global.fetch.mock.calls[3][1].body)).toEqual({ + channel_id: "channel-id", + message: "reply", + root_id: "root-post-id", + file_ids: ["file-one", "file-two"], + }); + }); + test("posts the reply without attachments when file upload fails", async () => { const loop = { apiUrl: "https://loop.example.invalid/api/v4/posts", @@ -85,9 +148,9 @@ describe("loop-client", () => { text: async () => JSON.stringify({ id: "reply-post-id" }), }, ]; - global.fetch = jest.fn().mockImplementation(() => - Promise.resolve(responses.shift()) - ); + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); await makeThreadedReportInLoop( { @@ -122,7 +185,58 @@ describe("loop-client", () => { expect(replyBody).not.toHaveProperty("file_ids"); expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining("Loop file upload failed; posting reply without attachments") + expect.stringContaining( + "Loop file upload failed; posting reply without attachments" + ) + ); + }); + + test("fails when strict file upload mode is enabled", async () => { + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + strictFileUploads: true, + }; + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: false, + status: 403, + text: async () => "permission denied", + }, + ]; + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await expect( + makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "chart.png", + buffer: Buffer.from("image-bytes"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + createCore() + ) + ).rejects.toThrow( + "Loop file upload failed with status 403: permission denied" ); + expect(global.fetch).toHaveBeenCalledTimes(2); }); }); From 098b0c50628a8c053d2a737dacbed630a59b67ef Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Wed, 20 May 2026 20:10:52 +0300 Subject: [PATCH 06/24] feat(ci, observability): replace e2e cluster chart set with five new visualizations Replaces the four ad-hoc cluster charts (top-slowest, duration-histogram, feature-totals, status-stacked) with a curated set of five: 1. status-doughnut - passed/failed/errors/skipped counts at a glance 2. pareto-slowest - top-N slowest specs + cumulative % of suite time 3. pass-rate-per-feature - 100% stacked horizontal bar, most-broken on top 4. quantiles-per-feature - p50/p90/max duration per feature 5. feature-totals - total duration per feature Also simplifies chart-config.js: a single pass aggregator feeds every chart, shared status and palette constants live at the top of the module, and only buildClusterChartConfigs(specTimings) is exported. The renderer now drives chart names from the builder list instead of hardcoding them. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/messenger-report.test.js | 4 +- .../__snapshots__/chart-config.test.js.snap | 485 +++++++++++------- .../report/messenger/charts/chart-config.js | 367 ++++++++----- .../messenger/charts/chart-config.test.js | 45 +- .../report/messenger/charts/chart-renderer.js | 14 +- .../messenger/charts/chart-renderer.test.js | 35 +- .../e2e/report/messenger/loop-client.test.js | 4 +- 7 files changed, 586 insertions(+), 368 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 1d96e047b7..8186b11c54 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -182,7 +182,7 @@ describe("messenger-report", () => { test("attaches duration chart files to thread reply without a text caption", async () => inTempDir(async (tempDir) => { const chartFile = { - name: "replicated-top-slowest.png", + name: "replicated-pareto-slowest.png", buffer: Buffer.from("png"), mimeType: "image/png", }; @@ -585,7 +585,7 @@ describe("messenger-report", () => { test("posts main report and per-cluster failed tests thread via Loop API", async () => inTempDir(async (tempDir) => { const chartFile = { - name: "replicated-top-slowest.png", + name: "replicated-pareto-slowest.png", buffer: Buffer.from("png"), mimeType: "image/png", }; diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 2a9e9877e6..869a623d1c 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -1,220 +1,321 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`chart-config builds deterministic duration histogram config 1`] = ` -Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#a371f7", - "data": Array [ - 1, - 1, - 0, - 1, - 1, - ], - "label": "Specs", - }, - ], - "labels": Array [ - "0-30s", - "30-60s", - "60-300s", - "300-600s", - ">600s", - ], - }, - "options": Object { - "animation": false, - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "E2E spec duration distribution", - }, +exports[`chart-config builds deterministic cluster chart configs 1`] = ` +Array [ + Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": Array [ + "#3fb950", + "#f85149", + "#d29922", + "#8b949e", + ], + "data": Array [ + 2, + 1, + 1, + 1, + ], + }, + ], + "labels": Array [ + "passed", + "failed", + "errors", + "skipped", + ], }, - "responsive": false, - "scales": Object { - "y": Object { - "beginAtZero": true, - "ticks": Object { - "precision": 0, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E spec status distribution", }, }, + "responsive": false, }, + "type": "doughnut", }, - "type": "bar", + "name": "status-doughnut", }, - "name": "duration-histogram", -} -`; - -exports[`chart-config builds deterministic feature totals config 1`] = ` -Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#f0883e", - "data": Array [ - 601, - 311, - 60, - ], - "label": "Total duration, seconds", - }, - ], - "labels": Array [ - "Network", - "VM", - "Disk", - ], - }, - "options": Object { - "animation": false, - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "E2E duration by feature", - }, + Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#58a6ff", + "data": Array [ + 601, + 301, + 60, + 45, + 10, + ], + "label": "Duration, seconds", + "order": 2, + "type": "bar", + "xAxisID": "x", + }, + Object { + "backgroundColor": "#a371f7", + "borderColor": "#a371f7", + "data": Array [ + 59.1, + 88.7, + 94.6, + 99, + 100, + ], + "label": "Cumulative % of suite time", + "order": 1, + "type": "line", + "xAxisID": "x1", + }, + ], + "labels": Array [ + "error", + "slow fail", + "medium skip", + "passing peer", + "fast pass", + ], }, - "responsive": false, - "scales": Object { - "y": Object { - "beginAtZero": true, + "options": Object { + "animation": false, + "indexAxis": "y", + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "Top slowest E2E specs (Pareto)", + }, + }, + "responsive": false, + "scales": Object { + "x": Object { + "beginAtZero": true, + "position": "bottom", + "title": Object { + "display": true, + "text": "Duration, seconds", + }, + }, + "x1": Object { + "beginAtZero": true, + "grid": Object { + "drawOnChartArea": false, + }, + "max": 100, + "position": "top", + "title": Object { + "display": true, + "text": "Cumulative %", + }, + }, }, }, + "type": "bar", }, - "type": "bar", + "name": "pareto-slowest", }, - "name": "feature-totals", -} -`; - -exports[`chart-config builds deterministic status stacked config 1`] = ` -Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#3fb950", - "data": Array [ - 0, - 0, - 10, - ], - "label": "passed", - }, - Object { - "backgroundColor": "#f85149", - "data": Array [ - 0, - 0, - 301, - ], - "label": "failed", - }, - Object { - "backgroundColor": "#d29922", - "data": Array [ - 0, - 601, - 0, - ], - "label": "errors", + Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#3fb950", + "data": Array [ + 0, + 66.7, + 0, + ], + "label": "passed", + }, + Object { + "backgroundColor": "#f85149", + "data": Array [ + 0, + 33.3, + 0, + ], + "label": "failed", + }, + Object { + "backgroundColor": "#d29922", + "data": Array [ + 100, + 0, + 0, + ], + "label": "errors", + }, + Object { + "backgroundColor": "#8b949e", + "data": Array [ + 0, + 0, + 100, + ], + "label": "skipped", + }, + ], + "labels": Array [ + "Network", + "VM", + "Disk", + ], + }, + "options": Object { + "animation": false, + "indexAxis": "y", + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "Pass rate by feature, %", + }, }, - Object { - "backgroundColor": "#8b949e", - "data": Array [ - 60, - 0, - 0, - ], - "label": "skipped", + "responsive": false, + "scales": Object { + "x": Object { + "beginAtZero": true, + "max": 100, + "stacked": true, + "title": Object { + "display": true, + "text": "% of specs", + }, + }, + "y": Object { + "stacked": true, + }, }, - ], - "labels": Array [ - "Disk", - "Network", - "VM", - ], + }, + "type": "bar", }, - "options": Object { - "animation": false, - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "E2E duration by feature and status", - }, + "name": "pass-rate-per-feature", + }, + Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#58a6ff", + "data": Array [ + 601, + 45, + 60, + ], + "label": "p50", + }, + Object { + "backgroundColor": "#d29922", + "data": Array [ + 601, + 249.8, + 60, + ], + "label": "p90", + }, + Object { + "backgroundColor": "#f85149", + "data": Array [ + 601, + 301, + 60, + ], + "label": "max", + }, + ], + "labels": Array [ + "Network", + "VM", + "Disk", + ], }, - "responsive": false, - "scales": Object { - "x": Object { - "stacked": true, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "Spec duration p50/p90/max by feature, seconds", + }, }, - "y": Object { - "beginAtZero": true, - "stacked": true, + "responsive": false, + "scales": Object { + "y": Object { + "beginAtZero": true, + "title": Object { + "display": true, + "text": "Seconds", + }, + }, }, }, + "type": "bar", }, - "type": "bar", + "name": "quantiles-per-feature", }, - "name": "status-stacked", -} -`; - -exports[`chart-config builds deterministic top-N config 1`] = ` -Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#58a6ff", - "data": Array [ - 601, - 301, - 60, - ], - "label": "Duration, seconds", - }, - ], - "labels": Array [ - "error", - "slow fail", - "medium skip", - ], - }, - "options": Object { - "animation": false, - "indexAxis": "y", - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "Top slowest E2E specs", - }, + Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#f0883e", + "data": Array [ + 601, + 356, + 60, + ], + "label": "Total duration, seconds", + }, + ], + "labels": Array [ + "Network", + "VM", + "Disk", + ], }, - "responsive": false, - "scales": Object { - "x": Object { - "beginAtZero": true, + "options": Object { + "animation": false, + "indexAxis": "y", + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "Total duration by feature", + }, + }, + "responsive": false, + "scales": Object { + "x": Object { + "beginAtZero": true, + "title": Object { + "display": true, + "text": "Seconds", + }, + }, }, }, + "type": "bar", }, - "type": "bar", + "name": "feature-totals", }, - "name": "top-slowest", -} +] `; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index 7072697c05..f9bf3076b1 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -10,24 +10,90 @@ // See the License for the specific language governing permissions and // limitations under the License. -const statusColors = { +const STATUSES = ["passed", "failed", "errors", "skipped"]; + +const STATUS_COLORS = { passed: "#3fb950", failed: "#f85149", errors: "#d29922", skipped: "#8b949e", }; -function normalizeTiming(timing) { +const PALETTE = { + bar: "#58a6ff", + cumulative: "#a371f7", + total: "#f0883e", + p50: "#58a6ff", + p90: "#d29922", + max: "#f85149", +}; + +const DEFAULT_TOP_N = 15; + +function toSeconds(ms) { + return Number((ms / 1000).toFixed(2)); +} + +function normalize(timing) { + const rawState = String((timing && timing.state) || "errors"); + const rawGroup = (timing && (timing.group || timing.name)) || "Ungrouped"; return { - name: String(timing.name || "Unnamed spec"), - group: String(timing.group || timing.name || "Ungrouped"), - state: String(timing.state || "errors"), - runtimeMs: Math.max(0, Number(timing.runtimeMs || 0)), + name: String((timing && timing.name) || "Unnamed spec"), + group: String(rawGroup), + state: STATUSES.includes(rawState) ? rawState : "errors", + runtimeMs: Math.max(0, Number((timing && timing.runtimeMs) || 0)), }; } -function seconds(ms) { - return Number((ms / 1000).toFixed(2)); +// Linear-interpolated quantile over a numerically sorted (asc) array. +// Mirrors Excel's PERCENTILE.INC / numpy's default percentile method. +function quantile(sortedAsc, q) { + if (sortedAsc.length === 0) { + return 0; + } + if (sortedAsc.length === 1) { + return sortedAsc[0]; + } + const pos = (sortedAsc.length - 1) * q; + const base = Math.floor(pos); + const upper = sortedAsc[base + 1]; + return upper === undefined + ? sortedAsc[base] + : sortedAsc[base] + (pos - base) * (upper - sortedAsc[base]); +} + +function emptyStatusCount() { + return { passed: 0, failed: 0, errors: 0, skipped: 0 }; +} + +// Single pass over the spec timings feeds every chart builder below. +function aggregate(specTimings) { + const all = []; + const byGroup = new Map(); + const byStatus = emptyStatusCount(); + let totalMs = 0; + + for (const raw of specTimings || []) { + const timing = normalize(raw); + all.push(timing); + totalMs += timing.runtimeMs; + byStatus[timing.state] += 1; + + let bucket = byGroup.get(timing.group); + if (!bucket) { + bucket = { + durations: [], + statusCount: emptyStatusCount(), + total: 0, + }; + byGroup.set(timing.group, bucket); + } + bucket.durations.push(timing.runtimeMs); + bucket.statusCount[timing.state] += 1; + bucket.total += timing.runtimeMs; + } + + return { all, byGroup, byStatus, totalMs }; } function baseOptions(title, extra = {}) { @@ -35,188 +101,247 @@ function baseOptions(title, extra = {}) { responsive: false, animation: false, plugins: { - title: { - display: true, - text: title, - }, - legend: { - display: true, - }, + title: { display: true, text: title }, + legend: { display: true }, }, ...extra, }; } -function sortByRuntimeDesc(left, right) { - return ( - right.runtimeMs - left.runtimeMs || left.name.localeCompare(right.name) - ); +function statusDoughnut({ byStatus }) { + return { + name: "status-doughnut", + config: { + type: "doughnut", + data: { + labels: STATUSES, + datasets: [ + { + data: STATUSES.map((status) => byStatus[status]), + backgroundColor: STATUSES.map((status) => STATUS_COLORS[status]), + }, + ], + }, + options: baseOptions("E2E spec status distribution"), + }, + }; } -function groupTotals(specTimings) { - const totals = new Map(); - for (const rawTiming of specTimings || []) { - const timing = normalizeTiming(rawTiming); - if (!totals.has(timing.group)) { - totals.set(timing.group, 0); - } - totals.set(timing.group, totals.get(timing.group) + timing.runtimeMs); - } - - return Array.from(totals, ([group, runtimeMs]) => ({ - group, - runtimeMs, - })).sort( - (left, right) => - right.runtimeMs - left.runtimeMs || left.group.localeCompare(right.group) - ); -} +function paretoSlowest({ all, totalMs }, topN = DEFAULT_TOP_N) { + const top = [...all] + .sort( + (left, right) => + right.runtimeMs - left.runtimeMs || left.name.localeCompare(right.name) + ) + .slice(0, topN); -function buildTopNConfig(specTimings, n = 15) { - const timings = (specTimings || []) - .map(normalizeTiming) - .sort(sortByRuntimeDesc) - .slice(0, n); + let runningMs = 0; + const cumulativePercents = top.map((timing) => { + runningMs += timing.runtimeMs; + return totalMs > 0 ? Number(((runningMs / totalMs) * 100).toFixed(1)) : 0; + }); return { - name: "top-slowest", + name: "pareto-slowest", config: { type: "bar", data: { - labels: timings.map((timing) => timing.name), + labels: top.map((timing) => timing.name), datasets: [ { + type: "bar", label: "Duration, seconds", - data: timings.map((timing) => seconds(timing.runtimeMs)), - backgroundColor: "#58a6ff", + data: top.map((timing) => toSeconds(timing.runtimeMs)), + backgroundColor: PALETTE.bar, + xAxisID: "x", + order: 2, + }, + { + type: "line", + label: "Cumulative % of suite time", + data: cumulativePercents, + borderColor: PALETTE.cumulative, + backgroundColor: PALETTE.cumulative, + xAxisID: "x1", + order: 1, }, ], }, - options: baseOptions("Top slowest E2E specs", { + options: baseOptions("Top slowest E2E specs (Pareto)", { indexAxis: "y", scales: { - x: { beginAtZero: true }, + x: { + beginAtZero: true, + position: "bottom", + title: { display: true, text: "Duration, seconds" }, + }, + x1: { + beginAtZero: true, + max: 100, + position: "top", + grid: { drawOnChartArea: false }, + title: { display: true, text: "Cumulative %" }, + }, }, }), }, }; } -function buildDurationHistogramConfig( - specTimings, - buckets = [30, 60, 300, 600, Infinity] -) { - const counts = buckets.map(() => 0); - for (const rawTiming of specTimings || []) { - const durationSeconds = - Number(normalizeTiming(rawTiming).runtimeMs || 0) / 1000; - const bucketIndex = buckets.findIndex( - (bucket) => durationSeconds <= bucket - ); - counts[bucketIndex >= 0 ? bucketIndex : buckets.length - 1] += 1; - } +function sortedGroups(byGroup, compareFn) { + return [...byGroup.entries()].sort(compareFn); +} - let previous = 0; - const labels = buckets.map((bucket) => { - const label = - bucket === Infinity ? `>${previous}s` : `${previous}-${bucket}s`; - previous = bucket; - return label; +function passRatePerFeature({ byGroup }) { + // Most-broken features go to the top: failures desc, then total runtime desc, + // then alphabetical for a stable order. + const entries = sortedGroups(byGroup, (left, right) => { + const failsLeft = left[1].statusCount.failed + left[1].statusCount.errors; + const failsRight = + right[1].statusCount.failed + right[1].statusCount.errors; + return ( + failsRight - failsLeft || + right[1].total - left[1].total || + left[0].localeCompare(right[0]) + ); }); + const labels = entries.map(([name]) => name); + const datasets = STATUSES.map((status) => ({ + label: status, + data: entries.map(([, group]) => { + const total = STATUSES.reduce( + (sum, candidate) => sum + group.statusCount[candidate], + 0 + ); + return total > 0 + ? Number(((group.statusCount[status] / total) * 100).toFixed(1)) + : 0; + }), + backgroundColor: STATUS_COLORS[status], + })); + return { - name: "duration-histogram", + name: "pass-rate-per-feature", config: { type: "bar", - data: { - labels, - datasets: [ - { - label: "Specs", - data: counts, - backgroundColor: "#a371f7", - }, - ], - }, - options: baseOptions("E2E spec duration distribution", { + data: { labels, datasets }, + options: baseOptions("Pass rate by feature, %", { + indexAxis: "y", scales: { - y: { beginAtZero: true, ticks: { precision: 0 } }, + x: { + stacked: true, + beginAtZero: true, + max: 100, + title: { display: true, text: "% of specs" }, + }, + y: { stacked: true }, }, }), }, }; } -function buildFeatureTotalsConfig(specTimings) { - const totals = groupTotals(specTimings); +function quantilesPerFeature({ byGroup }) { + const entries = sortedGroups( + byGroup, + (left, right) => + right[1].total - left[1].total || left[0].localeCompare(right[0]) + ); + const sortedDurations = entries.map(([, group]) => + [...group.durations].sort((left, right) => left - right) + ); return { - name: "feature-totals", + name: "quantiles-per-feature", config: { type: "bar", data: { - labels: totals.map((entry) => entry.group), + labels: entries.map(([name]) => name), datasets: [ { - label: "Total duration, seconds", - data: totals.map((entry) => seconds(entry.runtimeMs)), - backgroundColor: "#f0883e", + label: "p50", + data: sortedDurations.map((durations) => + toSeconds(quantile(durations, 0.5)) + ), + backgroundColor: PALETTE.p50, + }, + { + label: "p90", + data: sortedDurations.map((durations) => + toSeconds(quantile(durations, 0.9)) + ), + backgroundColor: PALETTE.p90, + }, + { + label: "max", + data: sortedDurations.map((durations) => + toSeconds(durations[durations.length - 1] || 0) + ), + backgroundColor: PALETTE.max, }, ], }, - options: baseOptions("E2E duration by feature", { + options: baseOptions("Spec duration p50/p90/max by feature, seconds", { scales: { - y: { beginAtZero: true }, + y: { + beginAtZero: true, + title: { display: true, text: "Seconds" }, + }, }, }), }, }; } -function buildStatusStackedConfig(specTimings) { - const groups = new Map(); - for (const rawTiming of specTimings || []) { - const timing = normalizeTiming(rawTiming); - if (!groups.has(timing.group)) { - groups.set(timing.group, { passed: 0, failed: 0, errors: 0, skipped: 0 }); - } - const status = Object.prototype.hasOwnProperty.call( - statusColors, - timing.state - ) - ? timing.state - : "errors"; - groups.get(timing.group)[status] += timing.runtimeMs; - } - - const labels = Array.from(groups.keys()).sort(); - const statuses = ["passed", "failed", "errors", "skipped"]; +function featureTotals({ byGroup }) { + const entries = sortedGroups( + byGroup, + (left, right) => + right[1].total - left[1].total || left[0].localeCompare(right[0]) + ); return { - name: "status-stacked", + name: "feature-totals", config: { type: "bar", data: { - labels, - datasets: statuses.map((status) => ({ - label: status, - data: labels.map((label) => seconds(groups.get(label)[status])), - backgroundColor: statusColors[status], - })), + labels: entries.map(([name]) => name), + datasets: [ + { + label: "Total duration, seconds", + data: entries.map(([, group]) => toSeconds(group.total)), + backgroundColor: PALETTE.total, + }, + ], }, - options: baseOptions("E2E duration by feature and status", { + options: baseOptions("Total duration by feature", { + indexAxis: "y", scales: { - x: { stacked: true }, - y: { beginAtZero: true, stacked: true }, + x: { + beginAtZero: true, + title: { display: true, text: "Seconds" }, + }, }, }), }, }; } +// Order of charts matches the order of attachments in the messenger thread. +const CHART_BUILDERS = [ + statusDoughnut, + paretoSlowest, + passRatePerFeature, + quantilesPerFeature, + featureTotals, +]; + +function buildClusterChartConfigs(specTimings) { + const data = aggregate(specTimings); + return CHART_BUILDERS.map((build) => build(data)); +} + module.exports = { - buildDurationHistogramConfig, - buildFeatureTotalsConfig, - buildStatusStackedConfig, - buildTopNConfig, + buildClusterChartConfigs, }; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js index adc26ca8db..09a2c48d98 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js @@ -10,34 +10,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { - buildDurationHistogramConfig, - buildFeatureTotalsConfig, - buildStatusStackedConfig, - buildTopNConfig, -} = require("./chart-config"); +const { buildClusterChartConfigs } = require("./chart-config"); const specTimings = [ { name: "fast pass", group: "VM", state: "passed", runtimeMs: 10_000 }, { name: "medium skip", group: "Disk", state: "skipped", runtimeMs: 60_000 }, { name: "slow fail", group: "VM", state: "failed", runtimeMs: 301_000 }, { name: "error", group: "Network", state: "errors", runtimeMs: 601_000 }, + { name: "passing peer", group: "VM", state: "passed", runtimeMs: 45_000 }, ]; describe("chart-config", () => { - test("builds deterministic top-N config", () => { - expect(buildTopNConfig(specTimings, 3)).toMatchSnapshot(); + test("builds deterministic cluster chart configs", () => { + expect(buildClusterChartConfigs(specTimings)).toMatchSnapshot(); }); - test("builds deterministic duration histogram config", () => { - expect(buildDurationHistogramConfig(specTimings)).toMatchSnapshot(); + test("returns the five chart configs in display order", () => { + const configs = buildClusterChartConfigs(specTimings); + expect(configs.map(({ name }) => name)).toEqual([ + "status-doughnut", + "pareto-slowest", + "pass-rate-per-feature", + "quantiles-per-feature", + "feature-totals", + ]); }); - test("builds deterministic feature totals config", () => { - expect(buildFeatureTotalsConfig(specTimings)).toMatchSnapshot(); - }); - - test("builds deterministic status stacked config", () => { - expect(buildStatusStackedConfig(specTimings)).toMatchSnapshot(); + test("handles an empty spec timings list", () => { + const configs = buildClusterChartConfigs([]); + expect(configs).toHaveLength(5); + const labelsByName = Object.fromEntries( + configs.map(({ name, config }) => [name, config.data.labels]) + ); + expect(labelsByName["status-doughnut"]).toEqual([ + "passed", + "failed", + "errors", + "skipped", + ]); + expect(labelsByName["pareto-slowest"]).toEqual([]); + expect(labelsByName["pass-rate-per-feature"]).toEqual([]); + expect(labelsByName["quantiles-per-feature"]).toEqual([]); + expect(labelsByName["feature-totals"]).toEqual([]); }); }); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js index 919a4d0db6..c3104af2ff 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -10,12 +10,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { - buildDurationHistogramConfig, - buildFeatureTotalsConfig, - buildStatusStackedConfig, - buildTopNConfig, -} = require("./chart-config"); +const { buildClusterChartConfigs } = require("./chart-config"); let canvasInstance; @@ -51,12 +46,7 @@ async function renderClusterCharts(report) { } const renderer = loadChartRenderer(); - const configs = [ - buildTopNConfig(report.specTimings), - buildDurationHistogramConfig(report.specTimings), - buildFeatureTotalsConfig(report.specTimings), - buildStatusStackedConfig(report.specTimings), - ]; + const configs = buildClusterChartConfigs(report.specTimings); const clusterName = sanitizeFilenamePart( report.cluster || report.storageType || "cluster" ); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js index 3551db51ec..4fc41d478b 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js @@ -23,35 +23,24 @@ describe("chart-renderer", () => { await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); }); - test("renders four cluster chart images", async () => { + test("renders five cluster chart images", async () => { const files = await renderClusterCharts({ cluster: "replicated", specTimings: [ - { name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }, + { name: "slow", group: "VM", state: "passed", runtimeMs: 90_000 }, ], }); - expect(files).toEqual([ - { - name: "replicated-top-slowest.png", - buffer: Buffer.from("png"), - mimeType: "image/png", - }, - { - name: "replicated-duration-histogram.png", - buffer: Buffer.from("png"), - mimeType: "image/png", - }, - { - name: "replicated-feature-totals.png", - buffer: Buffer.from("png"), - mimeType: "image/png", - }, - { - name: "replicated-status-stacked.png", - buffer: Buffer.from("png"), - mimeType: "image/png", - }, + expect(files.map(({ name }) => name)).toEqual([ + "replicated-status-doughnut.png", + "replicated-pareto-slowest.png", + "replicated-pass-rate-per-feature.png", + "replicated-quantiles-per-feature.png", + "replicated-feature-totals.png", ]); + for (const file of files) { + expect(file.buffer).toEqual(Buffer.from("png")); + expect(file.mimeType).toBe("image/png"); + } }); }); diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index de64cf72a0..1026adac32 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -94,12 +94,12 @@ describe("loop-client", () => { message: "reply", files: [ { - name: "top-slowest.png", + name: "pareto-slowest.png", buffer: Buffer.from("one"), mimeType: "image/png", }, { - name: "status-stacked.png", + name: "pass-rate-per-feature.png", buffer: Buffer.from("two"), mimeType: "image/png", }, From 509a01ac8991b2d4b66918416743a41de5ef6766 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Wed, 20 May 2026 22:08:06 +0300 Subject: [PATCH 07/24] refactor(ci, observability): focus e2e charts on failures and durations Reworks the experimental five-chart set into four triage-oriented charts: feature duration by status, slowest specs, duration buckets, and failed plus slow specs. The previous doughnut, quantile chart, and pareto percentage line are removed because they added noise without improving failure diagnosis. Adds local value labels to the chart configs so generated PNGs show concrete seconds/counts without extra dependencies. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/messenger-report.test.js | 4 +- .../__snapshots__/chart-config.test.js.snap | 272 +++++++------ .../report/messenger/charts/chart-config.js | 357 ++++++++++-------- .../messenger/charts/chart-config.test.js | 29 +- .../messenger/charts/chart-renderer.test.js | 11 +- .../e2e/report/messenger/loop-client.test.js | 4 +- 6 files changed, 361 insertions(+), 316 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 8186b11c54..36a697b5f5 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -182,7 +182,7 @@ describe("messenger-report", () => { test("attaches duration chart files to thread reply without a text caption", async () => inTempDir(async (tempDir) => { const chartFile = { - name: "replicated-pareto-slowest.png", + name: "replicated-slowest-specs.png", buffer: Buffer.from("png"), mimeType: "image/png", }; @@ -585,7 +585,7 @@ describe("messenger-report", () => { test("posts main report and per-cluster failed tests thread via Loop API", async () => inTempDir(async (tempDir) => { const chartFile = { - name: "replicated-pareto-slowest.png", + name: "replicated-slowest-specs.png", buffer: Buffer.from("png"), mimeType: "image/png", }; diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 869a623d1c..55c2de7e80 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -7,50 +7,100 @@ Array [ "data": Object { "datasets": Array [ Object { - "backgroundColor": Array [ - "#3fb950", - "#f85149", - "#d29922", - "#8b949e", + "backgroundColor": "#3fb950", + "data": Array [ + 0, + 55, + 0, ], + "label": "passed", + }, + Object { + "backgroundColor": "#f85149", "data": Array [ - 2, - 1, - 1, - 1, + 0, + 301, + 0, ], + "label": "failed", + }, + Object { + "backgroundColor": "#d29922", + "data": Array [ + 601, + 0, + 0, + ], + "label": "errors", + }, + Object { + "backgroundColor": "#8b949e", + "data": Array [ + 0, + 0, + 60, + ], + "label": "skipped", }, ], "labels": Array [ - "passed", - "failed", - "errors", - "skipped", + "Network", + "VM", + "Disk", ], }, "options": Object { "animation": false, + "indexAxis": "y", "plugins": Object { "legend": Object { "display": true, }, "title": Object { "display": true, - "text": "E2E spec status distribution", + "text": "E2E duration by feature and status", + }, + "valueLabels": Object { + "formatter": [Function], }, }, "responsive": false, + "scales": Object { + "x": Object { + "beginAtZero": true, + "stacked": true, + "title": Object { + "display": true, + "text": "Duration, seconds", + }, + }, + "y": Object { + "stacked": true, + }, + }, }, - "type": "doughnut", + "plugins": Array [ + Object { + "afterDatasetsDraw": [Function], + "id": "valueLabels", + }, + ], + "type": "bar", }, - "name": "status-doughnut", + "name": "feature-duration-status", }, Object { "config": Object { "data": Object { "datasets": Array [ Object { - "backgroundColor": "#58a6ff", + "backgroundColor": Array [ + "#d29922", + "#f85149", + "#8b949e", + "#3fb950", + "#3fb950", + ], "data": Array [ 601, 301, @@ -59,32 +109,14 @@ Array [ 10, ], "label": "Duration, seconds", - "order": 2, - "type": "bar", - "xAxisID": "x", - }, - Object { - "backgroundColor": "#a371f7", - "borderColor": "#a371f7", - "data": Array [ - 59.1, - 88.7, - 94.6, - 99, - 100, - ], - "label": "Cumulative % of suite time", - "order": 1, - "type": "line", - "xAxisID": "x1", }, ], "labels": Array [ - "error", - "slow fail", - "medium skip", - "passing peer", - "fast pass", + "Network / error", + "VM / slow fail", + "Disk / medium skip", + "VM / passing peer", + "VM / fast pass", ], }, "options": Object { @@ -92,40 +124,36 @@ Array [ "indexAxis": "y", "plugins": Object { "legend": Object { - "display": true, + "display": false, }, "title": Object { "display": true, - "text": "Top slowest E2E specs (Pareto)", + "text": "Top slowest E2E specs", + }, + "valueLabels": Object { + "formatter": [Function], }, }, "responsive": false, "scales": Object { "x": Object { "beginAtZero": true, - "position": "bottom", "title": Object { "display": true, "text": "Duration, seconds", }, }, - "x1": Object { - "beginAtZero": true, - "grid": Object { - "drawOnChartArea": false, - }, - "max": 100, - "position": "top", - "title": Object { - "display": true, - "text": "Cumulative %", - }, - }, }, }, + "plugins": Array [ + Object { + "afterDatasetsDraw": [Function], + "id": "valueLabels", + }, + ], "type": "bar", }, - "name": "pareto-slowest", + "name": "slowest-specs", }, Object { "config": Object { @@ -135,16 +163,16 @@ Array [ "backgroundColor": "#3fb950", "data": Array [ 0, - 66.7, 0, + 2, ], "label": "passed", }, Object { "backgroundColor": "#f85149", "data": Array [ + 1, 0, - 33.3, 0, ], "label": "failed", @@ -152,7 +180,7 @@ Array [ Object { "backgroundColor": "#d29922", "data": Array [ - 100, + 1, 0, 0, ], @@ -162,16 +190,16 @@ Array [ "backgroundColor": "#8b949e", "data": Array [ 0, + 1, 0, - 100, ], "label": "skipped", }, ], "labels": Array [ - "Network", - "VM", - "Disk", + "Slow >300s", + "Medium 60-300s", + "Fast <60s", ], }, "options": Object { @@ -183,18 +211,23 @@ Array [ }, "title": Object { "display": true, - "text": "Pass rate by feature, %", + "text": "E2E specs by duration bucket and status", + }, + "valueLabels": Object { + "formatter": [Function], }, }, "responsive": false, "scales": Object { "x": Object { "beginAtZero": true, - "max": 100, "stacked": true, + "ticks": Object { + "precision": 0, + }, "title": Object { "display": true, - "text": "% of specs", + "text": "Specs", }, }, "y": Object { @@ -202,92 +235,44 @@ Array [ }, }, }, + "plugins": Array [ + Object { + "afterDatasetsDraw": [Function], + "id": "valueLabels", + }, + ], "type": "bar", }, - "name": "pass-rate-per-feature", + "name": "duration-buckets", }, Object { "config": Object { "data": Object { "datasets": Array [ Object { - "backgroundColor": "#58a6ff", - "data": Array [ - 601, - 45, - 60, - ], - "label": "p50", - }, - Object { - "backgroundColor": "#d29922", - "data": Array [ - 601, - 249.8, - 60, + "backgroundColor": Array [ + "#d29922", + "#f85149", + "#8b949e", + "#3fb950", + "#3fb950", ], - "label": "p90", - }, - Object { - "backgroundColor": "#f85149", "data": Array [ 601, 301, 60, + 45, + 10, ], - "label": "max", - }, - ], - "labels": Array [ - "Network", - "VM", - "Disk", - ], - }, - "options": Object { - "animation": false, - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "Spec duration p50/p90/max by feature, seconds", - }, - }, - "responsive": false, - "scales": Object { - "y": Object { - "beginAtZero": true, - "title": Object { - "display": true, - "text": "Seconds", - }, - }, - }, - }, - "type": "bar", - }, - "name": "quantiles-per-feature", - }, - Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#f0883e", - "data": Array [ - 601, - 356, - 60, - ], - "label": "Total duration, seconds", + "label": "Duration, seconds", }, ], "labels": Array [ - "Network", - "VM", - "Disk", + "Network / error", + "VM / slow fail", + "Disk / medium skip", + "VM / passing peer", + "VM / fast pass", ], }, "options": Object { @@ -295,11 +280,14 @@ Array [ "indexAxis": "y", "plugins": Object { "legend": Object { - "display": true, + "display": false, }, "title": Object { "display": true, - "text": "Total duration by feature", + "text": "Failed/error specs and slowest successful specs", + }, + "valueLabels": Object { + "formatter": [Function], }, }, "responsive": false, @@ -308,14 +296,20 @@ Array [ "beginAtZero": true, "title": Object { "display": true, - "text": "Seconds", + "text": "Duration, seconds", }, }, }, }, + "plugins": Array [ + Object { + "afterDatasetsDraw": [Function], + "id": "valueLabels", + }, + ], "type": "bar", }, - "name": "feature-totals", + "name": "failed-and-slow-specs", }, ] `; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index f9bf3076b1..e32473cbf7 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -19,16 +19,9 @@ const STATUS_COLORS = { skipped: "#8b949e", }; -const PALETTE = { - bar: "#58a6ff", - cumulative: "#a371f7", - total: "#f0883e", - p50: "#58a6ff", - p90: "#d29922", - max: "#f85149", -}; - const DEFAULT_TOP_N = 15; +const SLOW_THRESHOLD_MS = 300_000; +const MEDIUM_THRESHOLD_MS = 60_000; function toSeconds(ms) { return Number((ms / 1000).toFixed(2)); @@ -37,32 +30,22 @@ function toSeconds(ms) { function normalize(timing) { const rawState = String((timing && timing.state) || "errors"); const rawGroup = (timing && (timing.group || timing.name)) || "Ungrouped"; + const name = String((timing && timing.name) || "Unnamed spec"); + const group = String(rawGroup); return { - name: String((timing && timing.name) || "Unnamed spec"), - group: String(rawGroup), + name, + group, + fullName: group === name ? name : `${group} / ${name}`, state: STATUSES.includes(rawState) ? rawState : "errors", runtimeMs: Math.max(0, Number((timing && timing.runtimeMs) || 0)), }; } -// Linear-interpolated quantile over a numerically sorted (asc) array. -// Mirrors Excel's PERCENTILE.INC / numpy's default percentile method. -function quantile(sortedAsc, q) { - if (sortedAsc.length === 0) { - return 0; - } - if (sortedAsc.length === 1) { - return sortedAsc[0]; - } - const pos = (sortedAsc.length - 1) * q; - const base = Math.floor(pos); - const upper = sortedAsc[base + 1]; - return upper === undefined - ? sortedAsc[base] - : sortedAsc[base] + (pos - base) * (upper - sortedAsc[base]); +function emptyStatusCount() { + return { passed: 0, failed: 0, errors: 0, skipped: 0 }; } -function emptyStatusCount() { +function emptyStatusDurations() { return { passed: 0, failed: 0, errors: 0, skipped: 0 }; } @@ -82,20 +65,92 @@ function aggregate(specTimings) { let bucket = byGroup.get(timing.group); if (!bucket) { bucket = { - durations: [], statusCount: emptyStatusCount(), + statusDurations: emptyStatusDurations(), total: 0, }; byGroup.set(timing.group, bucket); } - bucket.durations.push(timing.runtimeMs); bucket.statusCount[timing.state] += 1; + bucket.statusDurations[timing.state] += timing.runtimeMs; bucket.total += timing.runtimeMs; } return { all, byGroup, byStatus, totalMs }; } +function formatSeconds(seconds) { + return `${Number(seconds || 0).toFixed(seconds >= 10 ? 0 : 1)}s`; +} + +function formatCount(count) { + return String(Number(count || 0)); +} + +function drawValueLabels(chart, _args, options) { + const { ctx, data } = chart; + const formatter = options && options.formatter; + if (typeof formatter !== "function") { + return; + } + + ctx.save(); + ctx.font = "12px sans-serif"; + ctx.fillStyle = "#24292f"; + ctx.textBaseline = "middle"; + + chart.getSortedVisibleDatasetMetas().forEach((meta) => { + meta.data.forEach((element, dataIndex) => { + const rawValue = data.datasets[meta.index].data[dataIndex]; + if (!rawValue) { + return; + } + + const label = formatter(rawValue, { + chart, + dataIndex, + datasetIndex: meta.index, + }); + if (!label) { + return; + } + + const props = element.getProps(["x", "y", "base"], true); + const isHorizontal = chart.options.indexAxis === "y"; + const isStacked = Boolean( + isHorizontal + ? chart.options.scales && + chart.options.scales.x && + chart.options.scales.x.stacked + : chart.options.scales && + chart.options.scales.y && + chart.options.scales.y.stacked + ); + + if (isHorizontal) { + const barWidth = Math.abs(props.x - props.base); + ctx.textAlign = isStacked && barWidth > 34 ? "center" : "left"; + ctx.fillText( + label, + isStacked && barWidth > 34 ? (props.x + props.base) / 2 : props.x + 6, + props.y + ); + return; + } + + ctx.textAlign = "center"; + ctx.fillText(label, props.x, props.y - 8); + }); + }); + + ctx.restore(); +} + +const valueLabelsPlugin = { + id: "valueLabels", + afterDatasetsDraw: drawValueLabels, +}; + function baseOptions(title, extra = {}) { return { responsive: false, @@ -108,82 +163,44 @@ function baseOptions(title, extra = {}) { }; } -function statusDoughnut({ byStatus }) { - return { - name: "status-doughnut", - config: { - type: "doughnut", - data: { - labels: STATUSES, - datasets: [ - { - data: STATUSES.map((status) => byStatus[status]), - backgroundColor: STATUSES.map((status) => STATUS_COLORS[status]), - }, - ], - }, - options: baseOptions("E2E spec status distribution"), - }, - }; -} - -function paretoSlowest({ all, totalMs }, topN = DEFAULT_TOP_N) { +function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { const top = [...all] .sort( (left, right) => - right.runtimeMs - left.runtimeMs || left.name.localeCompare(right.name) + right.runtimeMs - left.runtimeMs || + left.fullName.localeCompare(right.fullName) ) .slice(0, topN); - let runningMs = 0; - const cumulativePercents = top.map((timing) => { - runningMs += timing.runtimeMs; - return totalMs > 0 ? Number(((runningMs / totalMs) * 100).toFixed(1)) : 0; - }); - return { - name: "pareto-slowest", + name: "slowest-specs", config: { type: "bar", data: { - labels: top.map((timing) => timing.name), + labels: top.map((timing) => timing.fullName), datasets: [ { - type: "bar", label: "Duration, seconds", data: top.map((timing) => toSeconds(timing.runtimeMs)), - backgroundColor: PALETTE.bar, - xAxisID: "x", - order: 2, - }, - { - type: "line", - label: "Cumulative % of suite time", - data: cumulativePercents, - borderColor: PALETTE.cumulative, - backgroundColor: PALETTE.cumulative, - xAxisID: "x1", - order: 1, + backgroundColor: top.map((timing) => STATUS_COLORS[timing.state]), }, ], }, - options: baseOptions("Top slowest E2E specs (Pareto)", { + options: baseOptions("Top slowest E2E specs", { indexAxis: "y", + plugins: { + title: { display: true, text: "Top slowest E2E specs" }, + legend: { display: false }, + valueLabels: { formatter: formatSeconds }, + }, scales: { x: { beginAtZero: true, - position: "bottom", title: { display: true, text: "Duration, seconds" }, }, - x1: { - beginAtZero: true, - max: 100, - position: "top", - grid: { drawOnChartArea: false }, - title: { display: true, text: "Cumulative %" }, - }, }, }), + plugins: [valueLabelsPlugin], }, }; } @@ -192,15 +209,16 @@ function sortedGroups(byGroup, compareFn) { return [...byGroup.entries()].sort(compareFn); } -function passRatePerFeature({ byGroup }) { +function problemCount(group) { + return group.statusCount.failed + group.statusCount.errors; +} + +function featureDurationStatus({ byGroup }) { // Most-broken features go to the top: failures desc, then total runtime desc, // then alphabetical for a stable order. const entries = sortedGroups(byGroup, (left, right) => { - const failsLeft = left[1].statusCount.failed + left[1].statusCount.errors; - const failsRight = - right[1].statusCount.failed + right[1].statusCount.errors; return ( - failsRight - failsLeft || + problemCount(right[1]) - problemCount(left[1]) || right[1].total - left[1].total || left[0].localeCompare(right[0]) ); @@ -209,132 +227,169 @@ function passRatePerFeature({ byGroup }) { const labels = entries.map(([name]) => name); const datasets = STATUSES.map((status) => ({ label: status, - data: entries.map(([, group]) => { - const total = STATUSES.reduce( - (sum, candidate) => sum + group.statusCount[candidate], - 0 - ); - return total > 0 - ? Number(((group.statusCount[status] / total) * 100).toFixed(1)) - : 0; - }), + data: entries.map(([, group]) => toSeconds(group.statusDurations[status])), backgroundColor: STATUS_COLORS[status], })); return { - name: "pass-rate-per-feature", + name: "feature-duration-status", config: { type: "bar", data: { labels, datasets }, - options: baseOptions("Pass rate by feature, %", { + options: baseOptions("E2E duration by feature and status", { indexAxis: "y", + plugins: { + title: { + display: true, + text: "E2E duration by feature and status", + }, + legend: { display: true }, + valueLabels: { formatter: formatSeconds }, + }, scales: { x: { stacked: true, beginAtZero: true, - max: 100, - title: { display: true, text: "% of specs" }, + title: { display: true, text: "Duration, seconds" }, }, y: { stacked: true }, }, }), + plugins: [valueLabelsPlugin], }, }; } -function quantilesPerFeature({ byGroup }) { - const entries = sortedGroups( - byGroup, - (left, right) => - right[1].total - left[1].total || left[0].localeCompare(right[0]) - ); - const sortedDurations = entries.map(([, group]) => - [...group.durations].sort((left, right) => left - right) - ); +function durationBucket(timing) { + if (timing.runtimeMs > SLOW_THRESHOLD_MS) { + return "slow"; + } + if (timing.runtimeMs >= MEDIUM_THRESHOLD_MS) { + return "medium"; + } + return "fast"; +} + +function durationBuckets({ all }) { + const buckets = [ + { key: "slow", label: "Slow >300s", counts: emptyStatusCount() }, + { key: "medium", label: "Medium 60-300s", counts: emptyStatusCount() }, + { key: "fast", label: "Fast <60s", counts: emptyStatusCount() }, + ]; + const byBucket = new Map(buckets.map((bucket) => [bucket.key, bucket])); + for (const timing of all) { + byBucket.get(durationBucket(timing)).counts[timing.state] += 1; + } return { - name: "quantiles-per-feature", + name: "duration-buckets", config: { type: "bar", data: { - labels: entries.map(([name]) => name), - datasets: [ - { - label: "p50", - data: sortedDurations.map((durations) => - toSeconds(quantile(durations, 0.5)) - ), - backgroundColor: PALETTE.p50, - }, - { - label: "p90", - data: sortedDurations.map((durations) => - toSeconds(quantile(durations, 0.9)) - ), - backgroundColor: PALETTE.p90, - }, - { - label: "max", - data: sortedDurations.map((durations) => - toSeconds(durations[durations.length - 1] || 0) - ), - backgroundColor: PALETTE.max, - }, - ], + labels: buckets.map((bucket) => bucket.label), + datasets: STATUSES.map((status) => ({ + label: status, + data: buckets.map((bucket) => bucket.counts[status]), + backgroundColor: STATUS_COLORS[status], + })), }, - options: baseOptions("Spec duration p50/p90/max by feature, seconds", { + options: baseOptions("E2E specs by duration bucket and status", { + indexAxis: "y", + plugins: { + title: { + display: true, + text: "E2E specs by duration bucket and status", + }, + legend: { display: true }, + valueLabels: { formatter: formatCount }, + }, scales: { - y: { + x: { + stacked: true, beginAtZero: true, - title: { display: true, text: "Seconds" }, + ticks: { precision: 0 }, + title: { display: true, text: "Specs" }, }, + y: { stacked: true }, }, }), + plugins: [valueLabelsPlugin], }, }; } -function featureTotals({ byGroup }) { - const entries = sortedGroups( - byGroup, +function failedAndSlowSpecs({ all }, topN = DEFAULT_TOP_N) { + const sorted = [...all].sort( (left, right) => - right[1].total - left[1].total || left[0].localeCompare(right[0]) + right.runtimeMs - left.runtimeMs || + left.fullName.localeCompare(right.fullName) ); + const selected = []; + const seen = new Set(); + + for (const timing of sorted) { + if (!["failed", "errors"].includes(timing.state)) { + continue; + } + selected.push(timing); + seen.add(timing.fullName); + } + + for (const timing of sorted) { + if (selected.length >= topN) { + break; + } + if (seen.has(timing.fullName)) { + continue; + } + selected.push(timing); + seen.add(timing.fullName); + } return { - name: "feature-totals", + name: "failed-and-slow-specs", config: { type: "bar", data: { - labels: entries.map(([name]) => name), + labels: selected.map((timing) => timing.fullName), datasets: [ { - label: "Total duration, seconds", - data: entries.map(([, group]) => toSeconds(group.total)), - backgroundColor: PALETTE.total, + label: "Duration, seconds", + data: selected.map((timing) => toSeconds(timing.runtimeMs)), + backgroundColor: selected.map( + (timing) => STATUS_COLORS[timing.state] + ), }, ], }, - options: baseOptions("Total duration by feature", { + options: baseOptions("Failed/error specs and slowest successful specs", { indexAxis: "y", + plugins: { + title: { + display: true, + text: "Failed/error specs and slowest successful specs", + }, + legend: { display: false }, + valueLabels: { formatter: formatSeconds }, + }, scales: { x: { beginAtZero: true, - title: { display: true, text: "Seconds" }, + title: { display: true, text: "Duration, seconds" }, }, }, }), + plugins: [valueLabelsPlugin], }, }; } // Order of charts matches the order of attachments in the messenger thread. const CHART_BUILDERS = [ - statusDoughnut, - paretoSlowest, - passRatePerFeature, - quantilesPerFeature, - featureTotals, + featureDurationStatus, + slowestSpecs, + durationBuckets, + failedAndSlowSpecs, ]; function buildClusterChartConfigs(specTimings) { diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js index 09a2c48d98..40d873284b 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js @@ -25,32 +25,29 @@ describe("chart-config", () => { expect(buildClusterChartConfigs(specTimings)).toMatchSnapshot(); }); - test("returns the five chart configs in display order", () => { + test("returns the four chart configs in display order", () => { const configs = buildClusterChartConfigs(specTimings); expect(configs.map(({ name }) => name)).toEqual([ - "status-doughnut", - "pareto-slowest", - "pass-rate-per-feature", - "quantiles-per-feature", - "feature-totals", + "feature-duration-status", + "slowest-specs", + "duration-buckets", + "failed-and-slow-specs", ]); }); test("handles an empty spec timings list", () => { const configs = buildClusterChartConfigs([]); - expect(configs).toHaveLength(5); + expect(configs).toHaveLength(4); const labelsByName = Object.fromEntries( configs.map(({ name, config }) => [name, config.data.labels]) ); - expect(labelsByName["status-doughnut"]).toEqual([ - "passed", - "failed", - "errors", - "skipped", + expect(labelsByName["feature-duration-status"]).toEqual([]); + expect(labelsByName["slowest-specs"]).toEqual([]); + expect(labelsByName["duration-buckets"]).toEqual([ + "Slow >300s", + "Medium 60-300s", + "Fast <60s", ]); - expect(labelsByName["pareto-slowest"]).toEqual([]); - expect(labelsByName["pass-rate-per-feature"]).toEqual([]); - expect(labelsByName["quantiles-per-feature"]).toEqual([]); - expect(labelsByName["feature-totals"]).toEqual([]); + expect(labelsByName["failed-and-slow-specs"]).toEqual([]); }); }); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js index 4fc41d478b..9caf19f164 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js @@ -23,7 +23,7 @@ describe("chart-renderer", () => { await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); }); - test("renders five cluster chart images", async () => { + test("renders four cluster chart images", async () => { const files = await renderClusterCharts({ cluster: "replicated", specTimings: [ @@ -32,11 +32,10 @@ describe("chart-renderer", () => { }); expect(files.map(({ name }) => name)).toEqual([ - "replicated-status-doughnut.png", - "replicated-pareto-slowest.png", - "replicated-pass-rate-per-feature.png", - "replicated-quantiles-per-feature.png", - "replicated-feature-totals.png", + "replicated-feature-duration-status.png", + "replicated-slowest-specs.png", + "replicated-duration-buckets.png", + "replicated-failed-and-slow-specs.png", ]); for (const file of files) { expect(file.buffer).toEqual(Buffer.from("png")); diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 1026adac32..3554fc5844 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -94,12 +94,12 @@ describe("loop-client", () => { message: "reply", files: [ { - name: "pareto-slowest.png", + name: "slowest-specs.png", buffer: Buffer.from("one"), mimeType: "image/png", }, { - name: "pass-rate-per-feature.png", + name: "feature-duration-status.png", buffer: Buffer.from("two"), mimeType: "image/png", }, From e7d12490c2fe63564681a1c1944754bbdc474fa6 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Wed, 20 May 2026 22:57:05 +0300 Subject: [PATCH 08/24] refactor(ci, observability): remove duplicate e2e failed-slow chart Drops the failed-and-slow-specs chart because it overlaps with slowest-specs when a run has few failures. The messenger report now renders three charts per cluster: feature duration by status, slowest specs, and duration buckets. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../__snapshots__/chart-config.test.js.snap | 66 ----------------- .../report/messenger/charts/chart-config.js | 73 +------------------ .../messenger/charts/chart-config.test.js | 6 +- .../messenger/charts/chart-renderer.test.js | 3 +- 4 files changed, 4 insertions(+), 144 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 55c2de7e80..bb78cb867c 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -245,71 +245,5 @@ Array [ }, "name": "duration-buckets", }, - Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": Array [ - "#d29922", - "#f85149", - "#8b949e", - "#3fb950", - "#3fb950", - ], - "data": Array [ - 601, - 301, - 60, - 45, - 10, - ], - "label": "Duration, seconds", - }, - ], - "labels": Array [ - "Network / error", - "VM / slow fail", - "Disk / medium skip", - "VM / passing peer", - "VM / fast pass", - ], - }, - "options": Object { - "animation": false, - "indexAxis": "y", - "plugins": Object { - "legend": Object { - "display": false, - }, - "title": Object { - "display": true, - "text": "Failed/error specs and slowest successful specs", - }, - "valueLabels": Object { - "formatter": [Function], - }, - }, - "responsive": false, - "scales": Object { - "x": Object { - "beginAtZero": true, - "title": Object { - "display": true, - "text": "Duration, seconds", - }, - }, - }, - }, - "plugins": Array [ - Object { - "afterDatasetsDraw": [Function], - "id": "valueLabels", - }, - ], - "type": "bar", - }, - "name": "failed-and-slow-specs", - }, ] `; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index e32473cbf7..d887207ccc 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -318,79 +318,8 @@ function durationBuckets({ all }) { }; } -function failedAndSlowSpecs({ all }, topN = DEFAULT_TOP_N) { - const sorted = [...all].sort( - (left, right) => - right.runtimeMs - left.runtimeMs || - left.fullName.localeCompare(right.fullName) - ); - const selected = []; - const seen = new Set(); - - for (const timing of sorted) { - if (!["failed", "errors"].includes(timing.state)) { - continue; - } - selected.push(timing); - seen.add(timing.fullName); - } - - for (const timing of sorted) { - if (selected.length >= topN) { - break; - } - if (seen.has(timing.fullName)) { - continue; - } - selected.push(timing); - seen.add(timing.fullName); - } - - return { - name: "failed-and-slow-specs", - config: { - type: "bar", - data: { - labels: selected.map((timing) => timing.fullName), - datasets: [ - { - label: "Duration, seconds", - data: selected.map((timing) => toSeconds(timing.runtimeMs)), - backgroundColor: selected.map( - (timing) => STATUS_COLORS[timing.state] - ), - }, - ], - }, - options: baseOptions("Failed/error specs and slowest successful specs", { - indexAxis: "y", - plugins: { - title: { - display: true, - text: "Failed/error specs and slowest successful specs", - }, - legend: { display: false }, - valueLabels: { formatter: formatSeconds }, - }, - scales: { - x: { - beginAtZero: true, - title: { display: true, text: "Duration, seconds" }, - }, - }, - }), - plugins: [valueLabelsPlugin], - }, - }; -} - // Order of charts matches the order of attachments in the messenger thread. -const CHART_BUILDERS = [ - featureDurationStatus, - slowestSpecs, - durationBuckets, - failedAndSlowSpecs, -]; +const CHART_BUILDERS = [featureDurationStatus, slowestSpecs, durationBuckets]; function buildClusterChartConfigs(specTimings) { const data = aggregate(specTimings); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js index 40d873284b..380f32c072 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js @@ -25,19 +25,18 @@ describe("chart-config", () => { expect(buildClusterChartConfigs(specTimings)).toMatchSnapshot(); }); - test("returns the four chart configs in display order", () => { + test("returns the three chart configs in display order", () => { const configs = buildClusterChartConfigs(specTimings); expect(configs.map(({ name }) => name)).toEqual([ "feature-duration-status", "slowest-specs", "duration-buckets", - "failed-and-slow-specs", ]); }); test("handles an empty spec timings list", () => { const configs = buildClusterChartConfigs([]); - expect(configs).toHaveLength(4); + expect(configs).toHaveLength(3); const labelsByName = Object.fromEntries( configs.map(({ name, config }) => [name, config.data.labels]) ); @@ -48,6 +47,5 @@ describe("chart-config", () => { "Medium 60-300s", "Fast <60s", ]); - expect(labelsByName["failed-and-slow-specs"]).toEqual([]); }); }); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js index 9caf19f164..c32c3b55eb 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js @@ -23,7 +23,7 @@ describe("chart-renderer", () => { await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); }); - test("renders four cluster chart images", async () => { + test("renders three cluster chart images", async () => { const files = await renderClusterCharts({ cluster: "replicated", specTimings: [ @@ -35,7 +35,6 @@ describe("chart-renderer", () => { "replicated-feature-duration-status.png", "replicated-slowest-specs.png", "replicated-duration-buckets.png", - "replicated-failed-and-slow-specs.png", ]); for (const file of files) { expect(file.buffer).toEqual(Buffer.from("png")); From 3c0031a01ec008ad61685aa059a258ecc64c64d4 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Wed, 20 May 2026 23:23:06 +0300 Subject: [PATCH 09/24] refactor(ci, observability): use duration gradient for slowest specs chart Changes slowest-specs from status-filled bars to a duration bucket gradient: cyan for fast specs, blue for medium specs, and purple for slow specs. Failed and errored specs remain distinct via red/amber borders and value-label suffixes. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../__snapshots__/chart-config.test.js.snap | 27 ++++++++++++++++-- .../report/messenger/charts/chart-config.js | 28 +++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index bb78cb867c..3124d16f91 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -95,11 +95,25 @@ Array [ "datasets": Array [ Object { "backgroundColor": Array [ + "#a371f7", + "#a371f7", + "#58a6ff", + "#76e3ea", + "#76e3ea", + ], + "borderColor": Array [ "#d29922", "#f85149", - "#8b949e", - "#3fb950", - "#3fb950", + "transparent", + "transparent", + "transparent", + ], + "borderWidth": Array [ + 3, + 3, + 0, + 0, + 0, ], "data": Array [ 601, @@ -109,6 +123,13 @@ Array [ 10, ], "label": "Duration, seconds", + "states": Array [ + "errors", + "failed", + "skipped", + "passed", + "passed", + ], }, ], "labels": Array [ diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index d887207ccc..677f1f781c 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -19,6 +19,12 @@ const STATUS_COLORS = { skipped: "#8b949e", }; +const DURATION_COLORS = { + fast: "#76e3ea", + medium: "#58a6ff", + slow: "#a371f7", +}; + const DEFAULT_TOP_N = 15; const SLOW_THRESHOLD_MS = 300_000; const MEDIUM_THRESHOLD_MS = 60_000; @@ -87,6 +93,13 @@ function formatCount(count) { return String(Number(count || 0)); } +function formatSlowestSpecLabel(seconds, { chart, dataIndex, datasetIndex }) { + const dataset = chart.data.datasets[datasetIndex] || {}; + const state = (dataset.states || [])[dataIndex]; + const suffix = ["failed", "errors"].includes(state) ? ` [${state}]` : ""; + return `${formatSeconds(seconds)}${suffix}`; +} + function drawValueLabels(chart, _args, options) { const { ctx, data } = chart; const formatter = options && options.formatter; @@ -182,7 +195,18 @@ function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { { label: "Duration, seconds", data: top.map((timing) => toSeconds(timing.runtimeMs)), - backgroundColor: top.map((timing) => STATUS_COLORS[timing.state]), + backgroundColor: top.map( + (timing) => DURATION_COLORS[durationBucket(timing)] + ), + borderColor: top.map((timing) => + ["failed", "errors"].includes(timing.state) + ? STATUS_COLORS[timing.state] + : "transparent" + ), + borderWidth: top.map((timing) => + ["failed", "errors"].includes(timing.state) ? 3 : 0 + ), + states: top.map((timing) => timing.state), }, ], }, @@ -191,7 +215,7 @@ function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { plugins: { title: { display: true, text: "Top slowest E2E specs" }, legend: { display: false }, - valueLabels: { formatter: formatSeconds }, + valueLabels: { formatter: formatSlowestSpecLabel }, }, scales: { x: { From 76d8ab2e1e27408dd9ded8c65bb3505a314079fa Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Thu, 21 May 2026 00:07:21 +0300 Subject: [PATCH 10/24] refactor(ci, observability): clarify e2e chart titles and layout Aligns chart titles with Ginkgo hierarchy terms, adds a drill-down legend and minute-based duration ticks to slowest-specs, and trims the chart canvas height. Duration bucket bars are made thinner so the three bucket rows read less heavy. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../__snapshots__/chart-config.test.js.snap | 28 ++++++- .../report/messenger/charts/chart-config.js | 84 +++++++++++++++---- .../report/messenger/charts/chart-renderer.js | 2 +- 3 files changed, 92 insertions(+), 22 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 3124d16f91..743864b6b9 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -58,7 +58,7 @@ Array [ }, "title": Object { "display": true, - "text": "E2E duration by feature and status", + "text": "Overall durations for Describes", }, "valueLabels": Object { "formatter": [Function], @@ -143,13 +143,22 @@ Array [ "options": Object { "animation": false, "indexAxis": "y", + "layout": Object { + "padding": Object { + "bottom": 8, + "top": 16, + }, + }, "plugins": Object { "legend": Object { - "display": false, + "display": true, + "labels": Object { + "generateLabels": [Function], + }, }, "title": Object { "display": true, - "text": "Top slowest E2E specs", + "text": "Top slowest successful specs and failed specs (It/Entry)", }, "valueLabels": Object { "formatter": [Function], @@ -159,6 +168,9 @@ Array [ "scales": Object { "x": Object { "beginAtZero": true, + "ticks": Object { + "stepSize": 60, + }, "title": Object { "display": true, "text": "Duration, seconds", @@ -182,6 +194,8 @@ Array [ "datasets": Array [ Object { "backgroundColor": "#3fb950", + "barPercentage": 0.5, + "categoryPercentage": 0.6, "data": Array [ 0, 0, @@ -191,6 +205,8 @@ Array [ }, Object { "backgroundColor": "#f85149", + "barPercentage": 0.5, + "categoryPercentage": 0.6, "data": Array [ 1, 0, @@ -200,6 +216,8 @@ Array [ }, Object { "backgroundColor": "#d29922", + "barPercentage": 0.5, + "categoryPercentage": 0.6, "data": Array [ 1, 0, @@ -209,6 +227,8 @@ Array [ }, Object { "backgroundColor": "#8b949e", + "barPercentage": 0.5, + "categoryPercentage": 0.6, "data": Array [ 0, 1, @@ -232,7 +252,7 @@ Array [ }, "title": Object { "display": true, - "text": "E2E specs by duration bucket and status", + "text": "It/Entry duration buckets by status", }, "valueLabels": Object { "formatter": [Function], diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index 677f1f781c..d33c1bc834 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -100,6 +100,41 @@ function formatSlowestSpecLabel(seconds, { chart, dataIndex, datasetIndex }) { return `${formatSeconds(seconds)}${suffix}`; } +function slowestSpecsLegendLabels() { + return [ + { + text: "Fast <60s", + fillStyle: DURATION_COLORS.fast, + strokeStyle: DURATION_COLORS.fast, + lineWidth: 0, + }, + { + text: "Medium 60-300s", + fillStyle: DURATION_COLORS.medium, + strokeStyle: DURATION_COLORS.medium, + lineWidth: 0, + }, + { + text: "Slow >300s", + fillStyle: DURATION_COLORS.slow, + strokeStyle: DURATION_COLORS.slow, + lineWidth: 0, + }, + { + text: "Failed border", + fillStyle: "#ffffff", + strokeStyle: STATUS_COLORS.failed, + lineWidth: 3, + }, + { + text: "Error border", + fillStyle: "#ffffff", + strokeStyle: STATUS_COLORS.errors, + lineWidth: 3, + }, + ]; +} + function drawValueLabels(chart, _args, options) { const { ctx, data } = chart; const formatter = options && options.formatter; @@ -210,20 +245,33 @@ function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { }, ], }, - options: baseOptions("Top slowest E2E specs", { - indexAxis: "y", - plugins: { - title: { display: true, text: "Top slowest E2E specs" }, - legend: { display: false }, - valueLabels: { formatter: formatSlowestSpecLabel }, - }, - scales: { - x: { - beginAtZero: true, - title: { display: true, text: "Duration, seconds" }, + options: baseOptions( + "Top slowest successful specs and failed specs (It/Entry)", + { + indexAxis: "y", + plugins: { + title: { + display: true, + text: "Top slowest successful specs and failed specs (It/Entry)", + }, + legend: { + display: true, + labels: { generateLabels: slowestSpecsLegendLabels }, + }, + valueLabels: { formatter: formatSlowestSpecLabel }, }, - }, - }), + scales: { + x: { + beginAtZero: true, + ticks: { stepSize: 60 }, + title: { display: true, text: "Duration, seconds" }, + }, + }, + layout: { + padding: { top: 16, bottom: 8 }, + }, + } + ), plugins: [valueLabelsPlugin], }, }; @@ -260,12 +308,12 @@ function featureDurationStatus({ byGroup }) { config: { type: "bar", data: { labels, datasets }, - options: baseOptions("E2E duration by feature and status", { + options: baseOptions("Overall durations for Describes", { indexAxis: "y", plugins: { title: { display: true, - text: "E2E duration by feature and status", + text: "Overall durations for Describes", }, legend: { display: true }, valueLabels: { formatter: formatSeconds }, @@ -315,14 +363,16 @@ function durationBuckets({ all }) { label: status, data: buckets.map((bucket) => bucket.counts[status]), backgroundColor: STATUS_COLORS[status], + barPercentage: 0.5, + categoryPercentage: 0.6, })), }, - options: baseOptions("E2E specs by duration bucket and status", { + options: baseOptions("It/Entry duration buckets by status", { indexAxis: "y", plugins: { title: { display: true, - text: "E2E specs by duration bucket and status", + text: "It/Entry duration buckets by status", }, legend: { display: true }, valueLabels: { formatter: formatCount }, diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js index c3104af2ff..50d3157afa 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -23,7 +23,7 @@ function loadChartRenderer() { const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); canvasInstance = new ChartJSNodeCanvas({ width: 1280, - height: 720, + height: 640, backgroundColour: "#ffffff", }); } From 08636d569a20af45cadd6c31ab4b962df39dfc12 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Thu, 21 May 2026 00:19:15 +0300 Subject: [PATCH 11/24] refactor(ci, observability): widen slowest specs chart output Adds per-chart canvas sizing and renders slowest-specs at 1920x720 so long It/Entry drill-down labels fit without clipping. Other E2E report charts keep the default 1280x640 size. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../__snapshots__/chart-config.test.js.snap | 4 ++ .../report/messenger/charts/chart-config.js | 1 + .../report/messenger/charts/chart-renderer.js | 37 +++++++++++-------- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 743864b6b9..d1d48b07fe 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -187,6 +187,10 @@ Array [ "type": "bar", }, "name": "slowest-specs", + "size": Object { + "height": 720, + "width": 1920, + }, }, Object { "config": Object { diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index d33c1bc834..98cbff195b 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -222,6 +222,7 @@ function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { return { name: "slowest-specs", + size: { width: 1920, height: 720 }, config: { type: "bar", data: { diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js index 50d3157afa..2db75624f8 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -12,23 +12,28 @@ const { buildClusterChartConfigs } = require("./chart-config"); -let canvasInstance; +const defaultChartSize = { width: 1280, height: 640 }; +const canvasInstances = new Map(); // Module-level singleton: ChartJSNodeCanvas startup (loading chart.js + setting // up the cairo-backed canvas) is non-trivial, and the renderer is stateless // between renderToBuffer calls. Reusing it across clusters keeps memory usage // flat when the messenger report grows. -function loadChartRenderer() { - if (!canvasInstance) { +function loadChartRenderer({ width, height } = defaultChartSize) { + const rendererKey = `${width}x${height}`; + if (!canvasInstances.has(rendererKey)) { const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); - canvasInstance = new ChartJSNodeCanvas({ - width: 1280, - height: 640, - backgroundColour: "#ffffff", - }); + canvasInstances.set( + rendererKey, + new ChartJSNodeCanvas({ + width, + height, + backgroundColour: "#ffffff", + }) + ); } - return canvasInstance; + return canvasInstances.get(rendererKey); } function sanitizeFilenamePart(value) { @@ -45,18 +50,20 @@ async function renderClusterCharts(report) { return []; } - const renderer = loadChartRenderer(); const configs = buildClusterChartConfigs(report.specTimings); const clusterName = sanitizeFilenamePart( report.cluster || report.storageType || "cluster" ); return Promise.all( - configs.map(async ({ name, config }) => ({ - name: `${clusterName}-${name}.png`, - buffer: await renderer.renderToBuffer(config, "image/png"), - mimeType: "image/png", - })) + configs.map(async ({ name, config, size }) => { + const renderer = loadChartRenderer(size || defaultChartSize); + return { + name: `${clusterName}-${name}.png`, + buffer: await renderer.renderToBuffer(config, "image/png"), + mimeType: "image/png", + }; + }) ); } From 90ff63f0b47c995800ba1decb3d9b86287665f31 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Thu, 21 May 2026 00:26:52 +0300 Subject: [PATCH 12/24] refactor(ci, observability): add margin to slowest specs chart Widens the slowest-specs output from 1920px to 2048px so long drill-down labels and value labels have a little more space from the image edge. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../messenger/charts/__snapshots__/chart-config.test.js.snap | 2 +- .github/scripts/js/e2e/report/messenger/charts/chart-config.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index d1d48b07fe..0f09bb5f41 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -189,7 +189,7 @@ Array [ "name": "slowest-specs", "size": Object { "height": 720, - "width": 1920, + "width": 2048, }, }, Object { diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index 98cbff195b..5d339ad461 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -222,7 +222,7 @@ function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { return { name: "slowest-specs", - size: { width: 1920, height: 720 }, + size: { width: 2048, height: 720 }, config: { type: "bar", data: { From 464e767a53454425f9e4f95da3ce38fb592ae24c Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Thu, 21 May 2026 00:48:21 +0300 Subject: [PATCH 13/24] fix(ci, observability): keep small duration bucket counts visible Draws count labels for tiny stacked duration-bucket segments as offset callouts instead of hiding them or placing overlapping text on the white background. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../__snapshots__/chart-config.test.js.snap | 1 + .../report/messenger/charts/chart-config.js | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 0f09bb5f41..97ae33086c 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -259,6 +259,7 @@ Array [ "text": "It/Entry duration buckets by status", }, "valueLabels": Object { + "calloutSmallStackedLabels": true, "formatter": [Function], }, }, diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index 5d339ad461..057d296ab5 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -177,6 +177,19 @@ function drawValueLabels(chart, _args, options) { if (isHorizontal) { const barWidth = Math.abs(props.x - props.base); + if (isStacked && options.calloutSmallStackedLabels && barWidth <= 34) { + const offsets = [-12, -4, 4, 12]; + const dataset = data.datasets[meta.index] || {}; + ctx.textAlign = "left"; + ctx.fillStyle = dataset.backgroundColor || "#24292f"; + ctx.fillText( + label, + props.x + 6, + props.y + (offsets[meta.index] || 0) + ); + ctx.fillStyle = "#24292f"; + return; + } ctx.textAlign = isStacked && barWidth > 34 ? "center" : "left"; ctx.fillText( label, @@ -376,7 +389,10 @@ function durationBuckets({ all }) { text: "It/Entry duration buckets by status", }, legend: { display: true }, - valueLabels: { formatter: formatCount }, + valueLabels: { + formatter: formatCount, + calloutSmallStackedLabels: true, + }, }, scales: { x: { From 2c8b1af6f995543df70f162d0e1b5a66c4b2ae4c Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Thu, 21 May 2026 16:13:18 +0300 Subject: [PATCH 14/24] refactor(ci, observability): separate slowest e2e charts from messenger Keep the Loop report focused on feature duration status while moving slowest-specs into local and CI artifact renderers for deeper triage. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/cluster-report.test.js | 2 +- .../js/e2e/report/messenger-report.test.js | 4 +- .../__snapshots__/chart-config.test.js.snap | 206 +-------- .../builders/feature-duration-status.js | 76 ++++ .../charts/builders/slowest-specs.js | 120 +++++ .../report/messenger/charts/chart-config.js | 411 +----------------- .../messenger/charts/chart-config.test.js | 46 +- .../report/messenger/charts/chart-renderer.js | 39 +- .../messenger/charts/chart-renderer.test.js | 17 +- .../js/e2e/report/messenger/charts/data.js | 141 ++++++ .../js/e2e/report/messenger/charts/index.js | 29 ++ .../js/e2e/report/messenger/charts/plugins.js | 91 ++++ .../js/e2e/report/messenger/loop-client.js | 70 +-- .../e2e/report/messenger/loop-client.test.js | 73 +++- .../e2e/report/render-slowest-for-describe.js | 175 ++++++++ .../render-slowest-for-describe.test.js | 89 ++++ .../js/e2e/report/render-top-describes.js | 157 +++++++ .../e2e/report/shared/ginkgo-report-utils.js | 2 +- .github/workflows/e2e-matrix.yml | 15 + Taskfile.yaml | 30 ++ 20 files changed, 1122 insertions(+), 671 deletions(-) create mode 100644 .github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/data.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/index.js create mode 100644 .github/scripts/js/e2e/report/messenger/charts/plugins.js create mode 100644 .github/scripts/js/e2e/report/render-slowest-for-describe.js create mode 100644 .github/scripts/js/e2e/report/render-slowest-for-describe.test.js create mode 100644 .github/scripts/js/e2e/report/render-top-describes.js diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index dc8486be1d..2af54e74a3 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -477,7 +477,7 @@ describe("cluster-report", () => { }, { name: "skipped", - group: "skipped", + group: "Top-level Its", state: "skipped", runtimeMs: 60000, labels: [], diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 36a697b5f5..551d3d5c90 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -182,7 +182,7 @@ describe("messenger-report", () => { test("attaches duration chart files to thread reply without a text caption", async () => inTempDir(async (tempDir) => { const chartFile = { - name: "replicated-slowest-specs.png", + name: "replicated-feature-duration-status.png", buffer: Buffer.from("png"), mimeType: "image/png", }; @@ -585,7 +585,7 @@ describe("messenger-report", () => { test("posts main report and per-cluster failed tests thread via Loop API", async () => inTempDir(async (tempDir) => { const chartFile = { - name: "replicated-slowest-specs.png", + name: "replicated-feature-duration-status.png", buffer: Buffer.from("png"), mimeType: "image/png", }; diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap index 97ae33086c..7875ec0ca6 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -69,105 +69,6 @@ Array [ "x": Object { "beginAtZero": true, "stacked": true, - "title": Object { - "display": true, - "text": "Duration, seconds", - }, - }, - "y": Object { - "stacked": true, - }, - }, - }, - "plugins": Array [ - Object { - "afterDatasetsDraw": [Function], - "id": "valueLabels", - }, - ], - "type": "bar", - }, - "name": "feature-duration-status", - }, - Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": Array [ - "#a371f7", - "#a371f7", - "#58a6ff", - "#76e3ea", - "#76e3ea", - ], - "borderColor": Array [ - "#d29922", - "#f85149", - "transparent", - "transparent", - "transparent", - ], - "borderWidth": Array [ - 3, - 3, - 0, - 0, - 0, - ], - "data": Array [ - 601, - 301, - 60, - 45, - 10, - ], - "label": "Duration, seconds", - "states": Array [ - "errors", - "failed", - "skipped", - "passed", - "passed", - ], - }, - ], - "labels": Array [ - "Network / error", - "VM / slow fail", - "Disk / medium skip", - "VM / passing peer", - "VM / fast pass", - ], - }, - "options": Object { - "animation": false, - "indexAxis": "y", - "layout": Object { - "padding": Object { - "bottom": 8, - "top": 16, - }, - }, - "plugins": Object { - "legend": Object { - "display": true, - "labels": Object { - "generateLabels": [Function], - }, - }, - "title": Object { - "display": true, - "text": "Top slowest successful specs and failed specs (It/Entry)", - }, - "valueLabels": Object { - "formatter": [Function], - }, - }, - "responsive": false, - "scales": Object { - "x": Object { - "beginAtZero": true, "ticks": Object { "stepSize": 60, }, @@ -176,106 +77,6 @@ Array [ "text": "Duration, seconds", }, }, - }, - }, - "plugins": Array [ - Object { - "afterDatasetsDraw": [Function], - "id": "valueLabels", - }, - ], - "type": "bar", - }, - "name": "slowest-specs", - "size": Object { - "height": 720, - "width": 2048, - }, - }, - Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#3fb950", - "barPercentage": 0.5, - "categoryPercentage": 0.6, - "data": Array [ - 0, - 0, - 2, - ], - "label": "passed", - }, - Object { - "backgroundColor": "#f85149", - "barPercentage": 0.5, - "categoryPercentage": 0.6, - "data": Array [ - 1, - 0, - 0, - ], - "label": "failed", - }, - Object { - "backgroundColor": "#d29922", - "barPercentage": 0.5, - "categoryPercentage": 0.6, - "data": Array [ - 1, - 0, - 0, - ], - "label": "errors", - }, - Object { - "backgroundColor": "#8b949e", - "barPercentage": 0.5, - "categoryPercentage": 0.6, - "data": Array [ - 0, - 1, - 0, - ], - "label": "skipped", - }, - ], - "labels": Array [ - "Slow >300s", - "Medium 60-300s", - "Fast <60s", - ], - }, - "options": Object { - "animation": false, - "indexAxis": "y", - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "It/Entry duration buckets by status", - }, - "valueLabels": Object { - "calloutSmallStackedLabels": true, - "formatter": [Function], - }, - }, - "responsive": false, - "scales": Object { - "x": Object { - "beginAtZero": true, - "stacked": true, - "ticks": Object { - "precision": 0, - }, - "title": Object { - "display": true, - "text": "Specs", - }, - }, "y": Object { "stacked": true, }, @@ -289,7 +90,12 @@ Array [ ], "type": "bar", }, - "name": "duration-buckets", + "name": "feature-duration-status", + "size": Object { + "height": 640, + "pixelRatio": 2, + "width": 1280, + }, }, ] `; diff --git a/.github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js b/.github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js new file mode 100644 index 0000000000..371df4a6ea --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js @@ -0,0 +1,76 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + STATUSES, + STATUS_COLORS, + toSeconds, + formatSeconds, + baseOptions, +} = require("../data"); +const { valueLabelsPlugin } = require("../plugins"); + +function sortedGroups(byGroup, compareFn) { + return [...byGroup.entries()].sort(compareFn); +} + +function problemCount(group) { + return group.statusCount.failed + group.statusCount.errors; +} + +function featureDurationStatus({ byGroup }) { + // Most-broken features go to the top: failures desc, then total runtime desc, + // then alphabetical for a stable order. + const entries = sortedGroups(byGroup, (left, right) => { + return ( + problemCount(right[1]) - problemCount(left[1]) || + right[1].total - left[1].total || + left[0].localeCompare(right[0]) + ); + }); + + const labels = entries.map(([name]) => name); + const datasets = STATUSES.map((status) => ({ + label: status, + data: entries.map(([, group]) => toSeconds(group.statusDurations[status])), + backgroundColor: STATUS_COLORS[status], + })); + const height = Math.max(640, 120 + labels.length * 36); + + return { + name: "feature-duration-status", + size: { width: 1280, height, pixelRatio: 2 }, + config: { + type: "bar", + data: { labels, datasets }, + options: baseOptions("Overall durations for Describes", { + indexAxis: "y", + plugins: { + legend: { display: true }, + valueLabels: { formatter: formatSeconds }, + }, + scales: { + x: { + stacked: true, + beginAtZero: true, + ticks: { stepSize: 60 }, + title: { display: true, text: "Duration, seconds" }, + }, + y: { stacked: true }, + }, + }), + plugins: [valueLabelsPlugin], + }, + }; +} + +module.exports = featureDurationStatus; diff --git a/.github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js b/.github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js new file mode 100644 index 0000000000..3045ad8b0b --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js @@ -0,0 +1,120 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + STATUS_COLORS, + DURATION_COLORS, + DURATION_LABELS, + DEFAULT_TOP_N, + toSeconds, + durationBucket, + formatSeconds, + baseOptions, +} = require("../data"); +const { valueLabelsPlugin } = require("../plugins"); + +function formatSlowestSpecLabel(seconds, { chart, dataIndex }) { + const dataset = chart.data.datasets[0] || {}; + const state = (dataset.states || [])[dataIndex]; + const suffix = ["failed", "errors"].includes(state) ? ` [${state}]` : ""; + return `${formatSeconds(seconds)}${suffix}`; +} + +function slowestSpecsLegendLabels() { + const durationLabels = Object.entries(DURATION_COLORS).map( + ([key, color]) => ({ + text: DURATION_LABELS[key], + fillStyle: color, + strokeStyle: color, + lineWidth: 0, + }) + ); + const statusOverlays = [ + ["failed", "Failed border"], + ["errors", "Error border"], + ].map(([status, text]) => ({ + text, + fillStyle: "#ffffff", + strokeStyle: STATUS_COLORS[status], + lineWidth: 3, + })); + + return [...durationLabels, ...statusOverlays]; +} + +function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { + const top = [...all] + .sort( + (left, right) => + right.runtimeMs - left.runtimeMs || + left.fullName.localeCompare(right.fullName) + ) + .slice(0, topN); + const annotated = top.map((timing) => { + const isFailure = timing.state === "failed" || timing.state === "errors"; + return { + timing, + bucketColor: DURATION_COLORS[durationBucket(timing)], + borderColor: isFailure ? STATUS_COLORS[timing.state] : "transparent", + borderWidth: isFailure ? 3 : 0, + }; + }); + + return { + name: "slowest-specs", + size: { width: 2048, height: 720, pixelRatio: 2 }, + config: { + type: "bar", + data: { + labels: annotated.map(({ timing }) => timing.fullName), + datasets: [ + { + label: "Duration, seconds", + data: annotated.map(({ timing }) => toSeconds(timing.runtimeMs)), + backgroundColor: annotated.map(({ bucketColor }) => bucketColor), + borderColor: annotated.map(({ borderColor }) => borderColor), + borderWidth: annotated.map(({ borderWidth }) => borderWidth), + barPercentage: 0.55, + categoryPercentage: 0.7, + states: annotated.map(({ timing }) => timing.state), + }, + ], + }, + options: baseOptions( + "Top slowest successful specs and failed specs (It/Entry)", + { + indexAxis: "y", + plugins: { + legend: { + display: true, + labels: { generateLabels: slowestSpecsLegendLabels }, + }, + valueLabels: { formatter: formatSlowestSpecLabel }, + }, + scales: { + x: { + beginAtZero: true, + ticks: { stepSize: 60 }, + title: { display: true, text: "Duration, seconds" }, + }, + }, + layout: { + padding: { top: 16, bottom: 8 }, + }, + } + ), + plugins: [valueLabelsPlugin], + }, + }; +} + +module.exports = slowestSpecs; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js index 057d296ab5..89566136f4 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -10,413 +10,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -const STATUSES = ["passed", "failed", "errors", "skipped"]; - -const STATUS_COLORS = { - passed: "#3fb950", - failed: "#f85149", - errors: "#d29922", - skipped: "#8b949e", -}; - -const DURATION_COLORS = { - fast: "#76e3ea", - medium: "#58a6ff", - slow: "#a371f7", -}; - -const DEFAULT_TOP_N = 15; -const SLOW_THRESHOLD_MS = 300_000; -const MEDIUM_THRESHOLD_MS = 60_000; - -function toSeconds(ms) { - return Number((ms / 1000).toFixed(2)); -} - -function normalize(timing) { - const rawState = String((timing && timing.state) || "errors"); - const rawGroup = (timing && (timing.group || timing.name)) || "Ungrouped"; - const name = String((timing && timing.name) || "Unnamed spec"); - const group = String(rawGroup); - return { - name, - group, - fullName: group === name ? name : `${group} / ${name}`, - state: STATUSES.includes(rawState) ? rawState : "errors", - runtimeMs: Math.max(0, Number((timing && timing.runtimeMs) || 0)), - }; -} - -function emptyStatusCount() { - return { passed: 0, failed: 0, errors: 0, skipped: 0 }; -} - -function emptyStatusDurations() { - return { passed: 0, failed: 0, errors: 0, skipped: 0 }; -} - -// Single pass over the spec timings feeds every chart builder below. -function aggregate(specTimings) { - const all = []; - const byGroup = new Map(); - const byStatus = emptyStatusCount(); - let totalMs = 0; - - for (const raw of specTimings || []) { - const timing = normalize(raw); - all.push(timing); - totalMs += timing.runtimeMs; - byStatus[timing.state] += 1; - - let bucket = byGroup.get(timing.group); - if (!bucket) { - bucket = { - statusCount: emptyStatusCount(), - statusDurations: emptyStatusDurations(), - total: 0, - }; - byGroup.set(timing.group, bucket); - } - bucket.statusCount[timing.state] += 1; - bucket.statusDurations[timing.state] += timing.runtimeMs; - bucket.total += timing.runtimeMs; - } - - return { all, byGroup, byStatus, totalMs }; -} - -function formatSeconds(seconds) { - return `${Number(seconds || 0).toFixed(seconds >= 10 ? 0 : 1)}s`; -} - -function formatCount(count) { - return String(Number(count || 0)); -} - -function formatSlowestSpecLabel(seconds, { chart, dataIndex, datasetIndex }) { - const dataset = chart.data.datasets[datasetIndex] || {}; - const state = (dataset.states || [])[dataIndex]; - const suffix = ["failed", "errors"].includes(state) ? ` [${state}]` : ""; - return `${formatSeconds(seconds)}${suffix}`; -} - -function slowestSpecsLegendLabels() { - return [ - { - text: "Fast <60s", - fillStyle: DURATION_COLORS.fast, - strokeStyle: DURATION_COLORS.fast, - lineWidth: 0, - }, - { - text: "Medium 60-300s", - fillStyle: DURATION_COLORS.medium, - strokeStyle: DURATION_COLORS.medium, - lineWidth: 0, - }, - { - text: "Slow >300s", - fillStyle: DURATION_COLORS.slow, - strokeStyle: DURATION_COLORS.slow, - lineWidth: 0, - }, - { - text: "Failed border", - fillStyle: "#ffffff", - strokeStyle: STATUS_COLORS.failed, - lineWidth: 3, - }, - { - text: "Error border", - fillStyle: "#ffffff", - strokeStyle: STATUS_COLORS.errors, - lineWidth: 3, - }, - ]; -} - -function drawValueLabels(chart, _args, options) { - const { ctx, data } = chart; - const formatter = options && options.formatter; - if (typeof formatter !== "function") { - return; - } - - ctx.save(); - ctx.font = "12px sans-serif"; - ctx.fillStyle = "#24292f"; - ctx.textBaseline = "middle"; - - chart.getSortedVisibleDatasetMetas().forEach((meta) => { - meta.data.forEach((element, dataIndex) => { - const rawValue = data.datasets[meta.index].data[dataIndex]; - if (!rawValue) { - return; - } - - const label = formatter(rawValue, { - chart, - dataIndex, - datasetIndex: meta.index, - }); - if (!label) { - return; - } - - const props = element.getProps(["x", "y", "base"], true); - const isHorizontal = chart.options.indexAxis === "y"; - const isStacked = Boolean( - isHorizontal - ? chart.options.scales && - chart.options.scales.x && - chart.options.scales.x.stacked - : chart.options.scales && - chart.options.scales.y && - chart.options.scales.y.stacked - ); - - if (isHorizontal) { - const barWidth = Math.abs(props.x - props.base); - if (isStacked && options.calloutSmallStackedLabels && barWidth <= 34) { - const offsets = [-12, -4, 4, 12]; - const dataset = data.datasets[meta.index] || {}; - ctx.textAlign = "left"; - ctx.fillStyle = dataset.backgroundColor || "#24292f"; - ctx.fillText( - label, - props.x + 6, - props.y + (offsets[meta.index] || 0) - ); - ctx.fillStyle = "#24292f"; - return; - } - ctx.textAlign = isStacked && barWidth > 34 ? "center" : "left"; - ctx.fillText( - label, - isStacked && barWidth > 34 ? (props.x + props.base) / 2 : props.x + 6, - props.y - ); - return; - } - - ctx.textAlign = "center"; - ctx.fillText(label, props.x, props.y - 8); - }); - }); - - ctx.restore(); -} - -const valueLabelsPlugin = { - id: "valueLabels", - afterDatasetsDraw: drawValueLabels, -}; - -function baseOptions(title, extra = {}) { - return { - responsive: false, - animation: false, - plugins: { - title: { display: true, text: title }, - legend: { display: true }, - }, - ...extra, - }; -} - -function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { - const top = [...all] - .sort( - (left, right) => - right.runtimeMs - left.runtimeMs || - left.fullName.localeCompare(right.fullName) - ) - .slice(0, topN); - - return { - name: "slowest-specs", - size: { width: 2048, height: 720 }, - config: { - type: "bar", - data: { - labels: top.map((timing) => timing.fullName), - datasets: [ - { - label: "Duration, seconds", - data: top.map((timing) => toSeconds(timing.runtimeMs)), - backgroundColor: top.map( - (timing) => DURATION_COLORS[durationBucket(timing)] - ), - borderColor: top.map((timing) => - ["failed", "errors"].includes(timing.state) - ? STATUS_COLORS[timing.state] - : "transparent" - ), - borderWidth: top.map((timing) => - ["failed", "errors"].includes(timing.state) ? 3 : 0 - ), - states: top.map((timing) => timing.state), - }, - ], - }, - options: baseOptions( - "Top slowest successful specs and failed specs (It/Entry)", - { - indexAxis: "y", - plugins: { - title: { - display: true, - text: "Top slowest successful specs and failed specs (It/Entry)", - }, - legend: { - display: true, - labels: { generateLabels: slowestSpecsLegendLabels }, - }, - valueLabels: { formatter: formatSlowestSpecLabel }, - }, - scales: { - x: { - beginAtZero: true, - ticks: { stepSize: 60 }, - title: { display: true, text: "Duration, seconds" }, - }, - }, - layout: { - padding: { top: 16, bottom: 8 }, - }, - } - ), - plugins: [valueLabelsPlugin], - }, - }; -} - -function sortedGroups(byGroup, compareFn) { - return [...byGroup.entries()].sort(compareFn); -} - -function problemCount(group) { - return group.statusCount.failed + group.statusCount.errors; -} - -function featureDurationStatus({ byGroup }) { - // Most-broken features go to the top: failures desc, then total runtime desc, - // then alphabetical for a stable order. - const entries = sortedGroups(byGroup, (left, right) => { - return ( - problemCount(right[1]) - problemCount(left[1]) || - right[1].total - left[1].total || - left[0].localeCompare(right[0]) - ); - }); - - const labels = entries.map(([name]) => name); - const datasets = STATUSES.map((status) => ({ - label: status, - data: entries.map(([, group]) => toSeconds(group.statusDurations[status])), - backgroundColor: STATUS_COLORS[status], - })); - - return { - name: "feature-duration-status", - config: { - type: "bar", - data: { labels, datasets }, - options: baseOptions("Overall durations for Describes", { - indexAxis: "y", - plugins: { - title: { - display: true, - text: "Overall durations for Describes", - }, - legend: { display: true }, - valueLabels: { formatter: formatSeconds }, - }, - scales: { - x: { - stacked: true, - beginAtZero: true, - title: { display: true, text: "Duration, seconds" }, - }, - y: { stacked: true }, - }, - }), - plugins: [valueLabelsPlugin], - }, - }; -} - -function durationBucket(timing) { - if (timing.runtimeMs > SLOW_THRESHOLD_MS) { - return "slow"; - } - if (timing.runtimeMs >= MEDIUM_THRESHOLD_MS) { - return "medium"; - } - return "fast"; -} - -function durationBuckets({ all }) { - const buckets = [ - { key: "slow", label: "Slow >300s", counts: emptyStatusCount() }, - { key: "medium", label: "Medium 60-300s", counts: emptyStatusCount() }, - { key: "fast", label: "Fast <60s", counts: emptyStatusCount() }, - ]; - const byBucket = new Map(buckets.map((bucket) => [bucket.key, bucket])); - for (const timing of all) { - byBucket.get(durationBucket(timing)).counts[timing.state] += 1; - } - - return { - name: "duration-buckets", - config: { - type: "bar", - data: { - labels: buckets.map((bucket) => bucket.label), - datasets: STATUSES.map((status) => ({ - label: status, - data: buckets.map((bucket) => bucket.counts[status]), - backgroundColor: STATUS_COLORS[status], - barPercentage: 0.5, - categoryPercentage: 0.6, - })), - }, - options: baseOptions("It/Entry duration buckets by status", { - indexAxis: "y", - plugins: { - title: { - display: true, - text: "It/Entry duration buckets by status", - }, - legend: { display: true }, - valueLabels: { - formatter: formatCount, - calloutSmallStackedLabels: true, - }, - }, - scales: { - x: { - stacked: true, - beginAtZero: true, - ticks: { precision: 0 }, - title: { display: true, text: "Specs" }, - }, - y: { stacked: true }, - }, - }), - plugins: [valueLabelsPlugin], - }, - }; -} - -// Order of charts matches the order of attachments in the messenger thread. -const CHART_BUILDERS = [featureDurationStatus, slowestSpecs, durationBuckets]; - -function buildClusterChartConfigs(specTimings) { - const data = aggregate(specTimings); - return CHART_BUILDERS.map((build) => build(data)); -} - -module.exports = { - buildClusterChartConfigs, -}; +module.exports = require("."); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js index 380f32c072..c678113b65 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js @@ -10,7 +10,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { buildClusterChartConfigs } = require("./chart-config"); +const { buildClusterChartConfigs, slowestSpecs } = require("./chart-config"); +const { aggregate } = require("./data"); const specTimings = [ { name: "fast pass", group: "VM", state: "passed", runtimeMs: 10_000 }, @@ -25,27 +26,52 @@ describe("chart-config", () => { expect(buildClusterChartConfigs(specTimings)).toMatchSnapshot(); }); - test("returns the three chart configs in display order", () => { + test("returns the messenger chart config in display order", () => { const configs = buildClusterChartConfigs(specTimings); expect(configs.map(({ name }) => name)).toEqual([ "feature-duration-status", - "slowest-specs", - "duration-buckets", ]); }); test("handles an empty spec timings list", () => { const configs = buildClusterChartConfigs([]); - expect(configs).toHaveLength(3); + expect(configs).toHaveLength(1); const labelsByName = Object.fromEntries( configs.map(({ name, config }) => [name, config.data.labels]) ); expect(labelsByName["feature-duration-status"]).toEqual([]); - expect(labelsByName["slowest-specs"]).toEqual([]); - expect(labelsByName["duration-buckets"]).toEqual([ - "Slow >300s", - "Medium 60-300s", - "Fast <60s", + }); + + test("normalizes non-numeric runtimes to zero", () => { + const configs = buildClusterChartConfigs([ + { runtimeMs: "slow", name: "x", group: "g", state: "passed" }, + ]); + const numericValues = configs.flatMap(({ config }) => + config.data.datasets.flatMap((dataset) => + dataset.data.filter((value) => typeof value === "number") + ) + ); + + expect(numericValues).toContain(0); + expect(numericValues.some((value) => Number.isNaN(value))).toBe(false); + }); + + test("builds slowest specs sorted by duration descending", () => { + const chart = slowestSpecs( + aggregate([ + { name: "middle", group: "VM", state: "passed", runtimeMs: 90_000 }, + { name: "slow b", group: "VM", state: "passed", runtimeMs: 180_000 }, + { name: "slow a", group: "Disk", state: "passed", runtimeMs: 180_000 }, + { name: "fast", group: "Network", state: "passed", runtimeMs: 10_000 }, + ]) + ); + + expect(chart.config.data.labels).toEqual([ + "Disk / slow a", + "VM / slow b", + "VM / middle", + "Network / fast", ]); + expect(chart.config.data.datasets[0].data).toEqual([180, 180, 90, 10]); }); }); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js index 2db75624f8..83798782ea 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -10,17 +10,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -const { buildClusterChartConfigs } = require("./chart-config"); +const { buildClusterChartConfigs } = require("."); -const defaultChartSize = { width: 1280, height: 640 }; +const defaultChartSize = { width: 1280, height: 640, pixelRatio: 2 }; const canvasInstances = new Map(); // Module-level singleton: ChartJSNodeCanvas startup (loading chart.js + setting // up the cairo-backed canvas) is non-trivial, and the renderer is stateless // between renderToBuffer calls. Reusing it across clusters keeps memory usage // flat when the messenger report grows. -function loadChartRenderer({ width, height } = defaultChartSize) { - const rendererKey = `${width}x${height}`; +function loadChartRenderer({ width, height, pixelRatio } = defaultChartSize) { + const rendererKey = `${width}x${height}@${pixelRatio}`; if (!canvasInstances.has(rendererKey)) { const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); canvasInstances.set( @@ -36,6 +36,32 @@ function loadChartRenderer({ width, height } = defaultChartSize) { return canvasInstances.get(rendererKey); } +function normalizeChartSize(size) { + return { + ...defaultChartSize, + ...(size || {}), + }; +} + +function withDevicePixelRatio(config, pixelRatio) { + return { + ...config, + options: { + ...(config.options || {}), + devicePixelRatio: pixelRatio, + }, + }; +} + +async function renderChartBuffer({ config, size }) { + const chartSize = normalizeChartSize(size); + const renderer = loadChartRenderer(chartSize); + return renderer.renderToBuffer( + withDevicePixelRatio(config, chartSize.pixelRatio), + "image/png" + ); +} + function sanitizeFilenamePart(value) { const fallback = "cluster"; const safe = String(value || fallback).replace(/[^a-zA-Z0-9_-]+/g, "_"); @@ -57,10 +83,9 @@ async function renderClusterCharts(report) { return Promise.all( configs.map(async ({ name, config, size }) => { - const renderer = loadChartRenderer(size || defaultChartSize); return { name: `${clusterName}-${name}.png`, - buffer: await renderer.renderToBuffer(config, "image/png"), + buffer: await renderChartBuffer({ config, size }), mimeType: "image/png", }; }) @@ -69,4 +94,6 @@ async function renderClusterCharts(report) { module.exports = { renderClusterCharts, + renderChartBuffer, + sanitizeFilenamePart, }; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js index c32c3b55eb..a30dd40725 100644 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js @@ -10,20 +10,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +const mockRenderToBuffer = jest.fn().mockResolvedValue(Buffer.from("png")); + jest.mock("chartjs-node-canvas", () => ({ ChartJSNodeCanvas: jest.fn().mockImplementation(() => ({ - renderToBuffer: jest.fn().mockResolvedValue(Buffer.from("png")), + renderToBuffer: mockRenderToBuffer, })), })); const { renderClusterCharts } = require("./chart-renderer"); +const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); describe("chart-renderer", () => { test("returns no files when spec timings are empty", async () => { await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); }); - test("renders three cluster chart images", async () => { + test("renders messenger cluster chart images", async () => { const files = await renderClusterCharts({ cluster: "replicated", specTimings: [ @@ -33,12 +36,18 @@ describe("chart-renderer", () => { expect(files.map(({ name }) => name)).toEqual([ "replicated-feature-duration-status.png", - "replicated-slowest-specs.png", - "replicated-duration-buckets.png", ]); for (const file of files) { expect(file.buffer).toEqual(Buffer.from("png")); expect(file.mimeType).toBe("image/png"); } + expect(ChartJSNodeCanvas).toHaveBeenCalledWith( + expect.objectContaining({ width: 1280, height: 640 }) + ); + expect( + mockRenderToBuffer.mock.calls.every( + ([config]) => config.options.devicePixelRatio === 2 + ) + ).toBe(true); }); }); diff --git a/.github/scripts/js/e2e/report/messenger/charts/data.js b/.github/scripts/js/e2e/report/messenger/charts/data.js new file mode 100644 index 0000000000..f4fa73fe03 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/data.js @@ -0,0 +1,141 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const STATUSES = ["passed", "failed", "errors", "skipped"]; + +const STATUS_COLORS = { + passed: "#3fb950", + failed: "#f85149", + errors: "#d29922", + skipped: "#8b949e", +}; + +const DURATION_COLORS = { + fast: "#7ee787", + medium: "#3fb950", + slow: "#238636", +}; +const DURATION_LABELS = { + fast: "Fast <60s", + medium: "Medium 60-300s", + slow: "Slow >300s", +}; + +const DEFAULT_TOP_N = 15; +const SLOW_THRESHOLD_MS = 300_000; +const MEDIUM_THRESHOLD_MS = 60_000; + +function toSeconds(ms) { + return Number((ms / 1000).toFixed(2)); +} + +function normalize(timing) { + const rawState = String((timing && timing.state) || "errors"); + const rawGroup = (timing && timing.group) || "Top-level Its"; + const name = String((timing && timing.name) || "Unnamed spec"); + const group = String(rawGroup); + const runtimeMs = Number(timing && timing.runtimeMs); + return { + name, + group, + fullName: group === name ? name : `${group} / ${name}`, + state: STATUSES.includes(rawState) ? rawState : "errors", + runtimeMs: Number.isFinite(runtimeMs) && runtimeMs > 0 ? runtimeMs : 0, + }; +} + +function emptyStatusMap() { + return Object.fromEntries(STATUSES.map((status) => [status, 0])); +} + +// Single pass over the spec timings feeds every chart builder below. +function aggregate(specTimings) { + const all = []; + const byGroup = new Map(); + + for (const raw of specTimings || []) { + const timing = normalize(raw); + all.push(timing); + + let bucket = byGroup.get(timing.group); + if (!bucket) { + bucket = { + statusCount: emptyStatusMap(), + statusDurations: emptyStatusMap(), + total: 0, + }; + byGroup.set(timing.group, bucket); + } + bucket.statusCount[timing.state] += 1; + bucket.statusDurations[timing.state] += timing.runtimeMs; + bucket.total += timing.runtimeMs; + } + + return { all, byGroup }; +} + +function durationBucket(timing) { + if (timing.runtimeMs > SLOW_THRESHOLD_MS) { + return "slow"; + } + if (timing.runtimeMs >= MEDIUM_THRESHOLD_MS) { + return "medium"; + } + return "fast"; +} + +function formatSeconds(seconds) { + return `${Number(seconds || 0).toFixed(seconds >= 10 ? 0 : 1)}s`; +} + +function formatCount(count) { + return String(Number(count || 0)); +} + +function mergeChartOptions(base, override) { + return { + ...base, + ...override, + plugins: { ...(base.plugins || {}), ...(override.plugins || {}) }, + scales: { ...(base.scales || {}), ...(override.scales || {}) }, + }; +} + +function baseOptions(title, extra = {}) { + return mergeChartOptions( + { + responsive: false, + animation: false, + plugins: { + title: { display: true, text: title }, + legend: { display: true }, + }, + }, + extra + ); +} + +module.exports = { + STATUSES, + STATUS_COLORS, + DURATION_COLORS, + DURATION_LABELS, + DEFAULT_TOP_N, + toSeconds, + normalize, + aggregate, + durationBucket, + formatSeconds, + formatCount, + emptyStatusMap, + baseOptions, +}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/index.js b/.github/scripts/js/e2e/report/messenger/charts/index.js new file mode 100644 index 0000000000..972fd25090 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/index.js @@ -0,0 +1,29 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { aggregate } = require("./data"); +const featureDurationStatus = require("./builders/feature-duration-status"); +const slowestSpecs = require("./builders/slowest-specs"); + +// Order of charts matches the order of attachments in the messenger thread. +const CHART_BUILDERS = [featureDurationStatus]; + +function buildClusterChartConfigs(specTimings) { + const data = aggregate(specTimings); + return CHART_BUILDERS.map((build) => build(data)); +} + +module.exports = { + CHART_BUILDERS, + buildClusterChartConfigs, + slowestSpecs, +}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/plugins.js b/.github/scripts/js/e2e/report/messenger/charts/plugins.js new file mode 100644 index 0000000000..06c411fccd --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/plugins.js @@ -0,0 +1,91 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const STACK_LABEL_INLINE_MIN_PX = 34; + +function isStackedChart(chart, isHorizontal) { + const axisKey = isHorizontal ? "x" : "y"; + return Boolean(chart.options.scales?.[axisKey]?.stacked); +} + +function drawHorizontalInline(ctx, props, label, isStacked, barWidth) { + ctx.textAlign = + isStacked && barWidth > STACK_LABEL_INLINE_MIN_PX ? "center" : "left"; + ctx.fillText( + label, + isStacked && barWidth > STACK_LABEL_INLINE_MIN_PX + ? (props.x + props.base) / 2 + : props.x + 6, + props.y + ); +} + +function drawVerticalAbove(ctx, props, label) { + ctx.textAlign = "center"; + ctx.fillText(label, props.x, props.y - 8); +} + +function drawValueLabels(chart, _args, options) { + const { ctx, data } = chart; + const formatter = options && options.formatter; + if (typeof formatter !== "function") { + return; + } + + ctx.save(); + ctx.font = "12px sans-serif"; + ctx.fillStyle = "#24292f"; + ctx.textBaseline = "middle"; + + chart.getSortedVisibleDatasetMetas().forEach((meta) => { + meta.data.forEach((element, dataIndex) => { + const rawValue = data.datasets[meta.index].data[dataIndex]; + if (!rawValue) { + return; + } + + const label = formatter(rawValue, { + chart, + dataIndex, + datasetIndex: meta.index, + }); + if (!label) { + return; + } + + const props = element.getProps(["x", "y", "base"], true); + const isHorizontal = chart.options.indexAxis === "y"; + const isStacked = isStackedChart(chart, isHorizontal); + + if (isHorizontal) { + const barWidth = Math.abs(props.x - props.base); + drawHorizontalInline(ctx, props, label, isStacked, barWidth); + return; + } + + drawVerticalAbove(ctx, props, label); + }); + }); + + ctx.restore(); +} + +const valueLabelsPlugin = { + id: "valueLabels", + afterDatasetsDraw: drawValueLabels, +}; + +module.exports = { + STACK_LABEL_INLINE_MIN_PX, + valueLabelsPlugin, + drawValueLabels, +}; diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index c6e369d131..f9d6bd2bcf 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -61,9 +61,17 @@ function parseLoopApiPayload(responseText, core) { * @param {string} [rootId] Optional thread root id for reply posts. * @param {LoopClientCore} core GitHub core API. * @param {string[]} [fileIds] Uploaded Loop file ids to attach. + * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<Record<string, any>>} Parsed Loop API response. */ -async function postToLoopApi(loop, message, rootId, core, fileIds = []) { +async function postToLoopApi( + loop, + message, + rootId, + core, + fileIds = [], + { fetch: fetchFn = globalThis.fetch } = {} +) { const body = { channel_id: loop.channelId, message, @@ -71,7 +79,7 @@ async function postToLoopApi(loop, message, rootId, core, fileIds = []) { ...(fileIds.length > 0 ? { file_ids: fileIds } : {}), }; - const response = await fetch(loop.apiUrl, { + const response = await fetchFn(loop.apiUrl, { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, @@ -103,7 +111,8 @@ function getFilesApiUrl(apiUrl) { * @param {string} fileName File name shown in Loop. * @param {Buffer} buffer File content. * @param {LoopClientCore} core GitHub core API. - * @param {string} [mimeType] File MIME type. + * @param {string} mimeType File MIME type. + * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<string>} Uploaded Loop file id. */ async function uploadFileToLoop( @@ -111,13 +120,14 @@ async function uploadFileToLoop( fileName, buffer, core, - mimeType = "image/png" + mimeType, + { fetch: fetchFn = globalThis.fetch } = {} ) { const formData = new FormData(); formData.append("channel_id", loop.channelId); formData.append("files", new Blob([buffer], { type: mimeType }), fileName); - const response = await fetch(getFilesApiUrl(loop.apiUrl), { + const response = await fetchFn(getFilesApiUrl(loop.apiUrl), { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, @@ -150,13 +160,17 @@ async function uploadFileToLoop( * * @param {LoopPublishParams} params Message payload and Loop credentials. * @param {LoopClientCore} core GitHub core API. + * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<void>} */ async function makeThreadedReportInLoop( { message, threadMessages, loop }, - core + core, + { fetch: fetchFn = globalThis.fetch } = {} ) { - const rootPost = await postToLoopApi(loop, message, undefined, core); + const rootPost = await postToLoopApi(loop, message, undefined, core, [], { + fetch: fetchFn, + }); if (!rootPost.id) { throw new Error( @@ -168,28 +182,32 @@ async function makeThreadedReportInLoop( const files = Array.isArray(reply.files) ? reply.files : []; let fileIds = []; if (files.length > 0) { - try { - fileIds = await Promise.all( - files.map((file) => - uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType) - ) - ); - } catch (error) { - if (loop.strictFileUploads) { - throw error; - } - - // Posting the reply without attachments is preferable to losing the - // whole thread (e.g. failed-tests table) when Loop rejects file - // uploads, typically with HTTP 403 when the bot token lacks the - // upload_file permission. - core.warning( - `Loop file upload failed; posting reply without attachments: ${error.message}` + const results = await Promise.allSettled( + files.map((file) => + uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType, { + fetch: fetchFn, + }) + ) + ); + fileIds = results + .filter((result) => result.status === "fulfilled") + .map((result) => result.value); + + const failures = results.filter((result) => result.status === "rejected"); + for (const failure of failures) { + const reason = failure.reason; + const details = reason && reason.message ? reason.message : reason; + core.warning(`Loop file upload failed for one attachment: ${details}`); + } + if (loop.strictFileUploads && failures.length > 0) { + throw new Error( + "Strict file uploads enabled; at least one attachment failed" ); - fileIds = []; } } - await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds); + await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds, { + fetch: fetchFn, + }); } } diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 3554fc5844..5e55dd37fc 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -94,12 +94,12 @@ describe("loop-client", () => { message: "reply", files: [ { - name: "slowest-specs.png", + name: "feature-duration-status.png", buffer: Buffer.from("one"), mimeType: "image/png", }, { - name: "feature-duration-status.png", + name: "feature-duration-status-2.png", buffer: Buffer.from("two"), mimeType: "image/png", }, @@ -120,7 +120,7 @@ describe("loop-client", () => { }); }); - test("posts the reply without attachments when file upload fails", async () => { + test("posts the reply with successful attachments when one upload fails", async () => { const loop = { apiUrl: "https://loop.example.invalid/api/v4/posts", channelId: "channel-id", @@ -133,6 +133,11 @@ describe("loop-client", () => { status: 201, text: async () => JSON.stringify({ id: "root-post-id" }), }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-one" }] }), + }, { ok: false, status: 403, @@ -160,8 +165,13 @@ describe("loop-client", () => { message: "reply", files: [ { - name: "chart.png", - buffer: Buffer.from("image-bytes"), + name: "feature-duration-status.png", + buffer: Buffer.from("one"), + mimeType: "image/png", + }, + { + name: "feature-duration-status-2.png", + buffer: Buffer.from("two"), mimeType: "image/png", }, ], @@ -172,23 +182,27 @@ describe("loop-client", () => { core ); - expect(global.fetch).toHaveBeenCalledTimes(3); + expect(global.fetch).toHaveBeenCalledTimes(4); expect(global.fetch.mock.calls[0][0]).toBe(loop.apiUrl); expect(global.fetch.mock.calls[1][0]).toBe( "https://loop.example.invalid/api/v4/files" ); - expect(global.fetch.mock.calls[2][0]).toBe(loop.apiUrl); + expect(global.fetch.mock.calls[2][0]).toBe( + "https://loop.example.invalid/api/v4/files" + ); + expect(global.fetch.mock.calls[3][0]).toBe(loop.apiUrl); - const replyBody = JSON.parse(global.fetch.mock.calls[2][1].body); + const replyBody = JSON.parse(global.fetch.mock.calls[3][1].body); expect(replyBody.root_id).toBe("root-post-id"); expect(replyBody.message).toBe("reply"); - expect(replyBody).not.toHaveProperty("file_ids"); + expect(replyBody.file_ids).toEqual(["file-one"]); expect(core.warning).toHaveBeenCalledWith( expect.stringContaining( - "Loop file upload failed; posting reply without attachments" + "Loop file upload failed for one attachment: Loop file upload failed with status 403" ) ); + expect(core.warning).toHaveBeenCalledTimes(1); }); test("fails when strict file upload mode is enabled", async () => { @@ -235,8 +249,45 @@ describe("loop-client", () => { createCore() ) ).rejects.toThrow( - "Loop file upload failed with status 403: permission denied" + "Strict file uploads enabled; at least one attachment failed" ); expect(global.fetch).toHaveBeenCalledTimes(2); }); + + test("uses injected fetch without touching the global fetch", async () => { + const originalFetch = globalThis.fetch; + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }; + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + const fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [{ message: "reply", files: [] }], + loop, + }, + createCore(), + { fetch } + ); + + expect(fetch).toHaveBeenCalledTimes(2); + expect(globalThis.fetch).toBe(originalFetch); + }); }); diff --git a/.github/scripts/js/e2e/report/render-slowest-for-describe.js b/.github/scripts/js/e2e/report/render-slowest-for-describe.js new file mode 100644 index 0000000000..ba7e535d87 --- /dev/null +++ b/.github/scripts/js/e2e/report/render-slowest-for-describe.js @@ -0,0 +1,175 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); +const path = require("path"); + +const { aggregate } = require("./messenger/charts/data"); +const { slowestSpecs } = require("./messenger/charts"); +const { + renderChartBuffer, + sanitizeFilenamePart, +} = require("./messenger/charts/chart-renderer"); +const { parseGinkgoReport } = require("./shared/ginkgo-report-utils"); + +function parseArgs(argv) { + const args = {}; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith("--")) { + continue; + } + + const key = token.slice(2); + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + args[key] = true; + continue; + } + + args[key] = value; + index += 1; + } + + return args; +} + +function deriveStorageType(reportPath, fallbackStorage) { + const baseName = path.basename(reportPath); + const datedMatch = baseName.match( + /^e2e_report_(.+)_(\d{4}-\d{2}-\d{2}.*)\.json$/ + ); + if (datedMatch) { + return datedMatch[1]; + } + + const genericMatch = baseName.match(/^e2e_report_(.+?)_.*\.json$/); + if (genericMatch) { + return genericMatch[1]; + } + + if (fallbackStorage) { + return fallbackStorage; + } + + throw new Error( + `Unable to derive storage type from file name "${baseName}". Pass --storage.` + ); +} + +function readReport(jsonPath) { + const content = fs.readFileSync(jsonPath, "utf8"); + const report = JSON.parse(content); + if (Array.isArray(report.specTimings)) { + return report; + } + + return { + specTimings: parseGinkgoReport(content).specTimings, + }; +} + +function availableDescribes(specTimings) { + return [ + ...new Set( + (specTimings || []) + .map((timing) => String((timing && timing.group) || "").trim()) + .filter(Boolean) + ), + ].sort((left, right) => left.localeCompare(right)); +} + +async function renderSlowestForDescribe({ + jsonPath, + describe, + outDir = "tmp/test-ci/report/out", + storage, +}) { + if (!jsonPath) { + throw new Error("--json is required"); + } + if (!describe) { + throw new Error("--describe is required"); + } + + const resolvedJsonPath = path.resolve(jsonPath); + const report = readReport(resolvedJsonPath); + const specTimings = Array.isArray(report.specTimings) + ? report.specTimings + : []; + const filteredTimings = specTimings.filter( + (timing) => String((timing && timing.group) || "") === describe + ); + + if (filteredTimings.length === 0) { + const describes = availableDescribes(specTimings); + throw new Error( + [ + `No specs found for Describe "${describe}".`, + "Available Describes:", + ...(describes.length > 0 ? describes : ["<none>"]).map( + (name) => `- ${name}` + ), + ].join("\n") + ); + } + + const chart = slowestSpecs(aggregate(filteredTimings)); + const buffer = await renderChartBuffer(chart); + const storageName = + storage || + report.storageType || + report.cluster || + deriveStorageType(resolvedJsonPath); + const fileName = `${sanitizeFilenamePart(storageName)}-${sanitizeFilenamePart( + describe + )}-${chart.name}.png`; + const chartDir = path.resolve(outDir, "charts"); + const targetPath = path.join(chartDir, fileName); + + fs.mkdirSync(chartDir, { recursive: true }); + fs.writeFileSync(targetPath, buffer); + + return targetPath; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log( + "Usage: node .github/scripts/js/e2e/report/render-slowest-for-describe.js --json <report.json> --describe <Describe> [--out-dir <dir>] [--storage <name>]" + ); + return; + } + + const targetPath = await renderSlowestForDescribe({ + jsonPath: args.json, + describe: args.describe, + outDir: args["out-dir"], + storage: args.storage, + }); + console.log(targetPath); +} + +if (require.main === module) { + main().catch((error) => { + console.error(`[ERROR] ${error.message}`); + process.exit(1); + }); +} + +module.exports = { + availableDescribes, + deriveStorageType, + renderSlowestForDescribe, +}; diff --git a/.github/scripts/js/e2e/report/render-slowest-for-describe.test.js b/.github/scripts/js/e2e/report/render-slowest-for-describe.test.js new file mode 100644 index 0000000000..69d6e48dfc --- /dev/null +++ b/.github/scripts/js/e2e/report/render-slowest-for-describe.test.js @@ -0,0 +1,89 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); +const path = require("path"); + +jest.mock("./messenger/charts/chart-renderer", () => ({ + renderChartBuffer: jest.fn().mockResolvedValue(Buffer.from("png")), + sanitizeFilenamePart: (value) => + String(value || "cluster").replace(/[^a-zA-Z0-9_-]+/g, "_") || "cluster", +})); + +const { renderSlowestForDescribe } = require("./render-slowest-for-describe"); +const { topDescribes } = require("./render-top-describes"); +const { withTempDir } = require("./shared/test-utils"); + +function writeReport(tempDir, report) { + const jsonPath = path.join(tempDir, "e2e_report_nfs_2026-05-15.json"); + fs.writeFileSync(jsonPath, JSON.stringify(report)); + return jsonPath; +} + +describe("render-slowest-for-describe", () => { + test("renders one slowest-specs PNG for the requested Describe", async () => + withTempDir("render-slowest-for-describe", async (tempDir) => { + const jsonPath = writeReport(tempDir, { + storageType: "nfs", + specTimings: [ + { name: "fast", group: "VM", state: "passed", runtimeMs: 10_000 }, + { name: "slow", group: "VM", state: "passed", runtimeMs: 90_000 }, + { name: "disk", group: "Disk", state: "passed", runtimeMs: 30_000 }, + ], + }); + + const targetPath = await renderSlowestForDescribe({ + jsonPath, + describe: "VM", + outDir: tempDir, + }); + + expect(targetPath).toBe( + path.join(tempDir, "charts", "nfs-VM-slowest-specs.png") + ); + expect(fs.readFileSync(targetPath)).toEqual(Buffer.from("png")); + })); + + test("fails with available Describe names when the requested one is absent", async () => + withTempDir("render-slowest-for-describe", async (tempDir) => { + const jsonPath = writeReport(tempDir, { + specTimings: [ + { name: "disk", group: "Disk", state: "passed", runtimeMs: 30_000 }, + { name: "vm", group: "VM", state: "passed", runtimeMs: 10_000 }, + ], + }); + + await expect( + renderSlowestForDescribe({ + jsonPath, + describe: "Network", + outDir: tempDir, + }) + ).rejects.toThrow("Available Describes:\n- Disk\n- VM"); + })); +}); + +describe("render-top-describes", () => { + test("selects top Describes by total duration with name tiebreak", () => { + expect( + topDescribes( + [ + { group: "VM", runtimeMs: 30_000 }, + { group: "Disk", runtimeMs: 20_000 }, + { group: "Network", runtimeMs: 20_000 }, + { group: "VM", runtimeMs: 5_000 }, + ], + 2 + ) + ).toEqual(["VM", "Disk"]); + }); +}); diff --git a/.github/scripts/js/e2e/report/render-top-describes.js b/.github/scripts/js/e2e/report/render-top-describes.js new file mode 100644 index 0000000000..c57101e506 --- /dev/null +++ b/.github/scripts/js/e2e/report/render-top-describes.js @@ -0,0 +1,157 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); + +const { listMatchingFiles } = require("./shared/fs-utils"); +const { REPORT_FILE_PATTERN } = require("./shared/report-model"); +const { getReportClusterKey } = require("./messenger/model"); +const { + deriveStorageType, + renderSlowestForDescribe, +} = require("./render-slowest-for-describe"); + +function parseArgs(argv) { + const args = {}; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith("--")) { + continue; + } + + const key = token.slice(2); + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + args[key] = true; + continue; + } + + args[key] = value; + index += 1; + } + + return args; +} + +function topDescribes(specTimings, topN = 5) { + const totals = new Map(); + + for (const timing of specTimings || []) { + const group = String((timing && timing.group) || "").trim(); + if (!group) { + continue; + } + + totals.set(group, (totals.get(group) || 0) + Number(timing.runtimeMs || 0)); + } + + return [...totals.entries()] + .sort( + (left, right) => right[1] - left[1] || left[0].localeCompare(right[0]) + ) + .slice(0, topN) + .map(([describe]) => describe); +} + +function readReport(jsonPath) { + return JSON.parse(fs.readFileSync(jsonPath, "utf8")); +} + +async function renderTopDescribesForCluster({ + jsonPath, + storage, + outDir = "tmp/ci-report/out", + topN = 5, +}) { + const report = readReport(jsonPath); + const describes = topDescribes(report.specTimings, topN); + const storageName = + storage || getReportClusterKey(report) || deriveStorageType(jsonPath); + const renderedFiles = []; + + for (const describe of describes) { + renderedFiles.push( + await renderSlowestForDescribe({ + jsonPath, + describe, + outDir, + storage: storageName, + }) + ); + } + + return renderedFiles; +} + +async function renderTopDescribes({ + core = console, + reportsDir = "downloaded-artifacts", + outDir = "tmp/ci-report/out", + topN = 5, +} = {}) { + const reportFiles = listMatchingFiles(reportsDir, REPORT_FILE_PATTERN); + const renderedFiles = []; + + for (const reportFile of reportFiles) { + try { + const files = await renderTopDescribesForCluster({ + jsonPath: reportFile, + outDir, + topN, + }); + renderedFiles.push(...files); + if (core.info) { + core.info( + `Rendered ${files.length} slowest-specs charts from ${reportFile}` + ); + } + } catch (error) { + if (core.warning) { + core.warning( + `Unable to render top Describe charts for ${reportFile}: ${error.message}` + ); + } + } + } + + return renderedFiles; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log( + "Usage: node .github/scripts/js/e2e/report/render-top-describes.js [--reports-dir downloaded-artifacts] [--out-dir tmp/ci-report/out] [--top-n 5]" + ); + return; + } + + const files = await renderTopDescribes({ + reportsDir: args["reports-dir"], + outDir: args["out-dir"], + topN: Number(args["top-n"] || 5), + }); + files.forEach((file) => console.log(file)); +} + +if (require.main === module) { + main().catch((error) => { + console.error(`[ERROR] ${error.message}`); + process.exit(1); + }); +} + +module.exports = renderTopDescribes; +module.exports.renderTopDescribes = renderTopDescribes; +module.exports.renderTopDescribesForCluster = renderTopDescribesForCluster; +module.exports.topDescribes = topDescribes; diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index eca0925605..40b2f3c1aa 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -209,7 +209,7 @@ function parseGinkgoReport(jsonContent) { const leafText = String(specReport.LeafNodeText || "").trim(); specTimings.push({ name: leafText, - group: hierarchyParts[0] || leafText, + group: hierarchyParts[0] || "Top-level Its", state: metricKey, runtimeMs: runtimeMs(specReport.RunTime), labels: flattenLabels([ diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 814a70dda8..45b39ffe0a 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -480,6 +480,7 @@ jobs: - uses: actions/checkout@v4 - name: Download E2E report artifacts + # v8 exists upstream and keeps report artifact downloads on the current action major. uses: actions/download-artifact@v8 continue-on-error: true id: download-artifacts-pattern @@ -515,3 +516,17 @@ jobs: script: | const renderMessengerReport = require('./.github/scripts/js/e2e/report/messenger-report'); await renderMessengerReport({core}); + + - name: Render top-5 slowest Describes per cluster + uses: actions/github-script@v7 + with: + script: | + const renderTopDescribes = require('./.github/scripts/js/e2e/report/render-top-describes'); + await renderTopDescribes({core}); + + - name: Upload top-5 slowest Describe charts + uses: actions/upload-artifact@v4 + with: + name: e2e-report-slowest-by-describe + path: tmp/ci-report/out/charts/ + if-no-files-found: warn diff --git a/Taskfile.yaml b/Taskfile.yaml index 95f2670d15..a6050a6827 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -50,6 +50,36 @@ tasks: - which jq >/dev/null || (echo "jq not found."; exit 1) silent: true + report:render:slowest: + desc: "Render slowest-specs chart for one Describe (DESCRIBE=..., JSON=...)" + silent: true + vars: + JSON: '{{.JSON | default ""}}' + DESCRIBE: '{{.DESCRIBE | default ""}}' + OUT_DIR: '{{.OUT_DIR | default "tmp/test-ci/report/out"}}' + cmds: + - test -n "{{.JSON}}" || (echo "JSON=... is required"; exit 1) + - test -n "{{.DESCRIBE}}" || (echo "DESCRIBE=... is required"; exit 1) + - >- + node .github/scripts/js/e2e/report/render-slowest-for-describe.js + --json "{{.JSON}}" + --describe "{{.DESCRIBE}}" --out-dir "{{.OUT_DIR}}" + + report:render:top-slowest: + desc: "Render slowest-specs charts for top-N slowest Describes" + silent: true + vars: + REPORTS_DIR: '{{.REPORTS_DIR | default "tmp/test-ci/report/out"}}' + OUT_DIR: '{{.OUT_DIR | default "tmp/test-ci/report/out"}}' + TOP_N: "{{.TOP_N | default 5}}" + cmds: + - mkdir -p "{{.OUT_DIR}}" "{{.REPORTS_DIR}}" + - >- + node .github/scripts/js/e2e/report/render-top-describes.js + --reports-dir "{{.REPORTS_DIR}}" + --out-dir "{{.OUT_DIR}}" + --top-n "{{.TOP_N}}" + check-helm: cmds: - which helm >/dev/null || (echo "helm not found."; exit 1) From 9dadce90a636b6d38341af2b3327a99a9f4c1f76 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Thu, 21 May 2026 18:16:46 +0300 Subject: [PATCH 15/24] fix(ci, observability): write slowest charts to tmp charts Use tmp/charts as the local and CI output directory for generated slowest-specs PNG artifacts. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../scripts/js/e2e/report/render-slowest-for-describe.js | 4 ++-- .github/scripts/js/e2e/report/render-top-describes.js | 6 +++--- .github/workflows/e2e-matrix.yml | 2 +- Taskfile.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/scripts/js/e2e/report/render-slowest-for-describe.js b/.github/scripts/js/e2e/report/render-slowest-for-describe.js index ba7e535d87..04365d5c24 100644 --- a/.github/scripts/js/e2e/report/render-slowest-for-describe.js +++ b/.github/scripts/js/e2e/report/render-slowest-for-describe.js @@ -92,7 +92,7 @@ function availableDescribes(specTimings) { async function renderSlowestForDescribe({ jsonPath, describe, - outDir = "tmp/test-ci/report/out", + outDir = "tmp", storage, }) { if (!jsonPath) { @@ -147,7 +147,7 @@ async function main() { const args = parseArgs(process.argv.slice(2)); if (args.help) { console.log( - "Usage: node .github/scripts/js/e2e/report/render-slowest-for-describe.js --json <report.json> --describe <Describe> [--out-dir <dir>] [--storage <name>]" + "Usage: node .github/scripts/js/e2e/report/render-slowest-for-describe.js --json <report.json> --describe <Describe> [--out-dir tmp] [--storage <name>]" ); return; } diff --git a/.github/scripts/js/e2e/report/render-top-describes.js b/.github/scripts/js/e2e/report/render-top-describes.js index c57101e506..a1424585e2 100644 --- a/.github/scripts/js/e2e/report/render-top-describes.js +++ b/.github/scripts/js/e2e/report/render-top-describes.js @@ -70,7 +70,7 @@ function readReport(jsonPath) { async function renderTopDescribesForCluster({ jsonPath, storage, - outDir = "tmp/ci-report/out", + outDir = "tmp", topN = 5, }) { const report = readReport(jsonPath); @@ -96,7 +96,7 @@ async function renderTopDescribesForCluster({ async function renderTopDescribes({ core = console, reportsDir = "downloaded-artifacts", - outDir = "tmp/ci-report/out", + outDir = "tmp", topN = 5, } = {}) { const reportFiles = listMatchingFiles(reportsDir, REPORT_FILE_PATTERN); @@ -131,7 +131,7 @@ async function main() { const args = parseArgs(process.argv.slice(2)); if (args.help) { console.log( - "Usage: node .github/scripts/js/e2e/report/render-top-describes.js [--reports-dir downloaded-artifacts] [--out-dir tmp/ci-report/out] [--top-n 5]" + "Usage: node .github/scripts/js/e2e/report/render-top-describes.js [--reports-dir downloaded-artifacts] [--out-dir tmp] [--top-n 5]" ); return; } diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 45b39ffe0a..a4c16a52de 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -528,5 +528,5 @@ jobs: uses: actions/upload-artifact@v4 with: name: e2e-report-slowest-by-describe - path: tmp/ci-report/out/charts/ + path: tmp/charts/ if-no-files-found: warn diff --git a/Taskfile.yaml b/Taskfile.yaml index a6050a6827..7ec0a72839 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -56,7 +56,7 @@ tasks: vars: JSON: '{{.JSON | default ""}}' DESCRIBE: '{{.DESCRIBE | default ""}}' - OUT_DIR: '{{.OUT_DIR | default "tmp/test-ci/report/out"}}' + OUT_DIR: '{{.OUT_DIR | default "tmp"}}' cmds: - test -n "{{.JSON}}" || (echo "JSON=... is required"; exit 1) - test -n "{{.DESCRIBE}}" || (echo "DESCRIBE=... is required"; exit 1) @@ -70,7 +70,7 @@ tasks: silent: true vars: REPORTS_DIR: '{{.REPORTS_DIR | default "tmp/test-ci/report/out"}}' - OUT_DIR: '{{.OUT_DIR | default "tmp/test-ci/report/out"}}' + OUT_DIR: '{{.OUT_DIR | default "tmp"}}' TOP_N: "{{.TOP_N | default 5}}" cmds: - mkdir -p "{{.OUT_DIR}}" "{{.REPORTS_DIR}}" From bf31aec6009c87556a6351a1afb56d1a1c65d4c8 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 22 May 2026 16:16:46 +0300 Subject: [PATCH 16/24] refactor(ci, observability): move e2e charts to python Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../scripts/js/e2e/report/messenger-report.js | 4 +- .../js/e2e/report/messenger-report.test.js | 22 +- .../js/e2e/report/messenger/chart-files.js | 48 ++ .../e2e/report/messenger/chart-files.test.js | 60 +++ .../__snapshots__/chart-config.test.js.snap | 101 ---- .../builders/feature-duration-status.js | 76 --- .../charts/builders/slowest-specs.js | 120 ----- .../report/messenger/charts/chart-config.js | 13 - .../messenger/charts/chart-config.test.js | 77 --- .../report/messenger/charts/chart-renderer.js | 99 ---- .../messenger/charts/chart-renderer.test.js | 53 -- .../js/e2e/report/messenger/charts/data.js | 141 ------ .../js/e2e/report/messenger/charts/index.js | 29 -- .../js/e2e/report/messenger/charts/plugins.js | 91 ---- .../js/e2e/report/messenger/markdown.js | 16 +- .../e2e/report/render-slowest-for-describe.js | 175 ------- .../render-slowest-for-describe.test.js | 89 ---- .../js/e2e/report/render-top-describes.js | 157 ------ .github/scripts/js/package.json | 4 - .github/scripts/python/e2e_report/__init__.py | 11 + .github/scripts/python/e2e_report/charts.py | 474 ++++++++++++++++++ .../scripts/python/e2e_report/charts_test.py | 131 +++++ .github/scripts/python/requirements.txt | 1 + .github/workflows/e2e-matrix.yml | 35 +- Taskfile.yaml | 6 +- 25 files changed, 774 insertions(+), 1259 deletions(-) create mode 100644 .github/scripts/js/e2e/report/messenger/chart-files.js create mode 100644 .github/scripts/js/e2e/report/messenger/chart-files.test.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-config.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-config.test.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-renderer.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/data.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/index.js delete mode 100644 .github/scripts/js/e2e/report/messenger/charts/plugins.js delete mode 100644 .github/scripts/js/e2e/report/render-slowest-for-describe.js delete mode 100644 .github/scripts/js/e2e/report/render-slowest-for-describe.test.js delete mode 100644 .github/scripts/js/e2e/report/render-top-describes.js create mode 100644 .github/scripts/python/e2e_report/__init__.py create mode 100644 .github/scripts/python/e2e_report/charts.py create mode 100644 .github/scripts/python/e2e_report/charts_test.py create mode 100644 .github/scripts/python/requirements.txt diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index 20eec223be..6a1c836c72 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -14,7 +14,7 @@ const fs = require("fs"); const { listMatchingFiles } = require("./shared/fs-utils"); const { REPORT_FILE_PATTERN } = require("./shared/report-model"); -const { renderClusterCharts } = require("./messenger/charts/chart-renderer"); +const { getClusterChartFiles } = require("./messenger/chart-files"); const { makeThreadedReportInLoop } = require("./messenger/loop-client"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { @@ -114,7 +114,7 @@ async function buildMessengerMessages({ }) { const orderedReports = readReports(reportsDir, configuredClusters, core); const threadMessages = await buildThreadMessages(orderedReports, { - renderClusterCharts, + getClusterChartFiles, core, }); return { diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 551d3d5c90..950380ca98 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -13,12 +13,12 @@ const fs = require("fs"); const path = require("path"); -jest.mock("./messenger/charts/chart-renderer", () => ({ - renderClusterCharts: jest.fn().mockResolvedValue([]), +jest.mock("./messenger/chart-files", () => ({ + getClusterChartFiles: jest.fn().mockResolvedValue([]), })); const renderMessengerReport = require("./messenger-report"); -const { renderClusterCharts } = require("./messenger/charts/chart-renderer"); +const { getClusterChartFiles } = require("./messenger/chart-files"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { createCore, withTempDir } = require("./shared/test-utils"); @@ -34,8 +34,8 @@ describe("messenger-report", () => { delete process.env.LOOP_STRICT_DELIVERY; delete process.env.LOOP_STRICT_FILE_UPLOAD; delete global.fetch; - renderClusterCharts.mockReset(); - renderClusterCharts.mockResolvedValue([]); + getClusterChartFiles.mockReset(); + getClusterChartFiles.mockResolvedValue([]); }); test("reads normalized messenger config from env", () => { @@ -186,7 +186,7 @@ describe("messenger-report", () => { buffer: Buffer.from("png"), mimeType: "image/png", }; - renderClusterCharts.mockResolvedValue([chartFile]); + getClusterChartFiles.mockResolvedValue([chartFile]); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ @@ -243,9 +243,11 @@ describe("messenger-report", () => { ); })); - test("warns and surfaces a placeholder when chart rendering fails", async () => + test("warns and surfaces a placeholder when chart files are unavailable", async () => inTempDir(async (tempDir) => { - renderClusterCharts.mockRejectedValue(new Error("canvas unavailable")); + getClusterChartFiles.mockRejectedValue( + new Error("chart cli unavailable") + ); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ @@ -278,7 +280,7 @@ describe("messenger-report", () => { expect(core.warning).toHaveBeenCalledWith( expect.stringContaining( - "Unable to render duration charts for cluster replicated" + "Unable to prepare duration chart files for cluster replicated" ) ); expect(result.threadMessages).toEqual([ @@ -589,7 +591,7 @@ describe("messenger-report", () => { buffer: Buffer.from("png"), mimeType: "image/png", }; - renderClusterCharts.mockResolvedValue([chartFile]); + getClusterChartFiles.mockResolvedValue([chartFile]); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ diff --git a/.github/scripts/js/e2e/report/messenger/chart-files.js b/.github/scripts/js/e2e/report/messenger/chart-files.js new file mode 100644 index 0000000000..c4f6852f19 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/chart-files.js @@ -0,0 +1,48 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); +const path = require("path"); + +const { getReportClusterKey } = require("./model"); + +const defaultManifestPath = "tmp/messenger-charts/manifest.json"; + +function readChartManifest(manifestPath) { + if (!fs.existsSync(manifestPath)) { + return { clusters: {} }; + } + + return JSON.parse(fs.readFileSync(manifestPath, "utf8")); +} + +function getClusterChartFiles(report) { + const clusterKey = getReportClusterKey(report); + if (!clusterKey) { + return []; + } + + const manifestPath = process.env.CHARTS_MANIFEST || defaultManifestPath; + const manifest = readChartManifest(manifestPath); + const files = ((manifest.clusters || {})[clusterKey] || []).map((file) => ({ + name: file.name, + buffer: fs.readFileSync(path.resolve(file.path)), + mimeType: file.mimeType || "image/png", + })); + + return files; +} + +module.exports = { + getClusterChartFiles, + readChartManifest, +}; diff --git a/.github/scripts/js/e2e/report/messenger/chart-files.test.js b/.github/scripts/js/e2e/report/messenger/chart-files.test.js new file mode 100644 index 0000000000..36a57012fa --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/chart-files.test.js @@ -0,0 +1,60 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); +const path = require("path"); + +const { getClusterChartFiles } = require("./chart-files"); +const { withTempDir } = require("../shared/test-utils"); + +describe("chart-files", () => { + afterEach(() => { + delete process.env.CHARTS_MANIFEST; + }); + + test("returns no files when the manifest is missing", () => { + expect(getClusterChartFiles({ cluster: "replicated" })).toEqual([]); + }); + + test("loads chart files listed in the Python manifest", async () => + withTempDir("chart-files", async (tempDir) => { + const chartPath = path.join( + tempDir, + "replicated-feature-duration-status.png" + ); + const manifestPath = path.join(tempDir, "manifest.json"); + fs.writeFileSync(chartPath, Buffer.from("png")); + fs.writeFileSync( + manifestPath, + JSON.stringify({ + clusters: { + replicated: [ + { + name: "replicated-feature-duration-status.png", + path: chartPath, + mimeType: "image/png", + }, + ], + }, + }) + ); + process.env.CHARTS_MANIFEST = manifestPath; + + const files = await getClusterChartFiles({ cluster: "replicated" }); + + expect(files.map(({ name }) => name)).toEqual([ + "replicated-feature-duration-status.png", + ]); + expect(files[0].buffer).toEqual(Buffer.from("png")); + expect(files[0].mimeType).toBe("image/png"); + })); +}); diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap deleted file mode 100644 index 7875ec0ca6..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap +++ /dev/null @@ -1,101 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`chart-config builds deterministic cluster chart configs 1`] = ` -Array [ - Object { - "config": Object { - "data": Object { - "datasets": Array [ - Object { - "backgroundColor": "#3fb950", - "data": Array [ - 0, - 55, - 0, - ], - "label": "passed", - }, - Object { - "backgroundColor": "#f85149", - "data": Array [ - 0, - 301, - 0, - ], - "label": "failed", - }, - Object { - "backgroundColor": "#d29922", - "data": Array [ - 601, - 0, - 0, - ], - "label": "errors", - }, - Object { - "backgroundColor": "#8b949e", - "data": Array [ - 0, - 0, - 60, - ], - "label": "skipped", - }, - ], - "labels": Array [ - "Network", - "VM", - "Disk", - ], - }, - "options": Object { - "animation": false, - "indexAxis": "y", - "plugins": Object { - "legend": Object { - "display": true, - }, - "title": Object { - "display": true, - "text": "Overall durations for Describes", - }, - "valueLabels": Object { - "formatter": [Function], - }, - }, - "responsive": false, - "scales": Object { - "x": Object { - "beginAtZero": true, - "stacked": true, - "ticks": Object { - "stepSize": 60, - }, - "title": Object { - "display": true, - "text": "Duration, seconds", - }, - }, - "y": Object { - "stacked": true, - }, - }, - }, - "plugins": Array [ - Object { - "afterDatasetsDraw": [Function], - "id": "valueLabels", - }, - ], - "type": "bar", - }, - "name": "feature-duration-status", - "size": Object { - "height": 640, - "pixelRatio": 2, - "width": 1280, - }, - }, -] -`; diff --git a/.github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js b/.github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js deleted file mode 100644 index 371df4a6ea..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/builders/feature-duration-status.js +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const { - STATUSES, - STATUS_COLORS, - toSeconds, - formatSeconds, - baseOptions, -} = require("../data"); -const { valueLabelsPlugin } = require("../plugins"); - -function sortedGroups(byGroup, compareFn) { - return [...byGroup.entries()].sort(compareFn); -} - -function problemCount(group) { - return group.statusCount.failed + group.statusCount.errors; -} - -function featureDurationStatus({ byGroup }) { - // Most-broken features go to the top: failures desc, then total runtime desc, - // then alphabetical for a stable order. - const entries = sortedGroups(byGroup, (left, right) => { - return ( - problemCount(right[1]) - problemCount(left[1]) || - right[1].total - left[1].total || - left[0].localeCompare(right[0]) - ); - }); - - const labels = entries.map(([name]) => name); - const datasets = STATUSES.map((status) => ({ - label: status, - data: entries.map(([, group]) => toSeconds(group.statusDurations[status])), - backgroundColor: STATUS_COLORS[status], - })); - const height = Math.max(640, 120 + labels.length * 36); - - return { - name: "feature-duration-status", - size: { width: 1280, height, pixelRatio: 2 }, - config: { - type: "bar", - data: { labels, datasets }, - options: baseOptions("Overall durations for Describes", { - indexAxis: "y", - plugins: { - legend: { display: true }, - valueLabels: { formatter: formatSeconds }, - }, - scales: { - x: { - stacked: true, - beginAtZero: true, - ticks: { stepSize: 60 }, - title: { display: true, text: "Duration, seconds" }, - }, - y: { stacked: true }, - }, - }), - plugins: [valueLabelsPlugin], - }, - }; -} - -module.exports = featureDurationStatus; diff --git a/.github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js b/.github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js deleted file mode 100644 index 3045ad8b0b..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/builders/slowest-specs.js +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const { - STATUS_COLORS, - DURATION_COLORS, - DURATION_LABELS, - DEFAULT_TOP_N, - toSeconds, - durationBucket, - formatSeconds, - baseOptions, -} = require("../data"); -const { valueLabelsPlugin } = require("../plugins"); - -function formatSlowestSpecLabel(seconds, { chart, dataIndex }) { - const dataset = chart.data.datasets[0] || {}; - const state = (dataset.states || [])[dataIndex]; - const suffix = ["failed", "errors"].includes(state) ? ` [${state}]` : ""; - return `${formatSeconds(seconds)}${suffix}`; -} - -function slowestSpecsLegendLabels() { - const durationLabels = Object.entries(DURATION_COLORS).map( - ([key, color]) => ({ - text: DURATION_LABELS[key], - fillStyle: color, - strokeStyle: color, - lineWidth: 0, - }) - ); - const statusOverlays = [ - ["failed", "Failed border"], - ["errors", "Error border"], - ].map(([status, text]) => ({ - text, - fillStyle: "#ffffff", - strokeStyle: STATUS_COLORS[status], - lineWidth: 3, - })); - - return [...durationLabels, ...statusOverlays]; -} - -function slowestSpecs({ all }, topN = DEFAULT_TOP_N) { - const top = [...all] - .sort( - (left, right) => - right.runtimeMs - left.runtimeMs || - left.fullName.localeCompare(right.fullName) - ) - .slice(0, topN); - const annotated = top.map((timing) => { - const isFailure = timing.state === "failed" || timing.state === "errors"; - return { - timing, - bucketColor: DURATION_COLORS[durationBucket(timing)], - borderColor: isFailure ? STATUS_COLORS[timing.state] : "transparent", - borderWidth: isFailure ? 3 : 0, - }; - }); - - return { - name: "slowest-specs", - size: { width: 2048, height: 720, pixelRatio: 2 }, - config: { - type: "bar", - data: { - labels: annotated.map(({ timing }) => timing.fullName), - datasets: [ - { - label: "Duration, seconds", - data: annotated.map(({ timing }) => toSeconds(timing.runtimeMs)), - backgroundColor: annotated.map(({ bucketColor }) => bucketColor), - borderColor: annotated.map(({ borderColor }) => borderColor), - borderWidth: annotated.map(({ borderWidth }) => borderWidth), - barPercentage: 0.55, - categoryPercentage: 0.7, - states: annotated.map(({ timing }) => timing.state), - }, - ], - }, - options: baseOptions( - "Top slowest successful specs and failed specs (It/Entry)", - { - indexAxis: "y", - plugins: { - legend: { - display: true, - labels: { generateLabels: slowestSpecsLegendLabels }, - }, - valueLabels: { formatter: formatSlowestSpecLabel }, - }, - scales: { - x: { - beginAtZero: true, - ticks: { stepSize: 60 }, - title: { display: true, text: "Duration, seconds" }, - }, - }, - layout: { - padding: { top: 16, bottom: 8 }, - }, - } - ), - plugins: [valueLabelsPlugin], - }, - }; -} - -module.exports = slowestSpecs; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js deleted file mode 100644 index 89566136f4..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module.exports = require("."); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js deleted file mode 100644 index c678113b65..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const { buildClusterChartConfigs, slowestSpecs } = require("./chart-config"); -const { aggregate } = require("./data"); - -const specTimings = [ - { name: "fast pass", group: "VM", state: "passed", runtimeMs: 10_000 }, - { name: "medium skip", group: "Disk", state: "skipped", runtimeMs: 60_000 }, - { name: "slow fail", group: "VM", state: "failed", runtimeMs: 301_000 }, - { name: "error", group: "Network", state: "errors", runtimeMs: 601_000 }, - { name: "passing peer", group: "VM", state: "passed", runtimeMs: 45_000 }, -]; - -describe("chart-config", () => { - test("builds deterministic cluster chart configs", () => { - expect(buildClusterChartConfigs(specTimings)).toMatchSnapshot(); - }); - - test("returns the messenger chart config in display order", () => { - const configs = buildClusterChartConfigs(specTimings); - expect(configs.map(({ name }) => name)).toEqual([ - "feature-duration-status", - ]); - }); - - test("handles an empty spec timings list", () => { - const configs = buildClusterChartConfigs([]); - expect(configs).toHaveLength(1); - const labelsByName = Object.fromEntries( - configs.map(({ name, config }) => [name, config.data.labels]) - ); - expect(labelsByName["feature-duration-status"]).toEqual([]); - }); - - test("normalizes non-numeric runtimes to zero", () => { - const configs = buildClusterChartConfigs([ - { runtimeMs: "slow", name: "x", group: "g", state: "passed" }, - ]); - const numericValues = configs.flatMap(({ config }) => - config.data.datasets.flatMap((dataset) => - dataset.data.filter((value) => typeof value === "number") - ) - ); - - expect(numericValues).toContain(0); - expect(numericValues.some((value) => Number.isNaN(value))).toBe(false); - }); - - test("builds slowest specs sorted by duration descending", () => { - const chart = slowestSpecs( - aggregate([ - { name: "middle", group: "VM", state: "passed", runtimeMs: 90_000 }, - { name: "slow b", group: "VM", state: "passed", runtimeMs: 180_000 }, - { name: "slow a", group: "Disk", state: "passed", runtimeMs: 180_000 }, - { name: "fast", group: "Network", state: "passed", runtimeMs: 10_000 }, - ]) - ); - - expect(chart.config.data.labels).toEqual([ - "Disk / slow a", - "VM / slow b", - "VM / middle", - "Network / fast", - ]); - expect(chart.config.data.datasets[0].data).toEqual([180, 180, 90, 10]); - }); -}); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js deleted file mode 100644 index 83798782ea..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const { buildClusterChartConfigs } = require("."); - -const defaultChartSize = { width: 1280, height: 640, pixelRatio: 2 }; -const canvasInstances = new Map(); - -// Module-level singleton: ChartJSNodeCanvas startup (loading chart.js + setting -// up the cairo-backed canvas) is non-trivial, and the renderer is stateless -// between renderToBuffer calls. Reusing it across clusters keeps memory usage -// flat when the messenger report grows. -function loadChartRenderer({ width, height, pixelRatio } = defaultChartSize) { - const rendererKey = `${width}x${height}@${pixelRatio}`; - if (!canvasInstances.has(rendererKey)) { - const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); - canvasInstances.set( - rendererKey, - new ChartJSNodeCanvas({ - width, - height, - backgroundColour: "#ffffff", - }) - ); - } - - return canvasInstances.get(rendererKey); -} - -function normalizeChartSize(size) { - return { - ...defaultChartSize, - ...(size || {}), - }; -} - -function withDevicePixelRatio(config, pixelRatio) { - return { - ...config, - options: { - ...(config.options || {}), - devicePixelRatio: pixelRatio, - }, - }; -} - -async function renderChartBuffer({ config, size }) { - const chartSize = normalizeChartSize(size); - const renderer = loadChartRenderer(chartSize); - return renderer.renderToBuffer( - withDevicePixelRatio(config, chartSize.pixelRatio), - "image/png" - ); -} - -function sanitizeFilenamePart(value) { - const fallback = "cluster"; - const safe = String(value || fallback).replace(/[^a-zA-Z0-9_-]+/g, "_"); - return safe || fallback; -} - -async function renderClusterCharts(report) { - if ( - !Array.isArray(report && report.specTimings) || - report.specTimings.length === 0 - ) { - return []; - } - - const configs = buildClusterChartConfigs(report.specTimings); - const clusterName = sanitizeFilenamePart( - report.cluster || report.storageType || "cluster" - ); - - return Promise.all( - configs.map(async ({ name, config, size }) => { - return { - name: `${clusterName}-${name}.png`, - buffer: await renderChartBuffer({ config, size }), - mimeType: "image/png", - }; - }) - ); -} - -module.exports = { - renderClusterCharts, - renderChartBuffer, - sanitizeFilenamePart, -}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js deleted file mode 100644 index a30dd40725..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const mockRenderToBuffer = jest.fn().mockResolvedValue(Buffer.from("png")); - -jest.mock("chartjs-node-canvas", () => ({ - ChartJSNodeCanvas: jest.fn().mockImplementation(() => ({ - renderToBuffer: mockRenderToBuffer, - })), -})); - -const { renderClusterCharts } = require("./chart-renderer"); -const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); - -describe("chart-renderer", () => { - test("returns no files when spec timings are empty", async () => { - await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); - }); - - test("renders messenger cluster chart images", async () => { - const files = await renderClusterCharts({ - cluster: "replicated", - specTimings: [ - { name: "slow", group: "VM", state: "passed", runtimeMs: 90_000 }, - ], - }); - - expect(files.map(({ name }) => name)).toEqual([ - "replicated-feature-duration-status.png", - ]); - for (const file of files) { - expect(file.buffer).toEqual(Buffer.from("png")); - expect(file.mimeType).toBe("image/png"); - } - expect(ChartJSNodeCanvas).toHaveBeenCalledWith( - expect.objectContaining({ width: 1280, height: 640 }) - ); - expect( - mockRenderToBuffer.mock.calls.every( - ([config]) => config.options.devicePixelRatio === 2 - ) - ).toBe(true); - }); -}); diff --git a/.github/scripts/js/e2e/report/messenger/charts/data.js b/.github/scripts/js/e2e/report/messenger/charts/data.js deleted file mode 100644 index f4fa73fe03..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/data.js +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const STATUSES = ["passed", "failed", "errors", "skipped"]; - -const STATUS_COLORS = { - passed: "#3fb950", - failed: "#f85149", - errors: "#d29922", - skipped: "#8b949e", -}; - -const DURATION_COLORS = { - fast: "#7ee787", - medium: "#3fb950", - slow: "#238636", -}; -const DURATION_LABELS = { - fast: "Fast <60s", - medium: "Medium 60-300s", - slow: "Slow >300s", -}; - -const DEFAULT_TOP_N = 15; -const SLOW_THRESHOLD_MS = 300_000; -const MEDIUM_THRESHOLD_MS = 60_000; - -function toSeconds(ms) { - return Number((ms / 1000).toFixed(2)); -} - -function normalize(timing) { - const rawState = String((timing && timing.state) || "errors"); - const rawGroup = (timing && timing.group) || "Top-level Its"; - const name = String((timing && timing.name) || "Unnamed spec"); - const group = String(rawGroup); - const runtimeMs = Number(timing && timing.runtimeMs); - return { - name, - group, - fullName: group === name ? name : `${group} / ${name}`, - state: STATUSES.includes(rawState) ? rawState : "errors", - runtimeMs: Number.isFinite(runtimeMs) && runtimeMs > 0 ? runtimeMs : 0, - }; -} - -function emptyStatusMap() { - return Object.fromEntries(STATUSES.map((status) => [status, 0])); -} - -// Single pass over the spec timings feeds every chart builder below. -function aggregate(specTimings) { - const all = []; - const byGroup = new Map(); - - for (const raw of specTimings || []) { - const timing = normalize(raw); - all.push(timing); - - let bucket = byGroup.get(timing.group); - if (!bucket) { - bucket = { - statusCount: emptyStatusMap(), - statusDurations: emptyStatusMap(), - total: 0, - }; - byGroup.set(timing.group, bucket); - } - bucket.statusCount[timing.state] += 1; - bucket.statusDurations[timing.state] += timing.runtimeMs; - bucket.total += timing.runtimeMs; - } - - return { all, byGroup }; -} - -function durationBucket(timing) { - if (timing.runtimeMs > SLOW_THRESHOLD_MS) { - return "slow"; - } - if (timing.runtimeMs >= MEDIUM_THRESHOLD_MS) { - return "medium"; - } - return "fast"; -} - -function formatSeconds(seconds) { - return `${Number(seconds || 0).toFixed(seconds >= 10 ? 0 : 1)}s`; -} - -function formatCount(count) { - return String(Number(count || 0)); -} - -function mergeChartOptions(base, override) { - return { - ...base, - ...override, - plugins: { ...(base.plugins || {}), ...(override.plugins || {}) }, - scales: { ...(base.scales || {}), ...(override.scales || {}) }, - }; -} - -function baseOptions(title, extra = {}) { - return mergeChartOptions( - { - responsive: false, - animation: false, - plugins: { - title: { display: true, text: title }, - legend: { display: true }, - }, - }, - extra - ); -} - -module.exports = { - STATUSES, - STATUS_COLORS, - DURATION_COLORS, - DURATION_LABELS, - DEFAULT_TOP_N, - toSeconds, - normalize, - aggregate, - durationBucket, - formatSeconds, - formatCount, - emptyStatusMap, - baseOptions, -}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/index.js b/.github/scripts/js/e2e/report/messenger/charts/index.js deleted file mode 100644 index 972fd25090..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/index.js +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const { aggregate } = require("./data"); -const featureDurationStatus = require("./builders/feature-duration-status"); -const slowestSpecs = require("./builders/slowest-specs"); - -// Order of charts matches the order of attachments in the messenger thread. -const CHART_BUILDERS = [featureDurationStatus]; - -function buildClusterChartConfigs(specTimings) { - const data = aggregate(specTimings); - return CHART_BUILDERS.map((build) => build(data)); -} - -module.exports = { - CHART_BUILDERS, - buildClusterChartConfigs, - slowestSpecs, -}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/plugins.js b/.github/scripts/js/e2e/report/messenger/charts/plugins.js deleted file mode 100644 index 06c411fccd..0000000000 --- a/.github/scripts/js/e2e/report/messenger/charts/plugins.js +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const STACK_LABEL_INLINE_MIN_PX = 34; - -function isStackedChart(chart, isHorizontal) { - const axisKey = isHorizontal ? "x" : "y"; - return Boolean(chart.options.scales?.[axisKey]?.stacked); -} - -function drawHorizontalInline(ctx, props, label, isStacked, barWidth) { - ctx.textAlign = - isStacked && barWidth > STACK_LABEL_INLINE_MIN_PX ? "center" : "left"; - ctx.fillText( - label, - isStacked && barWidth > STACK_LABEL_INLINE_MIN_PX - ? (props.x + props.base) / 2 - : props.x + 6, - props.y - ); -} - -function drawVerticalAbove(ctx, props, label) { - ctx.textAlign = "center"; - ctx.fillText(label, props.x, props.y - 8); -} - -function drawValueLabels(chart, _args, options) { - const { ctx, data } = chart; - const formatter = options && options.formatter; - if (typeof formatter !== "function") { - return; - } - - ctx.save(); - ctx.font = "12px sans-serif"; - ctx.fillStyle = "#24292f"; - ctx.textBaseline = "middle"; - - chart.getSortedVisibleDatasetMetas().forEach((meta) => { - meta.data.forEach((element, dataIndex) => { - const rawValue = data.datasets[meta.index].data[dataIndex]; - if (!rawValue) { - return; - } - - const label = formatter(rawValue, { - chart, - dataIndex, - datasetIndex: meta.index, - }); - if (!label) { - return; - } - - const props = element.getProps(["x", "y", "base"], true); - const isHorizontal = chart.options.indexAxis === "y"; - const isStacked = isStackedChart(chart, isHorizontal); - - if (isHorizontal) { - const barWidth = Math.abs(props.x - props.base); - drawHorizontalInline(ctx, props, label, isStacked, barWidth); - return; - } - - drawVerticalAbove(ctx, props, label); - }); - }); - - ctx.restore(); -} - -const valueLabelsPlugin = { - id: "valueLabels", - afterDatasetsDraw: drawValueLabels, -}; - -module.exports = { - STACK_LABEL_INLINE_MIN_PX, - valueLabelsPlugin, - drawValueLabels, -}; diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index 29cc6c9948..17f96c805f 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -341,23 +341,23 @@ function hasSpecTimings(report) { return Array.isArray(report.specTimings) && report.specTimings.length > 0; } -function renderChartCaption(_files, chartsUnavailable) { +function buildChartCaption(_files, chartsUnavailable) { return chartsUnavailable ? "Charts unavailable." : ""; } /** - * Builds optional per-cluster thread messages for failed tests and duration charts. + * Builds optional per-cluster thread messages for failed tests and chart attachments. * * @param {Array<Record<string, any>>} orderedReports Cluster reports in display order. * @param {{ - * renderClusterCharts?: function(Record<string, any>): Promise<Array<{name: string, buffer: Buffer, mimeType: string}>>, + * getClusterChartFiles?: function(Record<string, any>): Promise<Array<{name: string, buffer: Buffer, mimeType: string}>>, * core?: {warning?: function(string): void} * }} [options] * @returns {Promise<Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>>} Markdown thread payloads. */ async function buildThreadMessages( orderedReports, - { renderClusterCharts, core } = {} + { getClusterChartFiles, core } = {} ) { const testsReports = orderedReports.filter((report) => isTestResultReport(report) @@ -370,14 +370,14 @@ async function buildThreadMessages( let files = []; let chartsUnavailable = false; - if (renderClusterCharts && hasSpecTimings(report)) { + if (getClusterChartFiles && hasSpecTimings(report)) { try { - files = await renderClusterCharts(report); + files = await getClusterChartFiles(report); } catch (error) { chartsUnavailable = true; if (core && typeof core.warning === "function") { core.warning( - `Unable to render duration charts for cluster ${ + `Unable to prepare duration chart files for cluster ${ getReportClusterKey(report) || "unknown" }: ${error.message}` ); @@ -401,7 +401,7 @@ async function buildThreadMessages( messageParts.push(`**${formatClusterLink(report)}**`); } - const chartCaption = renderChartCaption(files, chartsUnavailable); + const chartCaption = buildChartCaption(files, chartsUnavailable); if (chartCaption) { messageParts.push(chartCaption); } diff --git a/.github/scripts/js/e2e/report/render-slowest-for-describe.js b/.github/scripts/js/e2e/report/render-slowest-for-describe.js deleted file mode 100644 index 04365d5c24..0000000000 --- a/.github/scripts/js/e2e/report/render-slowest-for-describe.js +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const fs = require("fs"); -const path = require("path"); - -const { aggregate } = require("./messenger/charts/data"); -const { slowestSpecs } = require("./messenger/charts"); -const { - renderChartBuffer, - sanitizeFilenamePart, -} = require("./messenger/charts/chart-renderer"); -const { parseGinkgoReport } = require("./shared/ginkgo-report-utils"); - -function parseArgs(argv) { - const args = {}; - - for (let index = 0; index < argv.length; index += 1) { - const token = argv[index]; - if (!token.startsWith("--")) { - continue; - } - - const key = token.slice(2); - const value = argv[index + 1]; - if (!value || value.startsWith("--")) { - args[key] = true; - continue; - } - - args[key] = value; - index += 1; - } - - return args; -} - -function deriveStorageType(reportPath, fallbackStorage) { - const baseName = path.basename(reportPath); - const datedMatch = baseName.match( - /^e2e_report_(.+)_(\d{4}-\d{2}-\d{2}.*)\.json$/ - ); - if (datedMatch) { - return datedMatch[1]; - } - - const genericMatch = baseName.match(/^e2e_report_(.+?)_.*\.json$/); - if (genericMatch) { - return genericMatch[1]; - } - - if (fallbackStorage) { - return fallbackStorage; - } - - throw new Error( - `Unable to derive storage type from file name "${baseName}". Pass --storage.` - ); -} - -function readReport(jsonPath) { - const content = fs.readFileSync(jsonPath, "utf8"); - const report = JSON.parse(content); - if (Array.isArray(report.specTimings)) { - return report; - } - - return { - specTimings: parseGinkgoReport(content).specTimings, - }; -} - -function availableDescribes(specTimings) { - return [ - ...new Set( - (specTimings || []) - .map((timing) => String((timing && timing.group) || "").trim()) - .filter(Boolean) - ), - ].sort((left, right) => left.localeCompare(right)); -} - -async function renderSlowestForDescribe({ - jsonPath, - describe, - outDir = "tmp", - storage, -}) { - if (!jsonPath) { - throw new Error("--json is required"); - } - if (!describe) { - throw new Error("--describe is required"); - } - - const resolvedJsonPath = path.resolve(jsonPath); - const report = readReport(resolvedJsonPath); - const specTimings = Array.isArray(report.specTimings) - ? report.specTimings - : []; - const filteredTimings = specTimings.filter( - (timing) => String((timing && timing.group) || "") === describe - ); - - if (filteredTimings.length === 0) { - const describes = availableDescribes(specTimings); - throw new Error( - [ - `No specs found for Describe "${describe}".`, - "Available Describes:", - ...(describes.length > 0 ? describes : ["<none>"]).map( - (name) => `- ${name}` - ), - ].join("\n") - ); - } - - const chart = slowestSpecs(aggregate(filteredTimings)); - const buffer = await renderChartBuffer(chart); - const storageName = - storage || - report.storageType || - report.cluster || - deriveStorageType(resolvedJsonPath); - const fileName = `${sanitizeFilenamePart(storageName)}-${sanitizeFilenamePart( - describe - )}-${chart.name}.png`; - const chartDir = path.resolve(outDir, "charts"); - const targetPath = path.join(chartDir, fileName); - - fs.mkdirSync(chartDir, { recursive: true }); - fs.writeFileSync(targetPath, buffer); - - return targetPath; -} - -async function main() { - const args = parseArgs(process.argv.slice(2)); - if (args.help) { - console.log( - "Usage: node .github/scripts/js/e2e/report/render-slowest-for-describe.js --json <report.json> --describe <Describe> [--out-dir tmp] [--storage <name>]" - ); - return; - } - - const targetPath = await renderSlowestForDescribe({ - jsonPath: args.json, - describe: args.describe, - outDir: args["out-dir"], - storage: args.storage, - }); - console.log(targetPath); -} - -if (require.main === module) { - main().catch((error) => { - console.error(`[ERROR] ${error.message}`); - process.exit(1); - }); -} - -module.exports = { - availableDescribes, - deriveStorageType, - renderSlowestForDescribe, -}; diff --git a/.github/scripts/js/e2e/report/render-slowest-for-describe.test.js b/.github/scripts/js/e2e/report/render-slowest-for-describe.test.js deleted file mode 100644 index 69d6e48dfc..0000000000 --- a/.github/scripts/js/e2e/report/render-slowest-for-describe.test.js +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const fs = require("fs"); -const path = require("path"); - -jest.mock("./messenger/charts/chart-renderer", () => ({ - renderChartBuffer: jest.fn().mockResolvedValue(Buffer.from("png")), - sanitizeFilenamePart: (value) => - String(value || "cluster").replace(/[^a-zA-Z0-9_-]+/g, "_") || "cluster", -})); - -const { renderSlowestForDescribe } = require("./render-slowest-for-describe"); -const { topDescribes } = require("./render-top-describes"); -const { withTempDir } = require("./shared/test-utils"); - -function writeReport(tempDir, report) { - const jsonPath = path.join(tempDir, "e2e_report_nfs_2026-05-15.json"); - fs.writeFileSync(jsonPath, JSON.stringify(report)); - return jsonPath; -} - -describe("render-slowest-for-describe", () => { - test("renders one slowest-specs PNG for the requested Describe", async () => - withTempDir("render-slowest-for-describe", async (tempDir) => { - const jsonPath = writeReport(tempDir, { - storageType: "nfs", - specTimings: [ - { name: "fast", group: "VM", state: "passed", runtimeMs: 10_000 }, - { name: "slow", group: "VM", state: "passed", runtimeMs: 90_000 }, - { name: "disk", group: "Disk", state: "passed", runtimeMs: 30_000 }, - ], - }); - - const targetPath = await renderSlowestForDescribe({ - jsonPath, - describe: "VM", - outDir: tempDir, - }); - - expect(targetPath).toBe( - path.join(tempDir, "charts", "nfs-VM-slowest-specs.png") - ); - expect(fs.readFileSync(targetPath)).toEqual(Buffer.from("png")); - })); - - test("fails with available Describe names when the requested one is absent", async () => - withTempDir("render-slowest-for-describe", async (tempDir) => { - const jsonPath = writeReport(tempDir, { - specTimings: [ - { name: "disk", group: "Disk", state: "passed", runtimeMs: 30_000 }, - { name: "vm", group: "VM", state: "passed", runtimeMs: 10_000 }, - ], - }); - - await expect( - renderSlowestForDescribe({ - jsonPath, - describe: "Network", - outDir: tempDir, - }) - ).rejects.toThrow("Available Describes:\n- Disk\n- VM"); - })); -}); - -describe("render-top-describes", () => { - test("selects top Describes by total duration with name tiebreak", () => { - expect( - topDescribes( - [ - { group: "VM", runtimeMs: 30_000 }, - { group: "Disk", runtimeMs: 20_000 }, - { group: "Network", runtimeMs: 20_000 }, - { group: "VM", runtimeMs: 5_000 }, - ], - 2 - ) - ).toEqual(["VM", "Disk"]); - }); -}); diff --git a/.github/scripts/js/e2e/report/render-top-describes.js b/.github/scripts/js/e2e/report/render-top-describes.js deleted file mode 100644 index a1424585e2..0000000000 --- a/.github/scripts/js/e2e/report/render-top-describes.js +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2026 Flant JSC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const fs = require("fs"); - -const { listMatchingFiles } = require("./shared/fs-utils"); -const { REPORT_FILE_PATTERN } = require("./shared/report-model"); -const { getReportClusterKey } = require("./messenger/model"); -const { - deriveStorageType, - renderSlowestForDescribe, -} = require("./render-slowest-for-describe"); - -function parseArgs(argv) { - const args = {}; - - for (let index = 0; index < argv.length; index += 1) { - const token = argv[index]; - if (!token.startsWith("--")) { - continue; - } - - const key = token.slice(2); - const value = argv[index + 1]; - if (!value || value.startsWith("--")) { - args[key] = true; - continue; - } - - args[key] = value; - index += 1; - } - - return args; -} - -function topDescribes(specTimings, topN = 5) { - const totals = new Map(); - - for (const timing of specTimings || []) { - const group = String((timing && timing.group) || "").trim(); - if (!group) { - continue; - } - - totals.set(group, (totals.get(group) || 0) + Number(timing.runtimeMs || 0)); - } - - return [...totals.entries()] - .sort( - (left, right) => right[1] - left[1] || left[0].localeCompare(right[0]) - ) - .slice(0, topN) - .map(([describe]) => describe); -} - -function readReport(jsonPath) { - return JSON.parse(fs.readFileSync(jsonPath, "utf8")); -} - -async function renderTopDescribesForCluster({ - jsonPath, - storage, - outDir = "tmp", - topN = 5, -}) { - const report = readReport(jsonPath); - const describes = topDescribes(report.specTimings, topN); - const storageName = - storage || getReportClusterKey(report) || deriveStorageType(jsonPath); - const renderedFiles = []; - - for (const describe of describes) { - renderedFiles.push( - await renderSlowestForDescribe({ - jsonPath, - describe, - outDir, - storage: storageName, - }) - ); - } - - return renderedFiles; -} - -async function renderTopDescribes({ - core = console, - reportsDir = "downloaded-artifacts", - outDir = "tmp", - topN = 5, -} = {}) { - const reportFiles = listMatchingFiles(reportsDir, REPORT_FILE_PATTERN); - const renderedFiles = []; - - for (const reportFile of reportFiles) { - try { - const files = await renderTopDescribesForCluster({ - jsonPath: reportFile, - outDir, - topN, - }); - renderedFiles.push(...files); - if (core.info) { - core.info( - `Rendered ${files.length} slowest-specs charts from ${reportFile}` - ); - } - } catch (error) { - if (core.warning) { - core.warning( - `Unable to render top Describe charts for ${reportFile}: ${error.message}` - ); - } - } - } - - return renderedFiles; -} - -async function main() { - const args = parseArgs(process.argv.slice(2)); - if (args.help) { - console.log( - "Usage: node .github/scripts/js/e2e/report/render-top-describes.js [--reports-dir downloaded-artifacts] [--out-dir tmp] [--top-n 5]" - ); - return; - } - - const files = await renderTopDescribes({ - reportsDir: args["reports-dir"], - outDir: args["out-dir"], - topN: Number(args["top-n"] || 5), - }); - files.forEach((file) => console.log(file)); -} - -if (require.main === module) { - main().catch((error) => { - console.error(`[ERROR] ${error.message}`); - process.exit(1); - }); -} - -module.exports = renderTopDescribes; -module.exports.renderTopDescribes = renderTopDescribes; -module.exports.renderTopDescribesForCluster = renderTopDescribesForCluster; -module.exports.topDescribes = topDescribes; diff --git a/.github/scripts/js/package.json b/.github/scripts/js/package.json index b038118441..7d57cc285c 100644 --- a/.github/scripts/js/package.json +++ b/.github/scripts/js/package.json @@ -19,9 +19,5 @@ "eslint": "^10.2.1", "jest": "28.1.2", "prettier": "^2.5.0" - }, - "dependencies": { - "chart.js": "^4.5.1", - "chartjs-node-canvas": "^5.0.0" } } diff --git a/.github/scripts/python/e2e_report/__init__.py b/.github/scripts/python/e2e_report/__init__.py new file mode 100644 index 0000000000..c7e0518801 --- /dev/null +++ b/.github/scripts/python/e2e_report/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/.github/scripts/python/e2e_report/charts.py b/.github/scripts/python/e2e_report/charts.py new file mode 100644 index 0000000000..2843a93dd0 --- /dev/null +++ b/.github/scripts/python/e2e_report/charts.py @@ -0,0 +1,474 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.ticker import FuncFormatter, MultipleLocator # noqa: E402 + + +STATUSES = ("passed", "failed", "errors", "skipped") +STATUS_COLORS = { + "passed": "#00b83f", + "failed": "#ff3333", + "errors": "#d9a300", + "skipped": "#8f9aa3", +} +DURATION_COLORS = { + "fast": "#7ee787", + "medium": "#3fb950", + "slow": "#238636", +} +DEFAULT_TOP_N = 15 +REPORT_FILE_PATTERN = re.compile(r"^e2e_report_.*\.json$") + + +@dataclass(frozen=True) +class Timing: + name: str + group: str + state: str + runtime_ms: float + + @property + def full_name(self) -> str: + return self.name if self.group == self.name else f"{self.group} / {self.name}" + + +def sanitize_filename_part(value: Any) -> str: + safe = re.sub(r"[^a-zA-Z0-9_-]+", "_", str(value or "cluster")) + return safe or "cluster" + + +def to_seconds(ms: float) -> float: + return round(float(ms) / 1000, 2) + + +def normalize_timing(raw: dict[str, Any] | None) -> Timing: + raw = raw or {} + state = str(raw.get("state") or "errors") + if state not in STATUSES: + state = "errors" + + runtime = raw.get("runtimeMs", 0) + try: + runtime_ms = float(runtime) + except (TypeError, ValueError): + runtime_ms = 0 + if not math.isfinite(runtime_ms) or runtime_ms < 0: + runtime_ms = 0 + + return Timing( + name=str(raw.get("name") or "Unnamed spec"), + group=str(raw.get("group") or "Top-level Its"), + state=state, + runtime_ms=runtime_ms, + ) + + +def aggregate(spec_timings: list[dict[str, Any]] | None) -> tuple[list[Timing], dict[str, dict[str, Any]]]: + timings: list[Timing] = [] + by_group: dict[str, dict[str, Any]] = {} + + for raw_timing in spec_timings or []: + timing = normalize_timing(raw_timing) + timings.append(timing) + group = by_group.setdefault( + timing.group, + { + "status_count": {status: 0 for status in STATUSES}, + "status_durations": {status: 0.0 for status in STATUSES}, + "total": 0.0, + }, + ) + group["status_count"][timing.state] += 1 + group["status_durations"][timing.state] += timing.runtime_ms + group["total"] += timing.runtime_ms + + return timings, by_group + + +def duration_bucket(timing: Timing) -> str: + if timing.runtime_ms > 300_000: + return "slow" + if timing.runtime_ms >= 60_000: + return "medium" + return "fast" + + +def format_seconds(seconds: float) -> str: + return f"{seconds:.0f}s" if seconds >= 10 else f"{seconds:.1f}s" + + +def format_axis_seconds(value: float, _position: int) -> str: + return f"{int(value):,}" + + +def next_tick(value: float, step: int) -> int: + if value <= 0: + return step + return int(math.ceil(value / step) * step) + + +def render_feature_duration_status(report: dict[str, Any], output_dir: Path) -> dict[str, str]: + _, by_group = aggregate(report.get("specTimings") or []) + entries = sorted( + by_group.items(), + key=lambda item: ( + -(item[1]["status_count"]["failed"] + item[1]["status_count"]["errors"]), + -item[1]["total"], + item[0], + ), + ) + labels = [name for name, _ in entries] + height = max(6.4, 0.75 + len(labels) * 0.285) + fig, ax = plt.subplots(figsize=(10.24, height), dpi=100) + left = [0.0] * len(entries) + + for status in STATUSES: + values = [to_seconds(group["status_durations"][status]) for _, group in entries] + ax.barh(labels, values, left=left, label=status, color=STATUS_COLORS[status], height=0.72) + for row, (offset, value) in enumerate(zip(left, values)): + if value <= 0: + continue + ax.text( + offset + value / 2, + row, + format_seconds(value), + ha="center", + va="center", + fontsize=6, + color="#333333", + ) + left = [current + value for current, value in zip(left, values)] + + x_limit = next_tick(max(left, default=0), 60) + ax.set_xlim(0, x_limit) + ax.set_title("Overall durations for Describes", fontsize=8, pad=12) + ax.set_xlabel("Duration, seconds", fontsize=7) + ax.invert_yaxis() + if labels: + ax.set_ylim(len(labels) - 0.6, -0.6) + ax.legend( + loc="upper center", + bbox_to_anchor=(0.5, 1.015), + ncol=len(STATUSES), + frameon=False, + fontsize=6, + handlelength=2.4, + columnspacing=0.8, + ) + ax.margins(y=0) + ax.xaxis.set_major_locator(MultipleLocator(60)) + ax.xaxis.set_major_formatter(FuncFormatter(format_axis_seconds)) + ax.grid(axis="x", color="#c8c8c8", alpha=0.45, linewidth=0.5) + ax.grid(axis="y", color="#d9d9d9", alpha=0.55, linewidth=0.5) + ax.set_axisbelow(True) + ax.tick_params(axis="both", labelsize=6, colors="#555555", length=0) + for spine in ax.spines.values(): + spine.set_color("#dddddd") + spine.set_linewidth(0.5) + + cluster_name = sanitize_filename_part(report.get("cluster") or report.get("storageType") or "cluster") + return save_figure(fig, output_dir / f"{cluster_name}-feature-duration-status.png") + + +def render_slowest_specs( + spec_timings: list[dict[str, Any]], + storage_name: str, + describe: str, + output_dir: Path, + top_n: int = DEFAULT_TOP_N, +) -> dict[str, str]: + timings, _ = aggregate(spec_timings) + top = sorted(timings, key=lambda timing: (-timing.runtime_ms, timing.full_name))[:top_n] + labels = [timing.full_name for timing in top] + values = [to_seconds(timing.runtime_ms) for timing in top] + colors = [DURATION_COLORS[duration_bucket(timing)] for timing in top] + edge_colors = [ + STATUS_COLORS[timing.state] if timing.state in {"failed", "errors"} else "none" + for timing in top + ] + line_widths = [2 if timing.state in {"failed", "errors"} else 0 for timing in top] + + fig, ax = plt.subplots(figsize=(20.48, 7.2), dpi=100) + ax.barh(labels, values, color=colors, edgecolor=edge_colors, linewidth=line_widths) + ax.set_title("Top slowest successful specs and failed specs (It/Entry)") + ax.set_xlabel("Duration, seconds") + ax.invert_yaxis() + ax.grid(axis="x", alpha=0.2) + + for row, (seconds, timing) in enumerate(zip(values, top)): + suffix = f" [{timing.state}]" if timing.state in {"failed", "errors"} else "" + ax.text(seconds, row, f" {format_seconds(seconds)}{suffix}", va="center", fontsize=8) + + file_name = ( + f"{sanitize_filename_part(storage_name)}-" + f"{sanitize_filename_part(describe)}-slowest-specs.png" + ) + return save_figure(fig, output_dir / file_name) + + +def save_figure(fig: plt.Figure, target_path: Path) -> dict[str, str]: + target_path.parent.mkdir(parents=True, exist_ok=True) + fig.tight_layout() + fig.savefig(target_path, format="png") + plt.close(fig) + return { + "name": target_path.name, + "path": str(target_path), + "mimeType": "image/png", + } + + +def runtime_ms(value: Any) -> int: + try: + runtime = float(value or 0) + except (TypeError, ValueError): + return 0 + return round(runtime / 1_000_000) if math.isfinite(runtime) else 0 + + +def metric_key_for_state(state: Any) -> str: + normalized = str(state or "").strip().lower() + if normalized in {"passed", "failed"}: + return normalized + if normalized in {"skipped", "pending"}: + return "skipped" + return "errors" + + +def parse_ginkgo_report(payload: Any) -> list[dict[str, Any]]: + suites = payload if isinstance(payload, list) else [payload] + timings: list[dict[str, Any]] = [] + + for suite in suites: + if not isinstance(suite, dict): + continue + for spec_report in suite.get("SpecReports") or []: + if not isinstance(spec_report, dict) or spec_report.get("LeafNodeType") != "It": + continue + hierarchy = [ + str(part).strip() + for part in spec_report.get("ContainerHierarchyTexts") or [] + if str(part).strip() + ] + timings.append( + { + "name": str(spec_report.get("LeafNodeText") or "").strip(), + "group": hierarchy[0] if hierarchy else "Top-level Its", + "state": metric_key_for_state(spec_report.get("State")), + "runtimeMs": runtime_ms(spec_report.get("RunTime")), + } + ) + + return timings + + +def read_report(json_path: str | Path) -> dict[str, Any]: + path = Path(json_path) + payload = json.loads(path.read_text()) + if isinstance(payload, dict) and isinstance(payload.get("specTimings"), list): + return payload + return {"specTimings": parse_ginkgo_report(payload)} + + +def available_describes(spec_timings: list[dict[str, Any]]) -> list[str]: + return sorted( + { + str(timing.get("group") or "").strip() + for timing in spec_timings + if str(timing.get("group") or "").strip() + } + ) + + +def derive_storage_type(report_path: str | Path, fallback_storage: str | None = None) -> str: + base_name = Path(report_path).name + dated_match = re.match(r"^e2e_report_(.+)_(\d{4}-\d{2}-\d{2}.*)\.json$", base_name) + if dated_match: + return dated_match.group(1) + generic_match = re.match(r"^e2e_report_(.+?)_.*\.json$", base_name) + if generic_match: + return generic_match.group(1) + if fallback_storage: + return fallback_storage + raise ValueError(f'Unable to derive storage type from file name "{base_name}". Pass --storage.') + + +def report_cluster_key(report: dict[str, Any]) -> str: + return str(report.get("cluster") or report.get("storageType") or "").strip() + + +def top_describes(spec_timings: list[dict[str, Any]] | None, top_n: int = 5) -> list[str]: + totals: dict[str, float] = {} + for raw_timing in spec_timings or []: + group = str(raw_timing.get("group") or "").strip() + if not group: + continue + try: + runtime = float(raw_timing.get("runtimeMs") or 0) + except (TypeError, ValueError): + runtime = 0 + totals[group] = totals.get(group, 0) + runtime + + return [name for name, _ in sorted(totals.items(), key=lambda item: (-item[1], item[0]))[:top_n]] + + +def render_cluster_charts(report: dict[str, Any], output_dir: Path) -> list[dict[str, str]]: + if not report.get("specTimings"): + return [] + return [render_feature_duration_status(report, output_dir)] + + +def render_messenger_charts( + reports_dir: str | Path = "downloaded-artifacts", + out_dir: str | Path = "tmp/messenger-charts", + manifest_path: str | Path = "tmp/messenger-charts/manifest.json", +) -> dict[str, dict[str, list[dict[str, str]]]]: + output_dir = Path(out_dir) + clusters: dict[str, list[dict[str, str]]] = {} + + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + cluster_name = report_cluster_key(report) or derive_storage_type(report_file) + files = render_cluster_charts(report, output_dir) + if files: + clusters[cluster_name] = files + + manifest = {"clusters": clusters} + target_path = Path(manifest_path) + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)) + return manifest + + +def render_slowest_for_describe( + json_path: str | Path, + describe: str, + out_dir: str | Path = "tmp", + storage: str | None = None, +) -> dict[str, str]: + if not describe: + raise ValueError("--describe is required") + + report = read_report(json_path) + spec_timings = report.get("specTimings") or [] + filtered_timings = [ + timing for timing in spec_timings if str(timing.get("group") or "") == describe + ] + if not filtered_timings: + describes = available_describes(spec_timings) + lines = [ + f'No specs found for Describe "{describe}".', + "Available Describes:", + *(f"- {name}" for name in describes or ["<none>"]), + ] + raise ValueError("\n".join(lines)) + + storage_name = ( + storage + or str(report.get("storageType") or report.get("cluster") or "").strip() + or derive_storage_type(json_path) + ) + return render_slowest_specs(filtered_timings, storage_name, describe, Path(out_dir) / "charts") + + +def list_report_files(reports_dir: str | Path) -> list[Path]: + root = Path(reports_dir) + if not root.exists(): + return [] + return sorted(path for path in root.rglob("*") if path.is_file() and REPORT_FILE_PATTERN.match(path.name)) + + +def render_top_describes( + reports_dir: str | Path = "downloaded-artifacts", + out_dir: str | Path = "tmp", + top_n: int = 5, +) -> list[dict[str, str]]: + rendered_files: list[dict[str, str]] = [] + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + storage_name = report_cluster_key(report) or derive_storage_type(report_file) + for describe in top_describes(report.get("specTimings") or [], top_n): + rendered_files.append( + render_slowest_for_describe( + report_file, + describe, + out_dir=out_dir, + storage=storage_name, + ) + ) + return rendered_files + + +def print_json(files: list[dict[str, str]]) -> None: + print(json.dumps(files, separators=(",", ":"))) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render E2E report charts") + subparsers = parser.add_subparsers(dest="command", required=True) + + messenger = subparsers.add_parser("messenger", help="Render charts for one messenger report") + messenger.add_argument("--json", required=True) + messenger.add_argument("--out-dir", default="tmp") + + messenger_all = subparsers.add_parser("messenger-all", help="Render messenger charts and write a manifest") + messenger_all.add_argument("--reports-dir", default="downloaded-artifacts") + messenger_all.add_argument("--out-dir", default="tmp/messenger-charts") + messenger_all.add_argument("--manifest", default="tmp/messenger-charts/manifest.json") + + slowest = subparsers.add_parser("slowest", help="Render slowest specs for one Describe") + slowest.add_argument("--json", required=True) + slowest.add_argument("--describe", required=True) + slowest.add_argument("--out-dir", default="tmp") + slowest.add_argument("--storage") + + top = subparsers.add_parser("top", help="Render slowest specs for top-N Describes") + top.add_argument("--reports-dir", default="downloaded-artifacts") + top.add_argument("--out-dir", default="tmp") + top.add_argument("--top-n", type=int, default=5) + + args = parser.parse_args() + + if args.command == "messenger": + print_json(render_cluster_charts(read_report(args.json), Path(args.out_dir))) + return + if args.command == "messenger-all": + render_messenger_charts(args.reports_dir, args.out_dir, args.manifest) + print(args.manifest) + return + if args.command == "slowest": + file = render_slowest_for_describe(args.json, args.describe, args.out_dir, args.storage) + print(file["path"]) + return + if args.command == "top": + files = render_top_describes(args.reports_dir, args.out_dir, args.top_n) + for file in files: + print(file["path"]) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/python/e2e_report/charts_test.py b/.github/scripts/python/e2e_report/charts_test.py new file mode 100644 index 0000000000..a4566545f2 --- /dev/null +++ b/.github/scripts/python/e2e_report/charts_test.py @@ -0,0 +1,131 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from e2e_report import charts + + +class ChartsTest(unittest.TestCase): + def test_aggregate_normalizes_timings(self) -> None: + timings, by_group = charts.aggregate( + [ + {"name": "fast", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + {"name": "bad", "group": "VM", "state": "unknown", "runtimeMs": "bad"}, + ] + ) + + self.assertEqual([timing.state for timing in timings], ["passed", "errors"]) + self.assertEqual(timings[1].runtime_ms, 0) + self.assertEqual(by_group["VM"]["status_count"]["errors"], 1) + + def test_top_describes_uses_duration_desc_then_name(self) -> None: + self.assertEqual( + charts.top_describes( + [ + {"group": "VM", "runtimeMs": 30_000}, + {"group": "Disk", "runtimeMs": 20_000}, + {"group": "Network", "runtimeMs": 20_000}, + {"group": "VM", "runtimeMs": 5_000}, + ], + 2, + ), + ["VM", "Disk"], + ) + + def test_render_slowest_for_describe_writes_expected_artifact(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "e2e_report_nfs_2026-05-15.json" + report_path.write_text( + json.dumps( + { + "storageType": "nfs", + "specTimings": [ + {"name": "fast", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, + ], + } + ) + ) + + rendered = charts.render_slowest_for_describe(report_path, "VM", out_dir=temp_dir) + + self.assertEqual(rendered["name"], "nfs-VM-slowest-specs.png") + self.assertEqual(Path(rendered["path"]), Path(temp_dir) / "charts" / rendered["name"]) + self.assertTrue(Path(rendered["path"]).is_file()) + + def test_render_slowest_for_describe_lists_available_describes(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "e2e_report_nfs_2026-05-15.json" + report_path.write_text( + json.dumps( + { + "specTimings": [ + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + ], + } + ) + ) + + with self.assertRaisesRegex(ValueError, "Available Describes:\\n- Disk\\n- VM"): + charts.render_slowest_for_describe(report_path, "Network", out_dir=temp_dir) + + def test_render_cluster_charts_writes_messenger_chart_only(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + files = charts.render_cluster_charts( + { + "cluster": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + Path(temp_dir), + ) + + self.assertEqual([file["name"] for file in files], ["replicated-feature-duration-status.png"]) + self.assertTrue(Path(files[0]["path"]).is_file()) + + def test_render_messenger_charts_writes_manifest(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + reports_dir = Path(temp_dir) / "reports" + out_dir = Path(temp_dir) / "charts" + manifest_path = out_dir / "manifest.json" + reports_dir.mkdir() + (reports_dir / "e2e_report_replicated.json").write_text( + json.dumps( + { + "storageType": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + } + ) + ) + + manifest = charts.render_messenger_charts(reports_dir, out_dir, manifest_path) + + self.assertEqual( + [file["name"] for file in manifest["clusters"]["replicated"]], + ["replicated-feature-duration-status.png"], + ) + self.assertEqual(json.loads(manifest_path.read_text()), manifest) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/python/requirements.txt b/.github/scripts/python/requirements.txt new file mode 100644 index 0000000000..6ccafc3f90 --- /dev/null +++ b/.github/scripts/python/requirements.txt @@ -0,0 +1 @@ +matplotlib diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index a4c16a52de..399861963c 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -489,26 +489,39 @@ jobs: path: downloaded-artifacts/ merge-multiple: false - # Node 20 is required because actions/github-script@v7 ships with Node 20 - # and chartjs-node-canvas pulls a native `canvas` binding whose ABI must - # match the runtime that ultimately requires it. Bump together with - # actions/github-script when v8 (Node 22) is available. - name: Set up Node.js for report rendering uses: actions/setup-node@v4 with: node-version: 20 cache: npm - cache-dependency-path: .github/scripts/js/package-lock.json + cache-dependency-path: .github/scripts/js/package.json + + - name: Set up Python for chart rendering + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: .github/scripts/python/requirements.txt - name: Install E2E report dependencies - run: npm ci + run: | + npm install + python3 -m pip install -r ../python/requirements.txt working-directory: .github/scripts/js + - name: Generate messenger chart files + run: >- + python3 .github/scripts/python/e2e_report/charts.py messenger-all + --reports-dir downloaded-artifacts + --out-dir tmp/messenger-charts + --manifest tmp/messenger-charts/manifest.json + - name: Send results to channel id: render-report uses: actions/github-script@v7 env: EXPECTED_STORAGE_TYPES: '["replicated","nfs"]' + CHARTS_MANIFEST: tmp/messenger-charts/manifest.json LOOP_API_BASE_URL: ${{ secrets.LOOP_API_BASE_URL }} LOOP_CHANNEL_ID: ${{ secrets.LOOP_CHANNEL_ID }} LOOP_TOKEN: ${{ secrets.LOOP_TOKEN }} @@ -518,11 +531,11 @@ jobs: await renderMessengerReport({core}); - name: Render top-5 slowest Describes per cluster - uses: actions/github-script@v7 - with: - script: | - const renderTopDescribes = require('./.github/scripts/js/e2e/report/render-top-describes'); - await renderTopDescribes({core}); + run: >- + python3 .github/scripts/python/e2e_report/charts.py top + --reports-dir downloaded-artifacts + --out-dir tmp + --top-n 5 - name: Upload top-5 slowest Describe charts uses: actions/upload-artifact@v4 diff --git a/Taskfile.yaml b/Taskfile.yaml index 7ec0a72839..36e186cd4c 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -61,8 +61,8 @@ tasks: - test -n "{{.JSON}}" || (echo "JSON=... is required"; exit 1) - test -n "{{.DESCRIBE}}" || (echo "DESCRIBE=... is required"; exit 1) - >- - node .github/scripts/js/e2e/report/render-slowest-for-describe.js - --json "{{.JSON}}" + python3 .github/scripts/python/e2e_report/charts.py slowest + --json "{{.JSON}}" --describe "{{.DESCRIBE}}" --out-dir "{{.OUT_DIR}}" report:render:top-slowest: @@ -75,7 +75,7 @@ tasks: cmds: - mkdir -p "{{.OUT_DIR}}" "{{.REPORTS_DIR}}" - >- - node .github/scripts/js/e2e/report/render-top-describes.js + python3 .github/scripts/python/e2e_report/charts.py top --reports-dir "{{.REPORTS_DIR}}" --out-dir "{{.OUT_DIR}}" --top-n "{{.TOP_N}}" From 0e0db2a654c29acbcd4a929572803740a5b3f3bb Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 22 May 2026 18:39:29 +0300 Subject: [PATCH 17/24] fix(ci, observability): apply e2e chart review fixes Align Python chart rendering with the JS report contract, make chart output directories explicit, harden top chart rendering, and expand coverage for parser and CLI behavior. Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .github/scripts/js/.prettierrc | 3 + .../scripts/js/e2e/report/cluster-report.js | 87 +-- .../js/e2e/report/cluster-report.test.js | 124 +--- .../scripts/js/e2e/report/messenger-report.js | 34 +- .../js/e2e/report/messenger-report.test.js | 65 +- .../js/e2e/report/messenger/chart-files.js | 1 - .../e2e/report/messenger/chart-files.test.js | 9 +- .../scripts/js/e2e/report/messenger/config.js | 4 +- .../js/e2e/report/messenger/loop-client.js | 49 +- .../e2e/report/messenger/loop-client.test.js | 32 +- .../js/e2e/report/messenger/markdown.js | 72 +- .../e2e/report/shared/ginkgo-report-utils.js | 57 +- .../js/e2e/report/shared/report-model.js | 43 +- .github/scripts/python/e2e_report/charts.py | 222 ++++-- .../scripts/python/e2e_report/charts_test.py | 291 +++++++- .github/scripts/python/requirements.txt | 2 +- .github/workflows/e2e-matrix.yml | 16 +- Taskfile.yaml | 4 +- tmp/test-ci/report/run-local-report.js | 657 ++++++++++++++++++ 19 files changed, 1226 insertions(+), 546 deletions(-) create mode 100644 .github/scripts/js/.prettierrc create mode 100644 tmp/test-ci/report/run-local-report.js diff --git a/.github/scripts/js/.prettierrc b/.github/scripts/js/.prettierrc new file mode 100644 index 0000000000..963354f231 --- /dev/null +++ b/.github/scripts/js/.prettierrc @@ -0,0 +1,3 @@ +{ + "printWidth": 120 +} diff --git a/.github/scripts/js/e2e/report/cluster-report.js b/.github/scripts/js/e2e/report/cluster-report.js index 5cd550470c..6e606342b2 100644 --- a/.github/scripts/js/e2e/report/cluster-report.js +++ b/.github/scripts/js/e2e/report/cluster-report.js @@ -13,10 +13,7 @@ const fs = require("fs"); const { findSingleMatchingFile } = require("./shared/fs-utils"); -const { - parseGinkgoOutput, - parseGinkgoReport, -} = require("./shared/ginkgo-report-utils"); +const { parseGinkgoOutput, parseGinkgoReport } = require("./shared/ginkgo-report-utils"); const { archivedReportPattern, buildClusterStatus, @@ -103,11 +100,7 @@ function readClusterReportConfigFromEnv(env = process.env) { }; } -const requiredClusterReportConfigKeys = [ - "storageType", - "reportsDir", - "reportFile", -]; +const requiredClusterReportConfigKeys = ["storageType", "reportsDir", "reportFile"]; function requireClusterReportConfig(config) { for (const key of requiredClusterReportConfigKeys) { @@ -148,9 +141,7 @@ async function listWorkflowRunJobs(github, context) { } function findWorkflowJob(jobs, pipelineJobName, jobName) { - const nestedJobName = pipelineJobName - ? `${pipelineJobName} / ${jobName}` - : ""; + const nestedJobName = pipelineJobName ? `${pipelineJobName} / ${jobName}` : ""; return ( jobs.find((job) => job.name === nestedJobName) || @@ -169,8 +160,7 @@ function readStageResultsFromEnv(env = process.env) { const stageResults = {}; for (const { name, needsJobId } of workflowStages) { - stageResults[name] = - String((needs[needsJobId] || {}).result || "").trim() || "skipped"; + stageResults[name] = String((needs[needsJobId] || {}).result || "").trim() || "skipped"; } return stageResults; } @@ -184,9 +174,7 @@ async function readStageJobUrlsFromApi(github, context, config, core) { if (job) { stageJobUrls[name] = job.html_url || ""; } else { - core.warning( - `Unable to find workflow job "${displayName}" for E2E report` - ); + core.warning(`Unable to find workflow job "${displayName}" for E2E report`); } } @@ -263,11 +251,7 @@ const ginkgoOutputSource = { * @returns {string|null} Path to the source file, or null when none exists. */ function findGinkgoSource(config, source) { - return findSingleMatchingFile( - config.reportsDir, - source.pattern(config.storageType), - source.label - ); + return findSingleMatchingFile(config.reportsDir, source.pattern(config.storageType), source.label); } /** @@ -301,21 +285,12 @@ function parseGinkgoFile(filePath, core, source) { source: source.okSource, }; } catch (error) { - core.warning( - `Unable to parse ${source.label} ${filePath}: ${error.message}` - ); + core.warning(`Unable to parse ${source.label} ${filePath}: ${error.message}`); return emptyParsedReport(source.invalidSource); } } -function buildReportPayload({ - config, - context, - fallbackWorkflowRunUrl, - branchName, - parsedReport, - sourcePath, -}) { +function buildReportPayload({ config, context, fallbackWorkflowRunUrl, branchName, parsedReport, sourcePath }) { const clusterStatus = buildClusterStatus(config.stageResults); const testStatus = buildTestStatus( config.stageResults["e2e-test"], @@ -323,16 +298,8 @@ function buildReportPayload({ clusterStatus, parsedReport.metrics ); - const reportSummary = buildReportSummary( - config.storageType, - clusterStatus, - testStatus - ); - const workflowRunUrl = getReportJobUrl( - reportSummary, - config.stageJobUrls, - fallbackWorkflowRunUrl - ); + const reportSummary = buildReportSummary(config.storageType, clusterStatus, testStatus); + const workflowRunUrl = getReportJobUrl(reportSummary, config.stageJobUrls, fallbackWorkflowRunUrl); return { schemaVersion: 1, @@ -360,11 +327,7 @@ function buildReportPayload({ }; } -function getReportJobUrl( - reportSummary, - stageJobUrls = {}, - fallbackWorkflowRunUrl -) { +function getReportJobUrl(reportSummary, stageJobUrls = {}, fallbackWorkflowRunUrl) { if (reportSummary.failedStage && stageJobUrls[reportSummary.failedStage]) { return stageJobUrls[reportSummary.failedStage]; } @@ -402,33 +365,22 @@ function setReportOutputs(report, reportFile, core) { * @throws {Error} If config is incomplete or the report file cannot be written. */ async function buildClusterReport({ core, context, github, config } = {}) { - const resolvedConfig = requireClusterReportConfig( - config || readClusterReportConfigFromEnv() - ); + const resolvedConfig = requireClusterReportConfig(config || readClusterReportConfigFromEnv()); if (!resolvedConfig.stageResults) { resolvedConfig.stageResults = readStageResultsFromEnv(); } if (!resolvedConfig.stageJobUrls && github) { - resolvedConfig.stageJobUrls = await readStageJobUrlsFromApi( - github, - context, - resolvedConfig, - core - ); + resolvedConfig.stageJobUrls = await readStageJobUrlsFromApi(github, context, resolvedConfig, core); } const fallbackWorkflowRunUrl = getWorkflowRunUrl(context); const branchName = getBranchName(context); const rawReportPath = findGinkgoSource(resolvedConfig, ginkgoJsonSource); - const outputPath = rawReportPath - ? null - : findGinkgoSource(resolvedConfig, ginkgoOutputSource); + const outputPath = rawReportPath ? null : findGinkgoSource(resolvedConfig, ginkgoOutputSource); const sourcePath = rawReportPath || outputPath; - const sourceDescriptor = rawReportPath - ? ginkgoJsonSource - : ginkgoOutputSource; + const sourceDescriptor = rawReportPath ? ginkgoJsonSource : ginkgoOutputSource; if (!rawReportPath) { core.warning( @@ -447,14 +399,9 @@ async function buildClusterReport({ core, context, github, config } = {}) { }); try { - fs.writeFileSync( - resolvedConfig.reportFile, - `${JSON.stringify(report, null, 2)}\n` - ); + fs.writeFileSync(resolvedConfig.reportFile, `${JSON.stringify(report, null, 2)}\n`); } catch (error) { - throw new Error( - `Unable to write cluster report file ${resolvedConfig.reportFile}: ${error.message}` - ); + throw new Error(`Unable to write cluster report file ${resolvedConfig.reportFile}: ${error.message}`); } setReportOutputs(report, resolvedConfig.reportFile, core); diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index 2af54e74a3..d88fcbb550 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -49,9 +49,7 @@ function createContext() { function createGithub(jobNames) { const jobs = jobNames.map((name, index) => ({ name, - html_url: `https://github.com/test/repo/actions/runs/12345/job/${ - index + 1 - }`, + html_url: `https://github.com/test/repo/actions/runs/12345/job/${index + 1}`, })); return { @@ -215,17 +213,13 @@ describe("cluster-report", () => { }); expect(report.cluster).toBe("nfs"); - expect(report.workflowRunUrl).toBe( - "https://github.com/test/repo/actions/runs/12345" - ); + expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); expect(report.branch).toBe("main"); expect(report.clusterStatus).toMatchObject({ status: "failure", stage: "configure-sdn", }); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe( - "nfs" - ); + expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe("nfs"); })); test("builds report from environment config", async () => @@ -256,13 +250,9 @@ describe("cluster-report", () => { }); expect(report.cluster).toBe("replicated"); - expect(report.workflowRunUrl).toBe( - "https://github.com/test/repo/actions/runs/12345" - ); + expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); expect(report.branch).toBe("main"); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe( - "replicated" - ); + expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe("replicated"); })); test("reads stage results from env vars", async () => @@ -290,9 +280,7 @@ describe("cluster-report", () => { stage: "configure-sdn", }); // No github — falls back to workflow run URL - expect(report.workflowRunUrl).toBe( - "https://github.com/test/repo/actions/runs/12345" - ); + expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); })); test("fetches job URLs from GitHub API", async () => @@ -327,9 +315,7 @@ describe("cluster-report", () => { stage: "configure-sdn", }); // github provided — URL points to the specific failed job - expect(report.workflowRunUrl).toBe( - "https://github.com/test/repo/actions/runs/12345/job/2" - ); + expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345/job/2"); })); test("works without github (no job URLs)", async () => @@ -354,17 +340,12 @@ describe("cluster-report", () => { expect(report.cluster).toBe("replicated"); // stageJobUrls is empty — falls back to workflow run URL - expect(report.workflowRunUrl).toBe( - "https://github.com/test/repo/actions/runs/12345" - ); + expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); })); test("marks Ginkgo JSON with failed specs as failed", async () => inTempDir(async (tempDir) => { - const rawReportPath = path.join( - tempDir, - "e2e_report_replicated_2026-04-15.json" - ); + const rawReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-15.json"); fs.writeFileSync( rawReportPath, createGinkgoReport({ @@ -437,10 +418,7 @@ describe("cluster-report", () => { total: 4, successRate: 33.33, }); - expect(report.failedTests).toEqual([ - "[It] Suite fails & burns [Slow]", - "[It] Other errors <loudly>", - ]); + expect(report.failedTests).toEqual(["[It] Suite fails & burns [Slow]", "[It] Other errors <loudly>"]); expect(report.failedTestDetails).toEqual([ { name: "[It] Suite fails & burns [Slow]", @@ -493,10 +471,7 @@ describe("cluster-report", () => { expect(core.setOutput).toHaveBeenCalledWith("report_kind", "tests"); expect(core.setOutput).toHaveBeenCalledWith("status", "failure"); expect(core.setOutput).toHaveBeenCalledWith("failed_stage", "e2e-test"); - expect(core.setOutput).toHaveBeenCalledWith( - "failed_stage_label", - "E2E TEST" - ); + expect(core.setOutput).toHaveBeenCalledWith("failed_stage_label", "E2E TEST"); expect(core.setOutput).toHaveBeenCalledWith( "workflow_run_url", "https://github.com/test/repo/actions/runs/12345" @@ -506,10 +481,7 @@ describe("cluster-report", () => { test("captures suite-level failures from Ginkgo JSON", async () => inTempDir(async (tempDir) => { - const rawReportPath = path.join( - tempDir, - "e2e_report_replicated_2026-04-15.json" - ); + const rawReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-15.json"); fs.writeFileSync( rawReportPath, createGinkgoReport({ @@ -519,8 +491,7 @@ describe("cluster-report", () => { leafNodeType: "SynchronizedBeforeSuite", state: "failed", failure: { - Message: - "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", + Message: "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", }, }), ], @@ -551,18 +522,14 @@ describe("cluster-report", () => { expect(report.failedTestDetails).toEqual([ { name: "[SynchronizedBeforeSuite]", - reason: - "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", + reason: "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", }, ]); })); test("uses Ginkgo output log when JSON report is missing", async () => inTempDir(async (tempDir) => { - const outputPath = path.join( - tempDir, - "e2e_output_replicated_2026-04-15.log" - ); + const outputPath = path.join(tempDir, "e2e_output_replicated_2026-04-15.log"); fs.writeFileSync( outputPath, [ @@ -607,10 +574,7 @@ describe("cluster-report", () => { test("parses real Ginkgo BeforeSuite failure stdout", async () => inTempDir(async (tempDir) => { - const outputPath = path.join( - tempDir, - "e2e_output_replicated_2026-05-14.log" - ); + const outputPath = path.join(tempDir, "e2e_output_replicated_2026-05-14.log"); fs.writeFileSync( outputPath, [ @@ -661,45 +625,30 @@ describe("cluster-report", () => { const detail = report.failedTestDetails[0]; expect(detail.name).toBe("[SynchronizedBeforeSuite]"); expect(detail.reason).toContain("Timed out after 300.001s."); - expect(detail.reason).toContain( - "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready" - ); + expect(detail.reason).toContain("object v12n-e2e-testdata-iso status.phase is Pending, expected Ready"); // The plain "[SynchronizedBeforeSuite]" header that follows the // "[FAILED] [307.964 seconds]" line must not leak into the reason. - expect(detail.reason.split("\n")[0]).not.toBe( - "[SynchronizedBeforeSuite]" - ); + expect(detail.reason.split("\n")[0]).not.toBe("[SynchronizedBeforeSuite]"); })); test("fails when multiple matching Ginkgo JSON reports exist", async () => inTempDir(async (tempDir) => { - const firstReportPath = path.join( - tempDir, - "nested", - "e2e_report_replicated_2026-04-15.json" - ); - const secondReportPath = path.join( - tempDir, - "e2e_report_replicated_2026-04-16.json" - ); + const firstReportPath = path.join(tempDir, "nested", "e2e_report_replicated_2026-04-15.json"); + const secondReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-16.json"); fs.mkdirSync(path.dirname(firstReportPath), { recursive: true }); fs.writeFileSync( firstReportPath, createGinkgoReport({ startedAt: "2026-04-15T09:30:44Z", - specs: [ - createSpecReport({ leafNodeText: "old pass", state: "passed" }), - ], + specs: [createSpecReport({ leafNodeText: "old pass", state: "passed" })], }) ); fs.writeFileSync( secondReportPath, createGinkgoReport({ startedAt: "2026-04-16T09:30:44Z", - specs: [ - createSpecReport({ leafNodeText: "latest pass", state: "passed" }), - ], + specs: [createSpecReport({ leafNodeText: "latest pass", state: "passed" })], }) ); @@ -721,10 +670,7 @@ describe("cluster-report", () => { test("falls back to missing-report status when raw Ginkgo JSON is invalid", async () => inTempDir(async (tempDir) => { - const rawReportPath = path.join( - tempDir, - "e2e_report_replicated_2026-04-15.json" - ); + const rawReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-15.json"); fs.writeFileSync(rawReportPath, "{not-valid-json"); const reportFile = path.join(tempDir, "report.json"); @@ -748,9 +694,7 @@ describe("cluster-report", () => { status: "missing", reason: "ginkgo-report-invalid", }); - expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining("Unable to parse Ginkgo JSON report") - ); + expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Unable to parse Ginkgo JSON report")); })); test("throws a descriptive error when writing the cluster report fails", async () => @@ -761,11 +705,9 @@ describe("cluster-report", () => { reportFile, }); - const writeSpy = jest - .spyOn(fs, "writeFileSync") - .mockImplementation(() => { - throw new Error("disk full"); - }); + const writeSpy = jest.spyOn(fs, "writeFileSync").mockImplementation(() => { + throw new Error("disk full"); + }); try { await expect( @@ -774,9 +716,7 @@ describe("cluster-report", () => { context: createContext(), config, }) - ).rejects.toThrow( - `Unable to write cluster report file ${reportFile}: disk full` - ); + ).rejects.toThrow(`Unable to write cluster report file ${reportFile}: disk full`); } finally { writeSpy.mockRestore(); } @@ -802,13 +742,9 @@ describe("cluster-report", () => { specs.push( createSpecReport({ - containerHierarchyTexts: [ - "VirtualMachineOperationRestore", - "restores a virtual machine from a snapshot", - ], + containerHierarchyTexts: ["VirtualMachineOperationRestore", "restores a virtual machine from a snapshot"], containerHierarchyLabels: [["Slow"], []], - leafNodeText: - "BestEffort restore mode; automatic restart approval mode; manual run policy", + leafNodeText: "BestEffort restore mode; automatic restart approval mode; manual run policy", state: "failed", }) ); diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index 6a1c836c72..00e9bc3633 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -17,14 +17,8 @@ const { REPORT_FILE_PATTERN } = require("./shared/report-model"); const { getClusterChartFiles } = require("./messenger/chart-files"); const { makeThreadedReportInLoop } = require("./messenger/loop-client"); const { readMessengerConfigFromEnv } = require("./messenger/config"); -const { - createMissingReport, - getReportClusterKey, -} = require("./messenger/model"); -const { - buildMainMessage, - buildThreadMessages, -} = require("./messenger/markdown"); +const { createMissingReport, getReportClusterKey } = require("./messenger/model"); +const { buildMainMessage, buildThreadMessages } = require("./messenger/markdown"); /** * @typedef {Object} MessengerReportCore @@ -79,9 +73,7 @@ function readReports(reportsDir, configuredClusters, core) { } // Configured clusters first, in declared order; missing ones get synthetic reports. - const result = configuredClusters.map( - (name) => reportsByCluster.get(name) ?? createMissingReport(name) - ); + const result = configuredClusters.map((name) => reportsByCluster.get(name) ?? createMissingReport(name)); // Any extra clusters not in the configured list, sorted alphabetically. const configuredSet = new Set(configuredClusters); @@ -91,9 +83,7 @@ function readReports(reportsDir, configuredClusters, core) { extras.push(report); } } - extras.sort((a, b) => - getReportClusterKey(a).localeCompare(getReportClusterKey(b)) - ); + extras.sort((a, b) => getReportClusterKey(a).localeCompare(getReportClusterKey(b))); return [...result, ...extras]; } @@ -107,11 +97,7 @@ function readReports(reportsDir, configuredClusters, core) { * threadMessages: Array<{message: string, files: Array<Record<string, any>>}> * }} Rendered markdown payloads. */ -async function buildMessengerMessages({ - reportsDir, - configuredClusters, - core, -}) { +async function buildMessengerMessages({ reportsDir, configuredClusters, core }) { const orderedReports = readReports(reportsDir, configuredClusters, core); const threadMessages = await buildThreadMessages(orderedReports, { getClusterChartFiles, @@ -143,17 +129,11 @@ async function renderMessengerReport({ core, reportsDir }) { core.info(message); core.setOutput("message", message); - core.setOutput( - "thread_messages", - JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message)) - ); + core.setOutput("thread_messages", JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message))); if (config.loop) { try { - await makeThreadedReportInLoop( - { message, threadMessages, loop: config.loop }, - core - ); + await makeThreadedReportInLoop({ message, threadMessages, loop: config.loop }, core); } catch (error) { core.warning(`Unable to deliver report to Loop API: ${error.message}`); if (config.loop.strictDelivery) { diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 950380ca98..e62875a21c 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -71,9 +71,7 @@ describe("messenger-report", () => { LOOP_API_BASE_URL: "https://loop.example.invalid", // LOOP_CHANNEL_ID and LOOP_TOKEN intentionally absent }) - ).toThrow( - "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" - ); + ).toThrow("LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required"); }); test("uses default configured clusters when env override is absent", () => { @@ -144,9 +142,7 @@ describe("messenger-report", () => { ); expect(result.message).not.toContain("⚠️ Errors"); expect(result.message).toContain("### Cluster failures"); - expect(result.message).toContain( - "- [nfs](https://example.invalid/nfs): CONFIGURE SDN" - ); + expect(result.message).toContain("- [nfs](https://example.invalid/nfs): CONFIGURE SDN"); expect(result.message).not.toContain("### Top slowest tests"); expect(result.message).not.toContain("### Failed tests"); expect(result.threadMessages).toEqual([ @@ -173,9 +169,7 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.message).toContain("### Missing reports"); - expect(result.message).toContain( - "- replicated: ⚠️ E2E REPORT ARTIFACT NOT FOUND" - ); + expect(result.message).toContain("- replicated: ⚠️ E2E REPORT ARTIFACT NOT FOUND"); expect(result.threadMessages).toEqual([]); })); @@ -231,12 +225,8 @@ describe("messenger-report", () => { files: [chartFile], }, ]); - expect(result.threadMessages[0].message).not.toContain( - "### Test durations" - ); - expect(result.threadMessages[0].message).not.toContain( - "Attached charts:" - ); + expect(result.threadMessages[0].message).not.toContain("### Test durations"); + expect(result.threadMessages[0].message).not.toContain("Attached charts:"); expect(core.setOutput).toHaveBeenCalledWith( "thread_messages", JSON.stringify([result.threadMessages[0].message]) @@ -245,9 +235,7 @@ describe("messenger-report", () => { test("warns and surfaces a placeholder when chart files are unavailable", async () => inTempDir(async (tempDir) => { - getClusterChartFiles.mockRejectedValue( - new Error("chart cli unavailable") - ); + getClusterChartFiles.mockRejectedValue(new Error("chart cli unavailable")); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ @@ -266,9 +254,7 @@ describe("messenger-report", () => { successRate: 100, }, failedTests: [], - specTimings: [ - { name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }, - ], + specTimings: [{ name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }], }) ); @@ -279,9 +265,7 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core }); expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining( - "Unable to prepare duration chart files for cluster replicated" - ) + expect.stringContaining("Unable to prepare duration chart files for cluster replicated") ); expect(result.threadMessages).toEqual([ { @@ -397,8 +381,7 @@ describe("messenger-report", () => { files: [], }, { - message: - "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + message: "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", files: [], }, ]); @@ -476,8 +459,7 @@ describe("messenger-report", () => { testStatus: { status: "not-run", reason: "cluster-stage-failure", - message: - "E2E tests were not run because cluster setup did not finish", + message: "E2E tests were not run because cluster setup did not finish", }, metrics: { passed: 0, @@ -496,9 +478,7 @@ describe("messenger-report", () => { expect(result.message).not.toContain("Branch: `main`"); expect(result.message).toContain("### Cluster failures"); - expect(result.message).toContain( - "- [replicated](https://example.invalid/replicated): ❌ CONFIGURE SDN FAILED" - ); + expect(result.message).toContain("- [replicated](https://example.invalid/replicated): ❌ CONFIGURE SDN FAILED"); expect(result.threadMessages).toEqual([]); })); @@ -520,8 +500,7 @@ describe("messenger-report", () => { testStatus: { status: "not-run", reason: "cluster-stage-failure", - message: - "E2E tests were not run because cluster setup did not finish", + message: "E2E tests were not run because cluster setup did not finish", }, metrics: { passed: 0, @@ -610,9 +589,7 @@ describe("messenger-report", () => { successRate: 83.33, }, failedTests: ["[It] fails"], - specTimings: [ - { name: "slow", group: "VM", state: "failed", runtimeMs: 90000 }, - ], + specTimings: [{ name: "slow", group: "VM", state: "failed", runtimeMs: 90000 }], }) ); @@ -724,9 +701,7 @@ describe("messenger-report", () => { // Empty body → no post id → thread replies cannot be sent → warning emitted. expect(global.fetch).toHaveBeenCalledTimes(1); - expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining("Loop API did not return a post id") - ); + expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Loop API did not return a post id")); // Report outputs are still set because the message was built before sending. expect(core.setOutput).toHaveBeenCalledWith("thread_messages", "[]"); })); @@ -771,12 +746,8 @@ describe("messenger-report", () => { // Non-JSON body → parse warning → no post id → delivery warning. expect(global.fetch).toHaveBeenCalledTimes(1); - expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining("Loop API returned a non-JSON response body") - ); - expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining("Loop API did not return a post id") - ); + expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Loop API returned a non-JSON response body")); + expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Loop API did not return a post id")); // Report outputs are still set because the message was built before sending. expect(core.setOutput).toHaveBeenCalledWith("thread_messages", "[]"); })); @@ -861,9 +832,7 @@ describe("messenger-report", () => { text: async () => "server exploded", }); - await expect( - renderMessengerReport({ core: createCore() }) - ).rejects.toThrow( + await expect(renderMessengerReport({ core: createCore() })).rejects.toThrow( "Loop API request failed with status 500: server exploded" ); })); diff --git a/.github/scripts/js/e2e/report/messenger/chart-files.js b/.github/scripts/js/e2e/report/messenger/chart-files.js index c4f6852f19..02ba14913c 100644 --- a/.github/scripts/js/e2e/report/messenger/chart-files.js +++ b/.github/scripts/js/e2e/report/messenger/chart-files.js @@ -44,5 +44,4 @@ function getClusterChartFiles(report) { module.exports = { getClusterChartFiles, - readChartManifest, }; diff --git a/.github/scripts/js/e2e/report/messenger/chart-files.test.js b/.github/scripts/js/e2e/report/messenger/chart-files.test.js index 36a57012fa..a68e366329 100644 --- a/.github/scripts/js/e2e/report/messenger/chart-files.test.js +++ b/.github/scripts/js/e2e/report/messenger/chart-files.test.js @@ -27,10 +27,7 @@ describe("chart-files", () => { test("loads chart files listed in the Python manifest", async () => withTempDir("chart-files", async (tempDir) => { - const chartPath = path.join( - tempDir, - "replicated-feature-duration-status.png" - ); + const chartPath = path.join(tempDir, "replicated-feature-duration-status.png"); const manifestPath = path.join(tempDir, "manifest.json"); fs.writeFileSync(chartPath, Buffer.from("png")); fs.writeFileSync( @@ -51,9 +48,7 @@ describe("chart-files", () => { const files = await getClusterChartFiles({ cluster: "replicated" }); - expect(files.map(({ name }) => name)).toEqual([ - "replicated-feature-duration-status.png", - ]); + expect(files.map(({ name }) => name)).toEqual(["replicated-feature-duration-status.png"]); expect(files[0].buffer).toEqual(Buffer.from("png")); expect(files[0].mimeType).toBe("image/png"); })); diff --git a/.github/scripts/js/e2e/report/messenger/config.js b/.github/scripts/js/e2e/report/messenger/config.js index cb31abde04..bc163d329c 100644 --- a/.github/scripts/js/e2e/report/messenger/config.js +++ b/.github/scripts/js/e2e/report/messenger/config.js @@ -81,9 +81,7 @@ function readLoopConfig(env = process.env) { return null; } if (!apiUrl || !channelId || !token) { - throw new Error( - "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" - ); + throw new Error("LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required"); } return { apiUrl, diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index f9d6bd2bcf..862724020f 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -46,9 +46,7 @@ function parseLoopApiPayload(responseText, core) { try { return JSON.parse(responseText); } catch (error) { - core.warning( - `Loop API returned a non-JSON response body: ${error.message}` - ); + core.warning(`Loop API returned a non-JSON response body: ${error.message}`); return {}; } } @@ -64,14 +62,7 @@ function parseLoopApiPayload(responseText, core) { * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<Record<string, any>>} Parsed Loop API response. */ -async function postToLoopApi( - loop, - message, - rootId, - core, - fileIds = [], - { fetch: fetchFn = globalThis.fetch } = {} -) { +async function postToLoopApi(loop, message, rootId, core, fileIds = [], { fetch: fetchFn = globalThis.fetch } = {}) { const body = { channel_id: loop.channelId, message, @@ -90,9 +81,7 @@ async function postToLoopApi( const responseText = await response.text(); if (!response.ok) { - throw new Error( - `Loop API request failed with status ${response.status}: ${responseText}` - ); + throw new Error(`Loop API request failed with status ${response.status}: ${responseText}`); } const payload = parseLoopApiPayload(responseText, core); @@ -115,14 +104,7 @@ function getFilesApiUrl(apiUrl) { * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<string>} Uploaded Loop file id. */ -async function uploadFileToLoop( - loop, - fileName, - buffer, - core, - mimeType, - { fetch: fetchFn = globalThis.fetch } = {} -) { +async function uploadFileToLoop(loop, fileName, buffer, core, mimeType, { fetch: fetchFn = globalThis.fetch } = {}) { const formData = new FormData(); formData.append("channel_id", loop.channelId); formData.append("files", new Blob([buffer], { type: mimeType }), fileName); @@ -137,21 +119,16 @@ async function uploadFileToLoop( const responseText = await response.text(); if (!response.ok) { - throw new Error( - `Loop file upload failed with status ${response.status}: ${responseText}` - ); + throw new Error(`Loop file upload failed with status ${response.status}: ${responseText}`); } const payload = parseLoopApiPayload(responseText, core); - const fileId = - payload.file_infos && payload.file_infos[0] && payload.file_infos[0].id; + const fileId = payload.file_infos && payload.file_infos[0] && payload.file_infos[0].id; if (!fileId) { throw new Error("Loop API did not return uploaded file id"); } - core.info( - `Loop API accepted file ${fileName} with status ${response.status}` - ); + core.info(`Loop API accepted file ${fileName} with status ${response.status}`); return fileId; } @@ -173,9 +150,7 @@ async function makeThreadedReportInLoop( }); if (!rootPost.id) { - throw new Error( - "Loop API did not return a post id; thread replies cannot be attached" - ); + throw new Error("Loop API did not return a post id; thread replies cannot be attached"); } for (const reply of threadMessages) { @@ -189,9 +164,7 @@ async function makeThreadedReportInLoop( }) ) ); - fileIds = results - .filter((result) => result.status === "fulfilled") - .map((result) => result.value); + fileIds = results.filter((result) => result.status === "fulfilled").map((result) => result.value); const failures = results.filter((result) => result.status === "rejected"); for (const failure of failures) { @@ -200,9 +173,7 @@ async function makeThreadedReportInLoop( core.warning(`Loop file upload failed for one attachment: ${details}`); } if (loop.strictFileUploads && failures.length > 0) { - throw new Error( - "Strict file uploads enabled; at least one attachment failed" - ); + throw new Error("Strict file uploads enabled; at least one attachment failed"); } } await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds, { diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 5e55dd37fc..4454b92de2 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -82,9 +82,7 @@ describe("loop-client", () => { text: async () => JSON.stringify({ id: "reply-post-id" }), }, ]; - global.fetch = jest - .fn() - .mockImplementation(() => Promise.resolve(responses.shift())); + global.fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); await makeThreadedReportInLoop( { @@ -153,9 +151,7 @@ describe("loop-client", () => { text: async () => JSON.stringify({ id: "reply-post-id" }), }, ]; - global.fetch = jest - .fn() - .mockImplementation(() => Promise.resolve(responses.shift())); + global.fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); await makeThreadedReportInLoop( { @@ -184,12 +180,8 @@ describe("loop-client", () => { expect(global.fetch).toHaveBeenCalledTimes(4); expect(global.fetch.mock.calls[0][0]).toBe(loop.apiUrl); - expect(global.fetch.mock.calls[1][0]).toBe( - "https://loop.example.invalid/api/v4/files" - ); - expect(global.fetch.mock.calls[2][0]).toBe( - "https://loop.example.invalid/api/v4/files" - ); + expect(global.fetch.mock.calls[1][0]).toBe("https://loop.example.invalid/api/v4/files"); + expect(global.fetch.mock.calls[2][0]).toBe("https://loop.example.invalid/api/v4/files"); expect(global.fetch.mock.calls[3][0]).toBe(loop.apiUrl); const replyBody = JSON.parse(global.fetch.mock.calls[3][1].body); @@ -198,9 +190,7 @@ describe("loop-client", () => { expect(replyBody.file_ids).toEqual(["file-one"]); expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining( - "Loop file upload failed for one attachment: Loop file upload failed with status 403" - ) + expect.stringContaining("Loop file upload failed for one attachment: Loop file upload failed with status 403") ); expect(core.warning).toHaveBeenCalledTimes(1); }); @@ -224,9 +214,7 @@ describe("loop-client", () => { text: async () => "permission denied", }, ]; - global.fetch = jest - .fn() - .mockImplementation(() => Promise.resolve(responses.shift())); + global.fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); await expect( makeThreadedReportInLoop( @@ -248,9 +236,7 @@ describe("loop-client", () => { }, createCore() ) - ).rejects.toThrow( - "Strict file uploads enabled; at least one attachment failed" - ); + ).rejects.toThrow("Strict file uploads enabled; at least one attachment failed"); expect(global.fetch).toHaveBeenCalledTimes(2); }); @@ -273,9 +259,7 @@ describe("loop-client", () => { text: async () => JSON.stringify({ id: "reply-post-id" }), }, ]; - const fetch = jest - .fn() - .mockImplementation(() => Promise.resolve(responses.shift())); + const fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); await makeThreadedReportInLoop( { diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index 17f96c805f..c0f39cb01b 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -38,9 +38,7 @@ function formatRate(value) { function formatClusterLink(report) { const clusterName = sanitizeCell(report.cluster || report.storageType); - return report.workflowRunUrl - ? `[${clusterName}](${report.workflowRunUrl})` - : clusterName; + return report.workflowRunUrl ? `[${clusterName}](${report.workflowRunUrl})` : clusterName; } function splitReportsBySection(orderedReports) { @@ -49,20 +47,14 @@ function splitReportsBySection(orderedReports) { return { testsReports: reports.filter(isTestResultReport), stageFailureReports: reports.filter(isClusterFailureReport), - missingReports: reports.filter( - (report) => isMissingReport(report) && !isClusterFailureReport(report) - ), + missingReports: reports.filter((report) => isMissingReport(report) && !isClusterFailureReport(report)), }; } function renderBranchLine(orderedReports) { - const branches = Array.from( - new Set(orderedReports.map((report) => report.branch).filter(Boolean)) - ); + const branches = Array.from(new Set(orderedReports.map((report) => report.branch).filter(Boolean))); - return branches.length === 1 && branches[0] !== "main" - ? [`Branch: \`${branches[0]}\``, ""] - : []; + return branches.length === 1 && branches[0] !== "main" ? [`Branch: \`${branches[0]}\``, ""] : []; } /** @@ -146,9 +138,7 @@ function renderTestResultsSection(testsReports) { return []; } - const hasGinkgoErrors = testsReports.some( - (report) => Number((report.metrics || {}).errors || 0) > 0 - ); + const hasGinkgoErrors = testsReports.some((report) => Number((report.metrics || {}).errors || 0) > 0); const columns = buildTestResultsColumns(hasGinkgoErrors); const rows = [ buildMarkdownRow(columns.map((column) => column.header)), @@ -157,9 +147,7 @@ function renderTestResultsSection(testsReports) { for (const report of testsReports) { const metrics = report.metrics || {}; - rows.push( - buildMarkdownRow(columns.map((column) => column.value(report, metrics))) - ); + rows.push(buildMarkdownRow(columns.map((column) => column.value(report, metrics)))); } return ["### Test results", "", ...rows, ""]; @@ -182,10 +170,7 @@ function renderBulletSection(title, reports, getMessage) { return []; } - const bullets = reports.map( - (report) => - `- ${formatClusterLink(report)}: ${sanitizeListItem(getMessage(report))}` - ); + const bullets = reports.map((report) => `- ${formatClusterLink(report)}: ${sanitizeListItem(getMessage(report))}`); return [`### ${title}`, "", ...bullets, ""]; } @@ -216,22 +201,13 @@ function getMissingReportMessage(report) { */ function buildMainMessage(orderedReports) { const reportDate = getReportDate(orderedReports); - const { testsReports, stageFailureReports, missingReports } = - splitReportsBySection(orderedReports); + const { testsReports, stageFailureReports, missingReports } = splitReportsBySection(orderedReports); const lines = [ `## :dvp: DVP | E2E on nested clusters | ${reportDate}`, "", ...renderBranchLine(orderedReports), - ...renderBulletSection( - "Cluster failures", - stageFailureReports, - getClusterFailureMessage - ), - ...renderBulletSection( - "Missing reports", - missingReports, - getMissingReportMessage - ), + ...renderBulletSection("Cluster failures", stageFailureReports, getClusterFailureMessage), + ...renderBulletSection("Missing reports", missingReports, getMissingReportMessage), ...renderTestResultsSection(testsReports), ]; @@ -268,10 +244,7 @@ function getFailedTestGroupName(testName) { } function getFailedTestEntries(report) { - if ( - Array.isArray(report.failedTestDetails) && - report.failedTestDetails.length > 0 - ) { + if (Array.isArray(report.failedTestDetails) && report.failedTestDetails.length > 0) { return report.failedTestDetails.map((test) => ({ name: test.name, reason: test.reason, @@ -321,9 +294,7 @@ function renderFailedTestsThreadMessage(report) { lines.push("| Tests | Reason |"); lines.push("|---|---|"); for (const group of failedGroups) { - lines.push( - `| ${sanitizeCell(group.name)} | ${sanitizeCell(group.reason)} |` - ); + lines.push(`| ${sanitizeCell(group.name)} | ${sanitizeCell(group.reason)} |`); } } else { lines.push( @@ -355,13 +326,8 @@ function buildChartCaption(_files, chartsUnavailable) { * }} [options] * @returns {Promise<Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>>} Markdown thread payloads. */ -async function buildThreadMessages( - orderedReports, - { getClusterChartFiles, core } = {} -) { - const testsReports = orderedReports.filter((report) => - isTestResultReport(report) - ); +async function buildThreadMessages(orderedReports, { getClusterChartFiles, core } = {}) { + const testsReports = orderedReports.filter((report) => isTestResultReport(report)); const threadMessages = []; let renderedFailedTestsHeading = false; @@ -377,9 +343,9 @@ async function buildThreadMessages( chartsUnavailable = true; if (core && typeof core.warning === "function") { core.warning( - `Unable to prepare duration chart files for cluster ${ - getReportClusterKey(report) || "unknown" - }: ${error.message}` + `Unable to prepare duration chart files for cluster ${getReportClusterKey(report) || "unknown"}: ${ + error.message + }` ); } } @@ -392,9 +358,7 @@ async function buildThreadMessages( if (hasFailedTests(report)) { const clusterMessage = renderFailedTestsThreadMessage(report); messageParts.push( - renderedFailedTestsHeading - ? clusterMessage - : ["### Failed tests", clusterMessage].join("\n\n") + renderedFailedTestsHeading ? clusterMessage : ["### Failed tests", clusterMessage].join("\n\n") ); renderedFailedTestsHeading = true; } else { diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index 40b2f3c1aa..6241c65cbb 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -73,19 +73,12 @@ function formatSpecName(specReport) { .filter(Boolean); const leafText = String(specReport.LeafNodeText || "").trim(); const labels = [ - ...new Set([ - ...flattenLabels(specReport.ContainerHierarchyLabels), - ...flattenLabels(specReport.LeafNodeLabels), - ]), + ...new Set([...flattenLabels(specReport.ContainerHierarchyLabels), ...flattenLabels(specReport.LeafNodeLabels)]), ]; const labelSuffix = labels.map((label) => `[${label}]`).join(" "); const body = [...hierarchyParts, leafText].filter(Boolean).join(" "); - return [`[${nodeType}]`, body, labelSuffix] - .filter(Boolean) - .join(" ") - .replace(/\s+/g, " ") - .trim(); + return [`[${nodeType}]`, body, labelSuffix].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(); } function runtimeMs(value) { @@ -122,25 +115,18 @@ function getMetricKeyForState(state) { function formatFailureReason(specReport) { const failure = (specReport && specReport.Failure) || {}; - return ( - String(failure.Message || failure.ForwardedPanic || "").trim() || - String(specReport.State || "failed").trim() - ); + return String(failure.Message || failure.ForwardedPanic || "").trim() || String(specReport.State || "failed").trim(); } const failureStates = new Set(["failed", "errors"]); function isSuiteNodeFailure(specReport) { - const leafNodeType = String( - (specReport && specReport.LeafNodeType) || "" - ).trim(); + const leafNodeType = String((specReport && specReport.LeafNodeType) || "").trim(); if (!leafNodeType || leafNodeType === "It") { return false; } - return failureStates.has( - getMetricKeyForState(specReport && specReport.State) - ); + return failureStates.has(getMetricKeyForState(specReport && specReport.State)); } function buildFailureDetail(specReport) { @@ -175,8 +161,7 @@ function parseGinkgoReport(jsonContent) { const failedTests = []; const failedTestDetails = []; const specTimings = []; - const startedAt = - suites.find((suite) => suite && suite.StartTime)?.StartTime || null; + const startedAt = suites.find((suite) => suite && suite.StartTime)?.StartTime || null; let suiteTotalMs = 0; for (const suite of suites) { @@ -212,10 +197,7 @@ function parseGinkgoReport(jsonContent) { group: hierarchyParts[0] || "Top-level Its", state: metricKey, runtimeMs: runtimeMs(specReport.RunTime), - labels: flattenLabels([ - ...toArray(specReport.ContainerHierarchyLabels), - ...toArray(specReport.LeafNodeLabels), - ]), + labels: flattenLabels([...toArray(specReport.ContainerHierarchyLabels), ...toArray(specReport.LeafNodeLabels)]), }); if (failureStates.has(metricKey)) { @@ -229,21 +211,13 @@ function parseGinkgoReport(jsonContent) { } const completedSpecs = metrics.passed + metrics.failed + metrics.errors; - metrics.successRate = - completedSpecs > 0 - ? Number(((metrics.passed / completedSpecs) * 100).toFixed(2)) - : 0; + metrics.successRate = completedSpecs > 0 ? Number(((metrics.passed / completedSpecs) * 100).toFixed(2)) : 0; return { metrics, failedTests: Array.from(new Set(failedTests)), failedTestDetails: Array.from( - new Map( - failedTestDetails.map((test) => [ - `${test.name}\u0000${test.reason}`, - test, - ]) - ).values() + new Map(failedTestDetails.map((test) => [`${test.name}\u0000${test.reason}`, test])).values() ), specTimings, suiteTotalMs, @@ -251,12 +225,7 @@ function parseGinkgoReport(jsonContent) { }; } -const suiteNodeTypes = [ - "SynchronizedBeforeSuite", - "BeforeSuite", - "SynchronizedAfterSuite", - "AfterSuite", -]; +const suiteNodeTypes = ["SynchronizedBeforeSuite", "BeforeSuite", "SynchronizedAfterSuite", "AfterSuite"]; // Match Ginkgo failure markers for suite-level nodes in two forms: // 1. "[<SuiteNode>] [FAILED]" — main failure line in the stdout body. @@ -303,11 +272,7 @@ function isReasonStopLine(line) { } function isReasonNoiseLine(line, suiteHeader, failedMarker) { - return ( - line === suiteHeader || - line.startsWith(failedMarker) || - line.startsWith("/") - ); + return line === suiteHeader || line.startsWith(failedMarker) || line.startsWith("/"); } /** diff --git a/.github/scripts/js/e2e/report/shared/report-model.js b/.github/scripts/js/e2e/report/shared/report-model.js index 19fedf5cf2..4d204d542e 100644 --- a/.github/scripts/js/e2e/report/shared/report-model.js +++ b/.github/scripts/js/e2e/report/shared/report-model.js @@ -63,12 +63,7 @@ const stageMessage = { "artifact-missing": "TEST REPORTS NOT FOUND", }; -const clusterSetupStages = [ - "bootstrap", - "configure-sdn", - "storage-setup", - "virtualization-setup", -]; +const clusterSetupStages = ["bootstrap", "configure-sdn", "storage-setup", "virtualization-setup"]; function zeroMetrics() { return { @@ -94,8 +89,7 @@ const statusMessageTemplates = { }; function buildStatusMessage(status, stageLabel) { - const template = - statusMessageTemplates[status] || statusMessageTemplates.failure; + const template = statusMessageTemplates[status] || statusMessageTemplates.failure; return template.replace("%s", stageLabel); } @@ -135,12 +129,7 @@ function buildClusterStatus(stageResults) { }; } -function buildTestStatus( - testResult, - reportSource, - clusterStatus, - metrics = {} -) { +function buildTestStatus(testResult, reportSource, clusterStatus, metrics = {}) { const stageLabel = stageMessage["e2e-test"]; if (clusterStatus.status !== "success") { @@ -154,20 +143,13 @@ function buildTestStatus( const normalizedResult = normalizeJobResult(testResult); if (reportSource === "ginkgo-json" || reportSource === "ginkgo-output") { - const hasReportedFailures = - Number(metrics.failed || 0) > 0 || Number(metrics.errors || 0) > 0; - const status = - normalizedResult === "success" && hasReportedFailures - ? "failure" - : normalizedResult; + const hasReportedFailures = Number(metrics.failed || 0) > 0 || Number(metrics.errors || 0) > 0; + const status = normalizedResult === "success" && hasReportedFailures ? "failure" : normalizedResult; return { status, reason: status === "success" ? "" : "ginkgo-failed", - message: - status === "success" - ? "✅ E2E TESTS PASSED" - : buildStatusMessage(status, stageLabel), + message: status === "success" ? "✅ E2E TESTS PASSED" : buildStatusMessage(status, stageLabel), }; } @@ -236,8 +218,7 @@ function buildReportSummary(storageType, clusterStatus, testStatus) { return { failedStage: testStatus.status === "success" ? "success" : "e2e-test", - failedStageLabel: - testStatus.status === "success" ? "SUCCESS" : stageMessage["e2e-test"], + failedStageLabel: testStatus.status === "success" ? "SUCCESS" : stageMessage["e2e-test"], failedJobName: `E2E test (${storageType})`, reportKind: "tests", status: testStatus.status, @@ -257,10 +238,7 @@ function isMissingReport(report) { function isClusterFailureReport(report) { if (report.clusterStatus) { - return ( - report.clusterStatus.status !== "success" && - report.clusterStatus.status !== "missing" - ); + return report.clusterStatus.status !== "success" && report.clusterStatus.status !== "missing"; } return report.reportKind !== "tests" && !isMissingReport(report); @@ -272,10 +250,7 @@ function isTestResultReport(report) { } if (report.testStatus) { - return ( - report.testStatus.status !== "not-run" && - report.testStatus.status !== "missing" - ); + return report.testStatus.status !== "not-run" && report.testStatus.status !== "missing"; } return report.reportKind === "tests"; diff --git a/.github/scripts/python/e2e_report/charts.py b/.github/scripts/python/e2e_report/charts.py index 2843a93dd0..9d64e83edc 100644 --- a/.github/scripts/python/e2e_report/charts.py +++ b/.github/scripts/python/e2e_report/charts.py @@ -10,15 +10,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""Render E2E report charts for CI and local debugging. + +Subcommands: +- messenger-all renders feature-duration-status PNGs and writes a manifest. +- slowest renders one slowest-specs PNG for a selected Describe. +- top renders slowest-specs PNGs for the top-N slowest Describes. + +The manifest schema is {"clusters": {"cluster": [{"name", "path", "mimeType"}]}}. +""" + from __future__ import annotations import argparse import json import math import re +import sys from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Literal import matplotlib @@ -28,8 +39,10 @@ from matplotlib.ticker import FuncFormatter, MultipleLocator # noqa: E402 -STATUSES = ("passed", "failed", "errors", "skipped") -STATUS_COLORS = { +StatusName = Literal["passed", "failed", "errors", "skipped"] + +STATUSES: tuple[StatusName, ...] = ("passed", "failed", "errors", "skipped") +STATUS_COLORS: dict[StatusName, str] = { "passed": "#00b83f", "failed": "#ff3333", "errors": "#d9a300", @@ -40,6 +53,8 @@ "medium": "#3fb950", "slow": "#238636", } +SLOW_THRESHOLD_MS = 300_000 +MEDIUM_THRESHOLD_MS = 60_000 DEFAULT_TOP_N = 15 REPORT_FILE_PATTERN = re.compile(r"^e2e_report_.*\.json$") @@ -57,11 +72,13 @@ def full_name(self) -> str: def sanitize_filename_part(value: Any) -> str: + """Return a path-safe file name part for user-controlled chart labels.""" safe = re.sub(r"[^a-zA-Z0-9_-]+", "_", str(value or "cluster")) return safe or "cluster" def to_seconds(ms: float) -> float: + """Convert milliseconds to rounded seconds for chart labels.""" return round(float(ms) / 1000, 2) @@ -110,29 +127,41 @@ def aggregate(spec_timings: list[dict[str, Any]] | None) -> tuple[list[Timing], def duration_bucket(timing: Timing) -> str: - if timing.runtime_ms > 300_000: + if timing.runtime_ms > SLOW_THRESHOLD_MS: return "slow" - if timing.runtime_ms >= 60_000: + if timing.runtime_ms >= MEDIUM_THRESHOLD_MS: return "medium" return "fast" def format_seconds(seconds: float) -> str: + """Format seconds compactly for bar labels.""" return f"{seconds:.0f}s" if seconds >= 10 else f"{seconds:.1f}s" def format_axis_seconds(value: float, _position: int) -> str: + """Format axis ticks as integer seconds.""" return f"{int(value):,}" def next_tick(value: float, step: int) -> int: + """Round value up to the next axis tick.""" if value <= 0: return step return int(math.ceil(value / step) * step) -def render_feature_duration_status(report: dict[str, Any], output_dir: Path) -> dict[str, str]: - _, by_group = aggregate(report.get("specTimings") or []) +def _cluster_key(report: dict[str, Any]) -> str: + return str(report.get("storageType") or report.get("cluster") or "").strip() + + +def _compute_feature_segments( + by_group: dict[str, dict[str, Any]], +) -> tuple[ + list[str], + list[tuple[StatusName, list[float], list[float]]], + list[float], +]: entries = sorted( by_group.items(), key=lambda item: ( @@ -142,28 +171,18 @@ def render_feature_duration_status(report: dict[str, Any], output_dir: Path) -> ), ) labels = [name for name, _ in entries] - height = max(6.4, 0.75 + len(labels) * 0.285) - fig, ax = plt.subplots(figsize=(10.24, height), dpi=100) left = [0.0] * len(entries) + segments: list[tuple[StatusName, list[float], list[float]]] = [] for status in STATUSES: values = [to_seconds(group["status_durations"][status]) for _, group in entries] - ax.barh(labels, values, left=left, label=status, color=STATUS_COLORS[status], height=0.72) - for row, (offset, value) in enumerate(zip(left, values)): - if value <= 0: - continue - ax.text( - offset + value / 2, - row, - format_seconds(value), - ha="center", - va="center", - fontsize=6, - color="#333333", - ) + segments.append((status, values, left.copy())) left = [current + value for current, value in zip(left, values)] - x_limit = next_tick(max(left, default=0), 60) + return labels, segments, left + + +def _apply_axes_style(ax: Any, labels: list[str], x_limit: int) -> None: ax.set_xlim(0, x_limit) ax.set_title("Overall durations for Describes", fontsize=8, pad=12) ax.set_xlabel("Duration, seconds", fontsize=7) @@ -190,7 +209,32 @@ def render_feature_duration_status(report: dict[str, Any], output_dir: Path) -> spine.set_color("#dddddd") spine.set_linewidth(0.5) - cluster_name = sanitize_filename_part(report.get("cluster") or report.get("storageType") or "cluster") + +def render_feature_duration_status(report: dict[str, Any], output_dir: Path) -> dict[str, str]: + _, by_group = aggregate(report.get("specTimings") or []) + labels, segments, left = _compute_feature_segments(by_group) + height = max(6.4, 0.75 + len(labels) * 0.285) + fig, ax = plt.subplots(figsize=(10.24, height), dpi=100) + + for status, values, segment_left in segments: + ax.barh(labels, values, left=segment_left, label=status, color=STATUS_COLORS[status], height=0.72) + for row, (offset, value) in enumerate(zip(segment_left, values)): + if value <= 0: + continue + ax.text( + offset + value / 2, + row, + format_seconds(value), + ha="center", + va="center", + fontsize=6, + color="#333333", + ) + + x_limit = next_tick(max(left, default=0), 60) + _apply_axes_style(ax, labels, x_limit) + + cluster_name = sanitize_filename_part(_cluster_key(report) or "cluster") return save_figure(fig, output_dir / f"{cluster_name}-feature-duration-status.png") @@ -212,7 +256,8 @@ def render_slowest_specs( ] line_widths = [2 if timing.state in {"failed", "errors"} else 0 for timing in top] - fig, ax = plt.subplots(figsize=(20.48, 7.2), dpi=100) + height = max(4.0, 0.6 + len(labels) * 0.45) + fig, ax = plt.subplots(figsize=(20.48, height), dpi=100) ax.barh(labels, values, color=colors, edgecolor=edge_colors, linewidth=line_widths) ax.set_title("Top slowest successful specs and failed specs (It/Entry)") ax.set_xlabel("Duration, seconds") @@ -242,18 +287,22 @@ def save_figure(fig: plt.Figure, target_path: Path) -> dict[str, str]: } -def runtime_ms(value: Any) -> int: +def runtime_ns_to_ms(value: Any) -> int: + """Ginkgo SpecReport.RunTime is in nanoseconds; round to milliseconds.""" try: runtime = float(value or 0) except (TypeError, ValueError): return 0 - return round(runtime / 1_000_000) if math.isfinite(runtime) else 0 + if not math.isfinite(runtime) or runtime < 0: + return 0 + return round(runtime / 1_000_000) -def metric_key_for_state(state: Any) -> str: +def metric_key_for_state(state: Any) -> StatusName: normalized = str(state or "").strip().lower() if normalized in {"passed", "failed"}: return normalized + # Keep the same pending-to-skipped collapse as shared/report-model.js. if normalized in {"skipped", "pending"}: return "skipped" return "errors" @@ -279,7 +328,7 @@ def parse_ginkgo_report(payload: Any) -> list[dict[str, Any]]: "name": str(spec_report.get("LeafNodeText") or "").strip(), "group": hierarchy[0] if hierarchy else "Top-level Its", "state": metric_key_for_state(spec_report.get("State")), - "runtimeMs": runtime_ms(spec_report.get("RunTime")), + "runtimeMs": runtime_ns_to_ms(spec_report.get("RunTime")), } ) @@ -288,7 +337,7 @@ def parse_ginkgo_report(payload: Any) -> list[dict[str, Any]]: def read_report(json_path: str | Path) -> dict[str, Any]: path = Path(json_path) - payload = json.loads(path.read_text()) + payload = json.loads(path.read_text(encoding="utf-8")) if isinstance(payload, dict) and isinstance(payload.get("specTimings"), list): return payload return {"specTimings": parse_ginkgo_report(payload)} @@ -318,7 +367,7 @@ def derive_storage_type(report_path: str | Path, fallback_storage: str | None = def report_cluster_key(report: dict[str, Any]) -> str: - return str(report.get("cluster") or report.get("storageType") or "").strip() + return _cluster_key(report) def top_describes(spec_timings: list[dict[str, Any]] | None, top_n: int = 5) -> list[str]: @@ -360,20 +409,19 @@ def render_messenger_charts( manifest = {"clusters": clusters} target_path = Path(manifest_path) target_path.parent.mkdir(parents=True, exist_ok=True) - target_path.write_text(json.dumps(manifest, indent=2, sort_keys=True)) + target_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") return manifest -def render_slowest_for_describe( - json_path: str | Path, +def _render_slowest_for_report( + report: dict[str, Any], + storage_name: str, describe: str, - out_dir: str | Path = "tmp", - storage: str | None = None, + output_dir: Path, ) -> dict[str, str]: if not describe: raise ValueError("--describe is required") - report = read_report(json_path) spec_timings = report.get("specTimings") or [] filtered_timings = [ timing for timing in spec_timings if str(timing.get("group") or "") == describe @@ -387,12 +435,22 @@ def render_slowest_for_describe( ] raise ValueError("\n".join(lines)) + return render_slowest_specs(filtered_timings, storage_name, describe, output_dir) + + +def render_slowest_for_describe( + json_path: str | Path, + describe: str, + out_dir: str | Path = "tmp", + storage: str | None = None, +) -> dict[str, str]: + report = read_report(json_path) storage_name = ( storage - or str(report.get("storageType") or report.get("cluster") or "").strip() + or _cluster_key(report) or derive_storage_type(json_path) ) - return render_slowest_specs(filtered_timings, storage_name, describe, Path(out_dir) / "charts") + return _render_slowest_for_report(report, storage_name, describe, Path(out_dir)) def list_report_files(reports_dir: str | Path) -> list[Path]: @@ -412,52 +470,92 @@ def render_top_describes( report = read_report(report_file) storage_name = report_cluster_key(report) or derive_storage_type(report_file) for describe in top_describes(report.get("specTimings") or [], top_n): - rendered_files.append( - render_slowest_for_describe( - report_file, - describe, - out_dir=out_dir, - storage=storage_name, + try: + rendered_files.append( + _render_slowest_for_report( + report, + storage_name, + describe, + Path(out_dir), + ) + ) + except Exception as error: + print( + f'Failed to render Describe "{describe}" from "{report_file}": {error}', + file=sys.stderr, ) - ) return rendered_files -def print_json(files: list[dict[str, str]]) -> None: - print(json.dumps(files, separators=(",", ":"))) +def collect_messenger_debug(reports_dir: str | Path) -> dict[str, dict[str, Any]]: + clusters: dict[str, dict[str, Any]] = {} + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + cluster_name = report_cluster_key(report) or derive_storage_type(report_file) + _, by_group = aggregate(report.get("specTimings") or []) + clusters[cluster_name] = by_group + return {"clusters": clusters} + + +def collect_top_debug(reports_dir: str | Path, top_n: int) -> dict[str, dict[str, Any]]: + clusters: dict[str, dict[str, Any]] = {} + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + cluster_name = report_cluster_key(report) or derive_storage_type(report_file) + clusters[cluster_name] = { + "topDescribes": top_describes(report.get("specTimings") or [], top_n), + } + return {"clusters": clusters} + + +def write_debug_json(payload: Any, debug_json_path: str | Path) -> None: + target_path = Path(debug_json_path) + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") -def main() -> None: +def main(argv: list[str] | None = None) -> None: parser = argparse.ArgumentParser(description="Render E2E report charts") subparsers = parser.add_subparsers(dest="command", required=True) - messenger = subparsers.add_parser("messenger", help="Render charts for one messenger report") - messenger.add_argument("--json", required=True) - messenger.add_argument("--out-dir", default="tmp") - messenger_all = subparsers.add_parser("messenger-all", help="Render messenger charts and write a manifest") messenger_all.add_argument("--reports-dir", default="downloaded-artifacts") - messenger_all.add_argument("--out-dir", default="tmp/messenger-charts") + messenger_all.add_argument( + "--out-dir", + default="tmp/messenger-charts", + help="Literal directory for rendered PNG files.", + ) messenger_all.add_argument("--manifest", default="tmp/messenger-charts/manifest.json") + messenger_all.add_argument("--debug-json", help="Write aggregate debug data to this JSON path.") slowest = subparsers.add_parser("slowest", help="Render slowest specs for one Describe") slowest.add_argument("--json", required=True) slowest.add_argument("--describe", required=True) - slowest.add_argument("--out-dir", default="tmp") + slowest.add_argument( + "--out-dir", + default="tmp", + help="Literal directory for the rendered PNG file.", + ) slowest.add_argument("--storage") top = subparsers.add_parser("top", help="Render slowest specs for top-N Describes") top.add_argument("--reports-dir", default="downloaded-artifacts") - top.add_argument("--out-dir", default="tmp") + top.add_argument( + "--out-dir", + default="tmp", + help="Literal directory for rendered PNG files.", + ) top.add_argument("--top-n", type=int, default=5) + top.add_argument("--debug-json", help="Write aggregate debug data to this JSON path.") - args = parser.parse_args() + args = parser.parse_args(argv) - if args.command == "messenger": - print_json(render_cluster_charts(read_report(args.json), Path(args.out_dir))) - return if args.command == "messenger-all": + if not list_report_files(args.reports_dir): + raise SystemExit(f'No report files found in "{args.reports_dir}".') render_messenger_charts(args.reports_dir, args.out_dir, args.manifest) + if args.debug_json: + write_debug_json(collect_messenger_debug(args.reports_dir), args.debug_json) print(args.manifest) return if args.command == "slowest": @@ -465,7 +563,11 @@ def main() -> None: print(file["path"]) return if args.command == "top": + if not list_report_files(args.reports_dir): + raise SystemExit(f'No report files found in "{args.reports_dir}".') files = render_top_describes(args.reports_dir, args.out_dir, args.top_n) + if args.debug_json: + write_debug_json(collect_top_debug(args.reports_dir, args.top_n), args.debug_json) for file in files: print(file["path"]) diff --git a/.github/scripts/python/e2e_report/charts_test.py b/.github/scripts/python/e2e_report/charts_test.py index a4566545f2..da058d97d6 100644 --- a/.github/scripts/python/e2e_report/charts_test.py +++ b/.github/scripts/python/e2e_report/charts_test.py @@ -12,6 +12,8 @@ from __future__ import annotations +import contextlib +import io import json import tempfile import unittest @@ -20,6 +22,10 @@ from e2e_report import charts +def write_json(path: Path, payload: object) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + class ChartsTest(unittest.TestCase): def test_aggregate_normalizes_timings(self) -> None: timings, by_group = charts.aggregate( @@ -50,37 +56,35 @@ def test_top_describes_uses_duration_desc_then_name(self) -> None: def test_render_slowest_for_describe_writes_expected_artifact(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: report_path = Path(temp_dir) / "e2e_report_nfs_2026-05-15.json" - report_path.write_text( - json.dumps( - { - "storageType": "nfs", - "specTimings": [ - {"name": "fast", "group": "VM", "state": "passed", "runtimeMs": 10_000}, - {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, - {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, - ], - } - ) + write_json( + report_path, + { + "storageType": "nfs", + "specTimings": [ + {"name": "fast", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, + ], + }, ) rendered = charts.render_slowest_for_describe(report_path, "VM", out_dir=temp_dir) self.assertEqual(rendered["name"], "nfs-VM-slowest-specs.png") - self.assertEqual(Path(rendered["path"]), Path(temp_dir) / "charts" / rendered["name"]) + self.assertEqual(Path(rendered["path"]), Path(temp_dir) / rendered["name"]) self.assertTrue(Path(rendered["path"]).is_file()) def test_render_slowest_for_describe_lists_available_describes(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: report_path = Path(temp_dir) / "e2e_report_nfs_2026-05-15.json" - report_path.write_text( - json.dumps( - { - "specTimings": [ - {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, - {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 10_000}, - ], - } - ) + write_json( + report_path, + { + "specTimings": [ + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + ], + }, ) with self.assertRaisesRegex(ValueError, "Available Describes:\\n- Disk\\n- VM"): @@ -101,21 +105,34 @@ def test_render_cluster_charts_writes_messenger_chart_only(self) -> None: self.assertEqual([file["name"] for file in files], ["replicated-feature-duration-status.png"]) self.assertTrue(Path(files[0]["path"]).is_file()) + def test_render_feature_duration_status_writes_non_empty_png(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + file = charts.render_feature_duration_status( + { + "storageType": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + Path(temp_dir), + ) + + self.assertGreater(Path(file["path"]).stat().st_size, 1000) + def test_render_messenger_charts_writes_manifest(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: reports_dir = Path(temp_dir) / "reports" out_dir = Path(temp_dir) / "charts" manifest_path = out_dir / "manifest.json" reports_dir.mkdir() - (reports_dir / "e2e_report_replicated.json").write_text( - json.dumps( - { - "storageType": "replicated", - "specTimings": [ - {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, - ], - } - ) + write_json( + reports_dir / "e2e_report_replicated.json", + { + "storageType": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, ) manifest = charts.render_messenger_charts(reports_dir, out_dir, manifest_path) @@ -124,7 +141,219 @@ def test_render_messenger_charts_writes_manifest(self) -> None: [file["name"] for file in manifest["clusters"]["replicated"]], ["replicated-feature-duration-status.png"], ) - self.assertEqual(json.loads(manifest_path.read_text()), manifest) + self.assertEqual(json.loads(manifest_path.read_text(encoding="utf-8")), manifest) + + def test_report_cluster_key_prefers_storage_type(self) -> None: + report = {"storageType": "storage-first", "cluster": "cluster-second"} + + self.assertEqual(charts.report_cluster_key(report), "storage-first") + + def test_render_messenger_charts_uses_storage_type_key(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + reports_dir = Path(temp_dir) / "reports" + out_dir = Path(temp_dir) / "charts" + manifest_path = out_dir / "manifest.json" + reports_dir.mkdir() + write_json( + reports_dir / "e2e_report_replicated.json", + { + "storageType": "storage-first", + "cluster": "cluster-second", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + ) + + manifest = charts.render_messenger_charts(reports_dir, out_dir, manifest_path) + + self.assertIn("storage-first", manifest["clusters"]) + self.assertNotIn("cluster-second", manifest["clusters"]) + + +class RuntimeNsToMsTest(unittest.TestCase): + def test_runtime_ns_to_ms(self) -> None: + cases = [ + (1_500_000.0, 2), + (1_000_000, 1), + (-1, 0), + (float("nan"), 0), + (None, 0), + ("bad", 0), + ] + + for value, expected in cases: + with self.subTest(value=value): + self.assertEqual(charts.runtime_ns_to_ms(value), expected) + + +class ParseGinkgoReportTest(unittest.TestCase): + def test_parse_ginkgo_report_filters_non_it_and_uses_default_group(self) -> None: + payload = { + "SpecReports": [ + { + "LeafNodeType": "It", + "LeafNodeText": "creates vm", + "ContainerHierarchyTexts": ["VM", "Nested"], + "State": "passed", + "RunTime": 1_500_000, + }, + { + "LeafNodeType": "BeforeSuite", + "LeafNodeText": "setup", + "ContainerHierarchyTexts": ["Suite"], + "State": "failed", + "RunTime": 10_000_000, + }, + { + "LeafNodeType": "It", + "LeafNodeText": "top level", + "ContainerHierarchyTexts": [], + "State": "pending", + "RunTime": 2_000_000, + }, + ], + } + + self.assertEqual( + charts.parse_ginkgo_report(payload), + [ + {"name": "creates vm", "group": "VM", "state": "passed", "runtimeMs": 2}, + {"name": "top level", "group": "Top-level Its", "state": "skipped", "runtimeMs": 2}, + ], + ) + + +class DeriveStorageTypeTest(unittest.TestCase): + def test_derive_storage_type(self) -> None: + self.assertEqual( + charts.derive_storage_type("e2e_report_replicated_2026-05-15.json"), + "replicated", + ) + self.assertEqual(charts.derive_storage_type("e2e_report_nfs_local.json"), "nfs") + self.assertEqual(charts.derive_storage_type("custom.json", "fallback"), "fallback") + + with self.assertRaisesRegex(ValueError, "Unable to derive storage type"): + charts.derive_storage_type("custom.json") + + +class ListReportFilesTest(unittest.TestCase): + def test_list_report_files_returns_sorted_matches(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + nested = root / "nested" + nested.mkdir() + write_json(nested / "e2e_report_z.json", {}) + write_json(root / "e2e_report_a.json", {}) + write_json(root / "other.json", {}) + (root / "e2e_report_dir.json").mkdir() + + self.assertEqual( + [path.name for path in charts.list_report_files(root)], + ["e2e_report_a.json", "e2e_report_z.json"], + ) + + +class RenderTopDescribesTest(unittest.TestCase): + def test_render_top_describes_writes_expected_pngs(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + reports_dir = root / "reports" + out_dir = root / "charts" + reports_dir.mkdir() + write_json( + reports_dir / "e2e_report_nfs_2026-05-15.json", + { + "storageType": "nfs", + "specTimings": [ + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 30_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 20_000}, + {"name": "net", "group": "Network", "state": "passed", "runtimeMs": 10_000}, + ], + }, + ) + + files = charts.render_top_describes(reports_dir, out_dir, top_n=2) + + self.assertEqual( + [file["name"] for file in files], + ["nfs-VM-slowest-specs.png", "nfs-Disk-slowest-specs.png"], + ) + self.assertTrue((out_dir / "nfs-VM-slowest-specs.png").is_file()) + + +class MainDispatchTest(unittest.TestCase): + def test_main_dispatches_messenger_all_slowest_and_top(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + reports_dir = root / "reports" + messenger_out = root / "messenger" + chart_out = root / "charts" + manifest_path = messenger_out / "manifest.json" + debug_path = root / "debug.json" + report_path = reports_dir / "e2e_report_nfs_2026-05-15.json" + reports_dir.mkdir() + write_json( + report_path, + { + "storageType": "nfs", + "specTimings": [ + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 30_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 20_000}, + ], + }, + ) + + with contextlib.redirect_stdout(io.StringIO()): + charts.main( + [ + "messenger-all", + "--reports-dir", + str(reports_dir), + "--out-dir", + str(messenger_out), + "--manifest", + str(manifest_path), + "--debug-json", + str(debug_path), + ] + ) + charts.main( + [ + "slowest", + "--json", + str(report_path), + "--describe", + "VM", + "--out-dir", + str(chart_out), + ] + ) + charts.main( + [ + "top", + "--reports-dir", + str(reports_dir), + "--out-dir", + str(chart_out), + "--top-n", + "1", + ] + ) + + self.assertTrue(manifest_path.is_file()) + self.assertTrue(debug_path.is_file()) + self.assertTrue((chart_out / "nfs-VM-slowest-specs.png").is_file()) + + def test_top_with_zero_matching_reports_fails_clearly(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex(SystemExit, "No report files found"): + charts.main(["top", "--reports-dir", temp_dir]) + + def test_messenger_all_with_zero_matching_reports_fails_clearly(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex(SystemExit, "No report files found"): + charts.main(["messenger-all", "--reports-dir", temp_dir]) if __name__ == "__main__": diff --git a/.github/scripts/python/requirements.txt b/.github/scripts/python/requirements.txt index 6ccafc3f90..8b6f0d08f1 100644 --- a/.github/scripts/python/requirements.txt +++ b/.github/scripts/python/requirements.txt @@ -1 +1 @@ -matplotlib +matplotlib==3.10.* diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 399861963c..91f39d6591 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -503,12 +503,18 @@ jobs: cache: pip cache-dependency-path: .github/scripts/python/requirements.txt - - name: Install E2E report dependencies - run: | - npm install - python3 -m pip install -r ../python/requirements.txt + - name: Install JS report deps + run: npm install working-directory: .github/scripts/js + - name: Install Python chart deps + run: python3 -m pip install -r .github/scripts/python/requirements.txt + + - name: Show installed Python tooling + run: | + python3 -V + python3 -m pip list + - name: Generate messenger chart files run: >- python3 .github/scripts/python/e2e_report/charts.py messenger-all @@ -534,7 +540,7 @@ jobs: run: >- python3 .github/scripts/python/e2e_report/charts.py top --reports-dir downloaded-artifacts - --out-dir tmp + --out-dir tmp/charts --top-n 5 - name: Upload top-5 slowest Describe charts diff --git a/Taskfile.yaml b/Taskfile.yaml index 36e186cd4c..e698b063ed 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -63,7 +63,7 @@ tasks: - >- python3 .github/scripts/python/e2e_report/charts.py slowest --json "{{.JSON}}" - --describe "{{.DESCRIBE}}" --out-dir "{{.OUT_DIR}}" + --describe "{{.DESCRIBE}}" --out-dir "{{.OUT_DIR}}/charts" report:render:top-slowest: desc: "Render slowest-specs charts for top-N slowest Describes" @@ -77,7 +77,7 @@ tasks: - >- python3 .github/scripts/python/e2e_report/charts.py top --reports-dir "{{.REPORTS_DIR}}" - --out-dir "{{.OUT_DIR}}" + --out-dir "{{.OUT_DIR}}/charts" --top-n "{{.TOP_N}}" check-helm: diff --git a/tmp/test-ci/report/run-local-report.js b/tmp/test-ci/report/run-local-report.js new file mode 100644 index 0000000000..8f71dfca3b --- /dev/null +++ b/tmp/test-ci/report/run-local-report.js @@ -0,0 +1,657 @@ +#!/usr/bin/env node + +const fs = require("fs"); +const path = require("path"); +const { spawnSync } = require("child_process"); + +function printUsage() { + console.log(`Usage: + node tmp/test-ci/report/run-local-report.js --json /absolute/path/e2e_report_replicated_2026-05-15.json [options] + node tmp/test-ci/report/run-local-report.js --json /path/replicated.json --json /path/nfs.json [options] + node tmp/test-ci/report/run-local-report.js --json-dir /absolute/path/with/reports [options] + node tmp/test-ci/report/run-local-report.js --cluster replicated --stage configure-sdn [options] + node tmp/test-ci/report/run-local-report.js --spoiler-samples [options] + +Options: + --storage <name> Storage/cluster name for a single JSON without a CI-style file name + --cluster <name> Build a local report entry without JSON input + --branch <name> Branch label in the report. Default: local-test + --run-url <url> URL used in report links. Default: https://example.invalid/local-run + --loop-api-base-url <url> + Optional Loop base URL. Examples: + https://loop.flant.ru + https://loop.flant.ru/api/v4 + --channel-id <id> Optional Loop channel id for API delivery + --token <token> Optional Loop bot token for API delivery + --strict-loop-delivery Fail the local run when Loop delivery fails + --strict-loop-file-upload + Fail the local run when chart upload fails instead of posting without files + --json-dir <path> Load all *.json files from a directory + --xml <path> Backward-compatible alias for --json + --xml-dir <path> Backward-compatible alias for --json-dir + --stage <name> Stage result to emulate. Default: success + Allowed: success, bootstrap, configure-sdn, storage-setup, virtualization-setup, e2e-test + --result <value> Result for the selected stage. Default: failure + Allowed for non-success stages: failure, cancelled, skipped + --out-dir <path> Output directory. Default: tmp/test-ci/report/out + --charts-dir <path> Chart PNG output directory. Default: tmp/charts + --charts-manifest <path> Messenger chart manifest. Default: <work-dir>/messenger-charts/manifest.json + --spoiler-samples Render/send several spoiler/collapsible markdown samples instead of an E2E report + +Examples: + node tmp/test-ci/report/run-local-report.js --json tmp/test-ci/report/e2e_report_replicated_2026-05-15.json + node tmp/test-ci/report/run-local-report.js --json tmp/test-ci/report/e2e_report_replicated_2026-05-15.json --cluster nfs --stage configure-sdn + node tmp/test-ci/report/run-local-report.js --json-dir /tmp/e2e-reports + node tmp/test-ci/report/run-local-report.js --cluster replicated --stage e2e-test --result cancelled + node tmp/test-ci/report/run-local-report.js --json-dir /tmp/e2e-reports --loop-api-base-url "https://loop.flant.ru" --channel-id "$LOOP_CHANNEL_ID" --token "$LOOP_TOKEN" + node tmp/test-ci/report/run-local-report.js --json-dir /tmp/e2e-reports --loop-api-base-url "https://loop.flant.ru" --channel-id "$LOOP_CHANNEL_ID" --token "$LOOP_TOKEN" --strict-loop-delivery --strict-loop-file-upload + node tmp/test-ci/report/run-local-report.js --spoiler-samples --loop-api-base-url "https://loop.flant.ru" --channel-id "$LOOP_CHANNEL_ID" --token "$LOOP_TOKEN" +`); +} + +function parseArgs(argv) { + const args = {}; + + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + if (!token.startsWith("--")) { + continue; + } + + const key = token.slice(2); + const value = argv[i + 1]; + if (!value || value.startsWith("--")) { + args[key] = true; + continue; + } + + if (Object.prototype.hasOwnProperty.call(args, key)) { + if (Array.isArray(args[key])) { + args[key].push(value); + } else { + args[key] = [args[key], value]; + } + } else { + args[key] = value; + } + i += 1; + } + + return args; +} + +function toArray(value) { + if (typeof value === "undefined") { + return []; + } + + return Array.isArray(value) ? value : [value]; +} + +function ensureFileExists(filePath, label) { + if (!filePath || !fs.existsSync(filePath)) { + throw new Error(`${label} does not exist: ${filePath || "<empty>"}`); + } +} + +function mkdirp(dirPath) { + fs.mkdirSync(dirPath, { recursive: true }); +} + +function copyFile(sourcePath, targetPath) { + mkdirp(path.dirname(targetPath)); + fs.copyFileSync(sourcePath, targetPath); +} + +function resetDirectory(dirPath) { + fs.rmSync(dirPath, { recursive: true, force: true }); + mkdirp(dirPath); +} + +function createCore(outputs) { + return { + info(message) { + console.log(`[INFO] ${message}`); + }, + warning(message) { + console.warn(`[WARN] ${message}`); + }, + debug(message) { + console.debug(`[DEBUG] ${message}`); + }, + setOutput(name, value) { + outputs[name] = value; + console.log(`[OUTPUT] ${name}=${value}`); + }, + }; +} + +function buildStageResults(stage, stageResult) { + const stageResults = { + bootstrap: "success", + "configure-sdn": "success", + "storage-setup": "success", + "virtualization-setup": "success", + "e2e-test": "success", + }; + + if (!stage || stage === "success") { + return stageResults; + } + + if (!Object.prototype.hasOwnProperty.call(stageResults, stage)) { + throw new Error(`Unsupported --stage value: ${stage}`); + } + if (!["failure", "cancelled", "skipped"].includes(stageResult)) { + throw new Error( + `Unsupported --result value: ${stageResult}. Allowed: failure, cancelled, skipped` + ); + } + + stageResults[stage] = stageResult; + return stageResults; +} + +function collectJsonFilesFromDir(dirPath) { + if (!fs.existsSync(dirPath)) { + throw new Error(`JSON directory does not exist: ${dirPath}`); + } + + const entries = fs + .readdirSync(dirPath, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => path.join(dirPath, entry.name)) + .sort((left, right) => left.localeCompare(right)); + + if (entries.length === 0) { + throw new Error(`No JSON files found in directory: ${dirPath}`); + } + + return entries; +} + +function deriveStorageType(reportPath, fallbackStorage) { + const baseName = path.basename(reportPath); + const datedMatch = baseName.match( + /^e2e_report_(.+)_(\d{4}-\d{2}-\d{2}.*)\.json$/ + ); + if (datedMatch) { + return datedMatch[1]; + } + + const genericMatch = baseName.match(/^e2e_report_(.+?)_.*\.json$/); + if (genericMatch) { + return genericMatch[1]; + } + + if (fallbackStorage) { + return fallbackStorage; + } + + throw new Error( + `Unable to derive storage type from file name "${baseName}". Use a CI-style name or pass --storage for a single JSON.` + ); +} + +function normalizeLoopApiBaseUrl(value) { + const trimmedValue = String(value || "") + .trim() + .replace(/\/+$/, ""); + + if (!trimmedValue) { + return ""; + } + + if (trimmedValue.endsWith("/api/v4/posts")) { + return trimmedValue; + } + + if (trimmedValue.endsWith("/api/v4")) { + return `${trimmedValue}/posts`; + } + + return `${trimmedValue}/api/v4/posts`; +} + +function buildLoopConfig( + loopApiBaseUrl, + loopChannelId, + loopToken, + options = {} +) { + const apiUrl = normalizeLoopApiBaseUrl(loopApiBaseUrl); + const channelId = String(loopChannelId || "").trim(); + const token = String(loopToken || "").trim(); + + if (!apiUrl && !channelId && !token) { + return null; + } + if (!apiUrl || !channelId || !token) { + throw new Error( + "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" + ); + } + + return { + apiUrl, + channelId, + token, + strictDelivery: Boolean(options.strictDelivery), + strictFileUploads: Boolean(options.strictFileUploads), + }; +} + +function buildSpoilerSampleMessages() { + const longReason = + "Timed out after 300.001s. object v12n-e2e-testdata-iso status.phase is Pending, expected Ready. Expected <string>: Pending to equal <string>: Ready."; + + return { + message: [ + "## :dvp: Spoiler/collapsible markdown test", + "", + "Root message. Replies contain different syntax variants for hiding long `Reason` text.", + ].join("\n"), + threadMessages: [ + { + message: [ + "**Variant 1: HTML `<details>` / `<summary>`**", + "", + "<details>", + "<summary>Show Reason</summary>", + "", + longReason, + "", + "</details>", + ].join("\n"), + files: [], + }, + { + message: [ + "**Variant 2: inline HTML `<details>` inside table cell**", + "", + "| Tests | Reason |", + "|---|---|", + `| SynchronizedBeforeSuite | <details><summary>Show</summary>${longReason}</details> |`, + ].join("\n"), + files: [], + }, + { + message: [ + "**Variant 3: StackExchange-style spoiler blockquote**", + "", + ">! " + longReason, + ].join("\n"), + files: [], + }, + { + message: [ + "**Variant 4: Discord/Telegram-style spoiler delimiters**", + "", + `||${longReason}||`, + ].join("\n"), + files: [], + }, + { + message: [ + "**Variant 5: Mattermost spoiler plugin command text**", + "", + "/spoiler " + longReason, + ].join("\n"), + files: [], + }, + ], + }; +} + +function normalizeThreadMessage(threadMessage) { + return typeof threadMessage === "string" + ? { message: threadMessage, files: [] } + : { + message: String(threadMessage.message || ""), + files: Array.isArray(threadMessage.files) ? threadMessage.files : [], + }; +} + +function writeThreadArtifacts(threadMessages, threadMessageFile, chartDir) { + const normalizedThreadMessages = threadMessages.map(normalizeThreadMessage); + if (normalizedThreadMessages.length === 0) { + return []; + } + + fs.writeFileSync( + threadMessageFile, + `${normalizedThreadMessages + .map((threadMessage) => threadMessage.message) + .join("\n\n---\n\n")}\n` + ); + + const writtenFiles = []; + for (const [ + messageIndex, + threadMessage, + ] of normalizedThreadMessages.entries()) { + for (const file of threadMessage.files) { + if (!file || !file.buffer || !file.name) { + continue; + } + + mkdirp(chartDir); + const targetPath = path.join( + chartDir, + `${messageIndex + 1}-${path.basename(file.name)}` + ); + fs.writeFileSync(targetPath, file.buffer); + writtenFiles.push(targetPath); + } + } + + return writtenFiles; +} + +function generateMessengerCharts({ + repoRoot, + reportsDir, + chartDir, + manifestPath, +}) { + const result = spawnSync( + process.env.PYTHON || "python3", + [ + path.join(repoRoot, ".github/scripts/python/e2e_report/charts.py"), + "messenger-all", + "--reports-dir", + reportsDir, + "--out-dir", + chartDir, + "--manifest", + manifestPath, + ], + { encoding: "utf8" } + ); + + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + const output = (result.stderr || result.stdout || "").trim(); + throw new Error(`Chart rendering failed with status ${result.status}: ${output}`); + } + + return manifestPath; +} + +async function renderSpoilerSamples({ core, outDir, loop }) { + const { makeThreadedReportInLoop } = require(path.join( + path.resolve(__dirname, "../../.."), + ".github/scripts/js/e2e/report/messenger/loop-client" + )); + const { message, threadMessages } = buildSpoilerSampleMessages(); + + resetDirectory(outDir); + fs.writeFileSync( + path.join(outDir, "spoiler-samples-main.md"), + `${message}\n` + ); + fs.writeFileSync( + path.join(outDir, "spoiler-samples-thread.md"), + `${threadMessages + .map((threadMessage) => threadMessage.message) + .join("\n\n---\n\n")}\n` + ); + + core.info(message); + core.setOutput("message", message); + core.setOutput("thread_messages", JSON.stringify(threadMessages)); + + if (loop) { + await makeThreadedReportInLoop({ message, threadMessages, loop }, core); + } + + console.log(""); + console.log("Artifacts written:"); + console.log( + `- Main markdown: ${path.join(outDir, "spoiler-samples-main.md")}` + ); + console.log( + `- Thread markdown: ${path.join(outDir, "spoiler-samples-thread.md")}` + ); + if (loop) { + console.log("- Loop delivery: attempted"); + } +} + +function collectInputEntries(args) { + const jsonArgs = [ + ...toArray(args.json), + ...toArray(args.report), + ...toArray(args.xml), + ].map((value) => path.resolve(String(value))); + const jsonDirArgs = [ + ...toArray(args["json-dir"]), + ...toArray(args["report-dir"]), + ...toArray(args["xml-dir"]), + ].map((value) => path.resolve(String(value))); + const clusterArgs = toArray(args.cluster) + .map((value) => String(value).trim()) + .filter(Boolean); + const dirReports = jsonDirArgs.flatMap((dirPath) => + collectJsonFilesFromDir(dirPath) + ); + const allReports = Array.from(new Set([...jsonArgs, ...dirReports])); + + if (allReports.length === 0 && clusterArgs.length === 0) { + throw new Error( + "Pass at least one --json, one --json-dir, or one --cluster." + ); + } + + allReports.forEach((reportPath) => + ensureFileExists(reportPath, "Ginkgo JSON report") + ); + + const fallbackStorage = args.storage ? String(args.storage) : ""; + const reportEntries = allReports.map((reportPath) => { + const storageType = deriveStorageType( + reportPath, + allReports.length === 1 ? fallbackStorage : "" + ); + return { reportPath, storageType }; + }); + const clusterEntries = clusterArgs.map((storageType) => ({ + reportPath: null, + storageType, + })); + const entries = [...reportEntries, ...clusterEntries]; + + const uniqueStorageTypes = new Set(entries.map((entry) => entry.storageType)); + if (uniqueStorageTypes.size !== entries.length) { + throw new Error( + "Detected duplicate storage types across JSON or cluster inputs. Multiple entries for the same storage are not merged in the local runner." + ); + } + + return entries; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + printUsage(); + process.exit(0); + } + + const hasInputs = + Boolean(args["spoiler-samples"]) || + toArray(args.json).length > 0 || + toArray(args.report).length > 0 || + toArray(args.xml).length > 0 || + toArray(args["json-dir"]).length > 0 || + toArray(args["report-dir"]).length > 0 || + toArray(args["xml-dir"]).length > 0 || + toArray(args.cluster).length > 0; + if (!hasInputs) { + console.log("No report inputs were provided."); + console.log("Pass at least one --json, one --json-dir, or one --cluster."); + console.log(""); + printUsage(); + return; + } + + const repoRoot = path.resolve(__dirname, "../../.."); + const clusterReport = require(path.join( + repoRoot, + ".github/scripts/js/e2e/report/cluster-report" + )); + const messengerReport = require(path.join( + repoRoot, + ".github/scripts/js/e2e/report/messenger-report" + )); + const branch = String(args.branch || "local-test"); + const runUrl = String(args["run-url"] || "https://example.invalid/local-run"); + const outDir = path.resolve( + String(args["out-dir"] || path.join(repoRoot, "tmp/test-ci/report/out")) + ); + const chartDir = path.resolve( + String(args["charts-dir"] || path.join(repoRoot, "tmp/charts")) + ); + const workDir = path.resolve( + String(args["work-dir"] || path.join(path.dirname(outDir), "work")) + ); + const messengerChartDir = path.join(workDir, "messenger-charts"); + const chartsManifest = path.resolve( + String( + args["charts-manifest"] || path.join(messengerChartDir, "manifest.json") + ) + ); + const messageFile = path.join(outDir, "messenger-report.md"); + const threadMessageFile = path.join(outDir, "messenger-thread.md"); + const loopApiBaseUrl = args["loop-api-base-url"] + ? String(args["loop-api-base-url"]) + : String(process.env.LOOP_API_BASE_URL || ""); + const loopChannelId = args["channel-id"] + ? String(args["channel-id"]) + : String(process.env.LOOP_CHANNEL_ID || ""); + const loopToken = args.token + ? String(args.token) + : String(process.env.LOOP_TOKEN || ""); + const stage = String(args.stage || "success"); + const stageResult = String(args.result || "failure"); + const strictLoopDelivery = Boolean(args["strict-loop-delivery"]); + const strictLoopFileUpload = Boolean(args["strict-loop-file-upload"]); + const loop = buildLoopConfig(loopApiBaseUrl, loopChannelId, loopToken, { + strictDelivery: strictLoopDelivery, + strictFileUploads: strictLoopFileUpload, + }); + + if (args["spoiler-samples"]) { + const outputs = {}; + await renderSpoilerSamples({ core: createCore(outputs), outDir, loop }); + return; + } + + const inputEntries = collectInputEntries(args); + const stageResults = buildStageResults(stage, stageResult); + + resetDirectory(outDir); + resetDirectory(chartDir); + resetDirectory(workDir); + + process.env.REPORTS_DIR = outDir; + process.env.EXPECTED_STORAGE_TYPES = JSON.stringify( + inputEntries.map((entry) => entry.storageType) + ); + process.env.LOOP_API_BASE_URL = loopApiBaseUrl; + process.env.LOOP_CHANNEL_ID = loopChannelId; + process.env.LOOP_TOKEN = loopToken; + process.env.LOOP_STRICT_DELIVERY = strictLoopDelivery ? "1" : ""; + process.env.LOOP_STRICT_FILE_UPLOAD = strictLoopFileUpload ? "1" : ""; + process.env.CHARTS_MANIFEST = chartsManifest; + + const outputs = {}; + const core = createCore(outputs); + const context = { + serverUrl: "https://github.com", + repo: { owner: "local", repo: "virtualization2" }, + runId: "local-test", + ref: `refs/heads/${branch}`, + }; + + const generatedReports = []; + for (const entry of inputEntries) { + const reportFile = path.join( + outDir, + `e2e_report_${entry.storageType}.json` + ); + const rawReportPath = entry.reportPath + ? path.join(workDir, `e2e_report_${entry.storageType}_local.json`) + : null; + + if (entry.reportPath) { + copyFile(entry.reportPath, rawReportPath); + } + + await clusterReport({ + core, + context, + config: { + storageType: entry.storageType, + pipelineJobName: `Local E2E (${entry.storageType})`, + reportsDir: workDir, + reportFile, + stageResults, + stageJobUrls: { + bootstrap: runUrl, + "configure-sdn": runUrl, + "storage-setup": runUrl, + "virtualization-setup": runUrl, + "e2e-test": runUrl, + }, + }, + }); + generatedReports.push(reportFile); + } + + generateMessengerCharts({ + repoRoot, + reportsDir: outDir, + chartDir: messengerChartDir, + manifestPath: chartsManifest, + }); + + const renderedMessages = await messengerReport({ core }); + const message = renderedMessages.message; + const threadMessages = renderedMessages.threadMessages || []; + + fs.writeFileSync(messageFile, `${message}\n`); + const chartFiles = writeThreadArtifacts( + threadMessages, + threadMessageFile, + chartDir + ); + + console.log(""); + console.log("Artifacts written:"); + generatedReports.forEach((reportFile) => { + console.log(`- JSON: ${reportFile}`); + }); + console.log(`- Main markdown: ${messageFile}`); + if (threadMessages.length > 0) { + console.log(`- Thread markdown: ${threadMessageFile}`); + } + console.log(`- Chart manifest: ${chartsManifest}`); + chartFiles.forEach((chartFile) => { + console.log(`- Chart: ${chartFile}`); + }); + + if (loopApiBaseUrl || loopChannelId || loopToken) { + console.log("- Loop delivery: attempted"); + } +} + +main().catch((error) => { + console.error(`[ERROR] ${error.message}`); + if (process.env.DEBUG_LOCAL_REPORT === "1" && error.stack) { + console.error(error.stack); + } + process.exit(1); +}); From 078539acf0627cb47cdf540f29ea3319133a2442 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 22 May 2026 19:31:26 +0300 Subject: [PATCH 18/24] fix formatting Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../scripts/js/e2e/report/cluster-report.js | 63 +++++++++++-------- .../js/e2e/report/messenger/loop-client.js | 11 +--- .../e2e/report/shared/ginkgo-report-utils.js | 43 ++++++++++--- .../js/e2e/report/shared/report-model.js | 44 ++++++++++--- 4 files changed, 107 insertions(+), 54 deletions(-) diff --git a/.github/scripts/js/e2e/report/cluster-report.js b/.github/scripts/js/e2e/report/cluster-report.js index 6e606342b2..5b7df991e4 100644 --- a/.github/scripts/js/e2e/report/cluster-report.js +++ b/.github/scripts/js/e2e/report/cluster-report.js @@ -66,27 +66,11 @@ const { */ const workflowStages = [ - { - name: "bootstrap", - displayName: "Bootstrap cluster", - needsJobId: "bootstrap", - }, - { - name: "configure-sdn", - displayName: "Configure SDN", - needsJobId: "configure-sdn", - }, - { - name: "storage-setup", - displayName: "Configure storage", - needsJobId: "configure-storage", - }, - { - name: "virtualization-setup", - displayName: "Configure Virtualization", - needsJobId: "configure-virtualization", - }, - { name: "e2e-test", displayName: "E2E test", needsJobId: "e2e-test" }, + { name: "bootstrap", displayName: "Bootstrap cluster", needsJobId: "bootstrap" }, + { name: "configure-sdn", displayName: "Configure SDN", needsJobId: "configure-sdn" }, + { name: "storage-setup", displayName: "Configure storage", needsJobId: "configure-storage" }, + { name: "virtualization-setup", displayName: "Configure Virtualization", needsJobId: "configure-virtualization" }, + { name: "e2e-test", displayName: "E2E test", needsJobId: "e2e-test" }, ]; function readClusterReportConfigFromEnv(env = process.env) { @@ -251,7 +235,11 @@ const ginkgoOutputSource = { * @returns {string|null} Path to the source file, or null when none exists. */ function findGinkgoSource(config, source) { - return findSingleMatchingFile(config.reportsDir, source.pattern(config.storageType), source.label); + return findSingleMatchingFile( + config.reportsDir, + source.pattern(config.storageType), + source.label + ); } /** @@ -285,12 +273,21 @@ function parseGinkgoFile(filePath, core, source) { source: source.okSource, }; } catch (error) { - core.warning(`Unable to parse ${source.label} ${filePath}: ${error.message}`); + core.warning( + `Unable to parse ${source.label} ${filePath}: ${error.message}` + ); return emptyParsedReport(source.invalidSource); } } -function buildReportPayload({ config, context, fallbackWorkflowRunUrl, branchName, parsedReport, sourcePath }) { +function buildReportPayload({ + config, + context, + fallbackWorkflowRunUrl, + branchName, + parsedReport, + sourcePath, +}) { const clusterStatus = buildClusterStatus(config.stageResults); const testStatus = buildTestStatus( config.stageResults["e2e-test"], @@ -298,8 +295,16 @@ function buildReportPayload({ config, context, fallbackWorkflowRunUrl, branchNam clusterStatus, parsedReport.metrics ); - const reportSummary = buildReportSummary(config.storageType, clusterStatus, testStatus); - const workflowRunUrl = getReportJobUrl(reportSummary, config.stageJobUrls, fallbackWorkflowRunUrl); + const reportSummary = buildReportSummary( + config.storageType, + clusterStatus, + testStatus + ); + const workflowRunUrl = getReportJobUrl( + reportSummary, + config.stageJobUrls, + fallbackWorkflowRunUrl + ); return { schemaVersion: 1, @@ -327,7 +332,11 @@ function buildReportPayload({ config, context, fallbackWorkflowRunUrl, branchNam }; } -function getReportJobUrl(reportSummary, stageJobUrls = {}, fallbackWorkflowRunUrl) { +function getReportJobUrl( + reportSummary, + stageJobUrls = {}, + fallbackWorkflowRunUrl +) { if (reportSummary.failedStage && stageJobUrls[reportSummary.failedStage]) { return stageJobUrls[reportSummary.failedStage]; } diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 862724020f..2db05e2109 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -140,14 +140,9 @@ async function uploadFileToLoop(loop, fileName, buffer, core, mimeType, { fetch: * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<void>} */ -async function makeThreadedReportInLoop( - { message, threadMessages, loop }, - core, - { fetch: fetchFn = globalThis.fetch } = {} -) { - const rootPost = await postToLoopApi(loop, message, undefined, core, [], { - fetch: fetchFn, - }); +async function makeThreadedReportInLoop({ message, threadMessages, loop }, core, { + fetch: fetchFn = globalThis.fetch } = {}) { + const rootPost = await postToLoopApi(loop, message, undefined, core, [], { fetch: fetchFn }); if (!rootPost.id) { throw new Error("Loop API did not return a post id; thread replies cannot be attached"); diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index 6241c65cbb..b8afdecf96 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -72,13 +72,18 @@ function formatSpecName(specReport) { .map((part) => String(part || "").trim()) .filter(Boolean); const leafText = String(specReport.LeafNodeText || "").trim(); - const labels = [ - ...new Set([...flattenLabels(specReport.ContainerHierarchyLabels), ...flattenLabels(specReport.LeafNodeLabels)]), - ]; + const labels = [...new Set([ + ...flattenLabels(specReport.ContainerHierarchyLabels), + ...flattenLabels(specReport.LeafNodeLabels), + ])]; const labelSuffix = labels.map((label) => `[${label}]`).join(" "); const body = [...hierarchyParts, leafText].filter(Boolean).join(" "); - return [`[${nodeType}]`, body, labelSuffix].filter(Boolean).join(" ").replace(/\s+/g, " ").trim(); + rreturn [`[${nodeType}]`, body, labelSuffix] + .filter(Boolean) + .join(" ") + .replace(/\s+/g, " ") + .trim(); } function runtimeMs(value) { @@ -115,7 +120,10 @@ function getMetricKeyForState(state) { function formatFailureReason(specReport) { const failure = (specReport && specReport.Failure) || {}; - return String(failure.Message || failure.ForwardedPanic || "").trim() || String(specReport.State || "failed").trim(); + return ( + String(failure.Message || failure.ForwardedPanic || "").trim() || + String(specReport.State || "failed").trim() + ); } const failureStates = new Set(["failed", "errors"]); @@ -211,13 +219,21 @@ function parseGinkgoReport(jsonContent) { } const completedSpecs = metrics.passed + metrics.failed + metrics.errors; - metrics.successRate = completedSpecs > 0 ? Number(((metrics.passed / completedSpecs) * 100).toFixed(2)) : 0; + metrics.successRate = + completedSpecs > 0 + ? Number(((metrics.passed / completedSpecs) * 100).toFixed(2)) + : 0; return { metrics, failedTests: Array.from(new Set(failedTests)), failedTestDetails: Array.from( - new Map(failedTestDetails.map((test) => [`${test.name}\u0000${test.reason}`, test])).values() + new Map( + failedTestDetails.map((test) => [ + `${test.name}\u0000${test.reason}`, + test, + ]) + ).values() ), specTimings, suiteTotalMs, @@ -225,7 +241,12 @@ function parseGinkgoReport(jsonContent) { }; } -const suiteNodeTypes = ["SynchronizedBeforeSuite", "BeforeSuite", "SynchronizedAfterSuite", "AfterSuite"]; +const suiteNodeTypes = [ + "SynchronizedBeforeSuite", + "BeforeSuite", + "SynchronizedAfterSuite", + "AfterSuite", +]; // Match Ginkgo failure markers for suite-level nodes in two forms: // 1. "[<SuiteNode>] [FAILED]" — main failure line in the stdout body. @@ -272,7 +293,11 @@ function isReasonStopLine(line) { } function isReasonNoiseLine(line, suiteHeader, failedMarker) { - return line === suiteHeader || line.startsWith(failedMarker) || line.startsWith("/"); + return ( + line === suiteHeader || + line.startsWith(failedMarker) || + line.startsWith("/") + ); } /** diff --git a/.github/scripts/js/e2e/report/shared/report-model.js b/.github/scripts/js/e2e/report/shared/report-model.js index 4d204d542e..3e8523e9e0 100644 --- a/.github/scripts/js/e2e/report/shared/report-model.js +++ b/.github/scripts/js/e2e/report/shared/report-model.js @@ -54,16 +54,21 @@ function ginkgoOutputPattern(storageType) { } const stageMessage = { - bootstrap: "BOOTSTRAP CLUSTER", + "bootstrap": "BOOTSTRAP CLUSTER", "configure-sdn": "CONFIGURE SDN", "storage-setup": "STORAGE SETUP", "virtualization-setup": "VIRTUALIZATION SETUP", "e2e-test": "E2E TEST", - ready: "CLUSTER READY", + "ready": "CLUSTER READY", "artifact-missing": "TEST REPORTS NOT FOUND", }; -const clusterSetupStages = ["bootstrap", "configure-sdn", "storage-setup", "virtualization-setup"]; +const clusterSetupStages = [ + "bootstrap", + "configure-sdn", + "storage-setup", + "virtualization-setup", +]; function zeroMetrics() { return { @@ -129,7 +134,12 @@ function buildClusterStatus(stageResults) { }; } -function buildTestStatus(testResult, reportSource, clusterStatus, metrics = {}) { +function buildTestStatus( + testResult, + reportSource, + clusterStatus, + metrics = {} +) { const stageLabel = stageMessage["e2e-test"]; if (clusterStatus.status !== "success") { @@ -143,13 +153,20 @@ function buildTestStatus(testResult, reportSource, clusterStatus, metrics = {}) const normalizedResult = normalizeJobResult(testResult); if (reportSource === "ginkgo-json" || reportSource === "ginkgo-output") { - const hasReportedFailures = Number(metrics.failed || 0) > 0 || Number(metrics.errors || 0) > 0; - const status = normalizedResult === "success" && hasReportedFailures ? "failure" : normalizedResult; + const hasReportedFailures = + Number(metrics.failed || 0) > 0 || Number(metrics.errors || 0) > 0; + const status = + normalizedResult === "success" && hasReportedFailures + ? "failure" + : normalizedResult; return { status, reason: status === "success" ? "" : "ginkgo-failed", - message: status === "success" ? "✅ E2E TESTS PASSED" : buildStatusMessage(status, stageLabel), + message: + status === "success" + ? "✅ E2E TESTS PASSED" + : buildStatusMessage(status, stageLabel), }; } @@ -218,7 +235,8 @@ function buildReportSummary(storageType, clusterStatus, testStatus) { return { failedStage: testStatus.status === "success" ? "success" : "e2e-test", - failedStageLabel: testStatus.status === "success" ? "SUCCESS" : stageMessage["e2e-test"], + failedStageLabel: + testStatus.status === "success" ? "SUCCESS" : stageMessage["e2e-test"], failedJobName: `E2E test (${storageType})`, reportKind: "tests", status: testStatus.status, @@ -238,7 +256,10 @@ function isMissingReport(report) { function isClusterFailureReport(report) { if (report.clusterStatus) { - return report.clusterStatus.status !== "success" && report.clusterStatus.status !== "missing"; + return ( + report.clusterStatus.status !== "success" && + report.clusterStatus.status !== "missing" + ); } return report.reportKind !== "tests" && !isMissingReport(report); @@ -250,7 +271,10 @@ function isTestResultReport(report) { } if (report.testStatus) { - return report.testStatus.status !== "not-run" && report.testStatus.status !== "missing"; + return ( + report.testStatus.status !== "not-run" && + report.testStatus.status !== "missing" + ); } return report.reportKind === "tests"; From ac3e0ebf251e108495ca9ec89e6f10a21cf27615 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 22 May 2026 20:06:20 +0300 Subject: [PATCH 19/24] fix formatting 2 Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../scripts/js/e2e/report/cluster-report.js | 29 ++++- .../js/e2e/report/cluster-report.test.js | 115 ++++++++++++------ .../scripts/js/e2e/report/messenger-report.js | 23 +++- .../js/e2e/report/messenger-report.test.js | 19 ++- .../js/e2e/report/messenger/loop-client.js | 49 ++++++-- .../e2e/report/messenger/loop-client.test.js | 8 +- .../js/e2e/report/messenger/markdown.js | 57 +++++++-- .../e2e/report/shared/ginkgo-report-utils.js | 18 +-- 8 files changed, 234 insertions(+), 84 deletions(-) diff --git a/.github/scripts/js/e2e/report/cluster-report.js b/.github/scripts/js/e2e/report/cluster-report.js index 5b7df991e4..8a357d13ac 100644 --- a/.github/scripts/js/e2e/report/cluster-report.js +++ b/.github/scripts/js/e2e/report/cluster-report.js @@ -13,7 +13,10 @@ const fs = require("fs"); const { findSingleMatchingFile } = require("./shared/fs-utils"); -const { parseGinkgoOutput, parseGinkgoReport } = require("./shared/ginkgo-report-utils"); +const { + parseGinkgoOutput, + parseGinkgoReport, +} = require("./shared/ginkgo-report-utils"); const { archivedReportPattern, buildClusterStatus, @@ -374,20 +377,29 @@ function setReportOutputs(report, reportFile, core) { * @throws {Error} If config is incomplete or the report file cannot be written. */ async function buildClusterReport({ core, context, github, config } = {}) { - const resolvedConfig = requireClusterReportConfig(config || readClusterReportConfigFromEnv()); + const resolvedConfig = requireClusterReportConfig( + config || readClusterReportConfigFromEnv() + ); if (!resolvedConfig.stageResults) { resolvedConfig.stageResults = readStageResultsFromEnv(); } if (!resolvedConfig.stageJobUrls && github) { - resolvedConfig.stageJobUrls = await readStageJobUrlsFromApi(github, context, resolvedConfig, core); + resolvedConfig.stageJobUrls = await readStageJobUrlsFromApi( + github, + context, + resolvedConfig, + core + ); } const fallbackWorkflowRunUrl = getWorkflowRunUrl(context); const branchName = getBranchName(context); const rawReportPath = findGinkgoSource(resolvedConfig, ginkgoJsonSource); - const outputPath = rawReportPath ? null : findGinkgoSource(resolvedConfig, ginkgoOutputSource); + const outputPath = rawReportPath + ? null + : findGinkgoSource(resolvedConfig, ginkgoOutputSource); const sourcePath = rawReportPath || outputPath; const sourceDescriptor = rawReportPath ? ginkgoJsonSource : ginkgoOutputSource; @@ -408,9 +420,14 @@ async function buildClusterReport({ core, context, github, config } = {}) { }); try { - fs.writeFileSync(resolvedConfig.reportFile, `${JSON.stringify(report, null, 2)}\n`); + fs.writeFileSync( + resolvedConfig.reportFile, + `${JSON.stringify(report, null, 2)}\n` + ); } catch (error) { - throw new Error(`Unable to write cluster report file ${resolvedConfig.reportFile}: ${error.message}`); + throw new Error( + `Unable to write cluster report file ${resolvedConfig.reportFile}: ${error.message}` + ); } setReportOutputs(report, resolvedConfig.reportFile, core); diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index d88fcbb550..ce322c3ecb 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -47,10 +47,14 @@ function createContext() { * @returns {Record<string, any>} Mocked GitHub client. */ function createGithub(jobNames) { - const jobs = jobNames.map((name, index) => ({ - name, - html_url: `https://github.com/test/repo/actions/runs/12345/job/${index + 1}`, - })); + const jobs = jobNames.map( + (name, index) => ({ + name, + html_url: `https://github.com/test/repo/actions/runs/12345/job/${ + index + 1 + }`, + }) + ); return { rest: { @@ -213,13 +217,17 @@ describe("cluster-report", () => { }); expect(report.cluster).toBe("nfs"); - expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); + expect(report.workflowRunUrl).toBe( + "https://github.com/test/repo/actions/runs/12345" + ); expect(report.branch).toBe("main"); expect(report.clusterStatus).toMatchObject({ status: "failure", stage: "configure-sdn", }); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe("nfs"); + expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe( + "nfs" + ); })); test("builds report from environment config", async () => @@ -230,11 +238,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - bootstrap: { result: "success" }, - "configure-sdn": { result: "success" }, - "configure-storage": { result: "success" }, + "bootstrap": { result: "success" }, + "configure-sdn": { result: "success" }, + "configure-storage": { result: "success" }, "configure-virtualization": { result: "success" }, - "e2e-test": { result: "success" }, + "e2e-test": { result: "success" }, }); expect(readClusterReportConfigFromEnv()).toMatchObject({ @@ -250,9 +258,13 @@ describe("cluster-report", () => { }); expect(report.cluster).toBe("replicated"); - expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); + expect(report.workflowRunUrl).toBe( + "https://github.com/test/repo/actions/runs/12345" + ); expect(report.branch).toBe("main"); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe("replicated"); + expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).cluster).toBe( + "replicated" + ); })); test("reads stage results from env vars", async () => @@ -263,11 +275,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - bootstrap: { result: "success" }, - "configure-sdn": { result: "failure" }, - "configure-storage": { result: "skipped" }, + "bootstrap": { result: "success" }, + "configure-sdn": { result: "failure" }, + "configure-storage": { result: "skipped" }, "configure-virtualization": { result: "skipped" }, - "e2e-test": { result: "skipped" }, + "e2e-test": { result: "skipped" }, }); const report = await buildClusterReport({ @@ -280,7 +292,9 @@ describe("cluster-report", () => { stage: "configure-sdn", }); // No github — falls back to workflow run URL - expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); + expect(report.workflowRunUrl).toBe( + "https://github.com/test/repo/actions/runs/12345" + ); })); test("fetches job URLs from GitHub API", async () => @@ -291,11 +305,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - bootstrap: { result: "success" }, - "configure-sdn": { result: "failure" }, - "configure-storage": { result: "skipped" }, + "bootstrap": { result: "success" }, + "configure-sdn": { result: "failure" }, + "configure-storage": { result: "skipped" }, "configure-virtualization": { result: "skipped" }, - "e2e-test": { result: "skipped" }, + "e2e-test": { result: "skipped" }, }); const report = await buildClusterReport({ @@ -315,7 +329,9 @@ describe("cluster-report", () => { stage: "configure-sdn", }); // github provided — URL points to the specific failed job - expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345/job/2"); + expect(report.workflowRunUrl).toBe( + "https://github.com/test/repo/actions/runs/12345/job/2" + ); })); test("works without github (no job URLs)", async () => @@ -325,11 +341,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - bootstrap: { result: "success" }, - "configure-sdn": { result: "success" }, - "configure-storage": { result: "success" }, + "bootstrap": { result: "success" }, + "configure-sdn": { result: "success" }, + "configure-storage": { result: "success" }, "configure-virtualization": { result: "success" }, - "e2e-test": { result: "success" }, + "e2e-test": { result: "success" }, }); const report = await buildClusterReport({ @@ -340,12 +356,17 @@ describe("cluster-report", () => { expect(report.cluster).toBe("replicated"); // stageJobUrls is empty — falls back to workflow run URL - expect(report.workflowRunUrl).toBe("https://github.com/test/repo/actions/runs/12345"); + expect(report.workflowRunUrl).toBe( + "https://github.com/test/repo/actions/runs/12345" + ); })); test("marks Ginkgo JSON with failed specs as failed", async () => inTempDir(async (tempDir) => { - const rawReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-15.json"); + const rawReportPath = path.join( + tempDir, + "e2e_report_replicated_2026-04-15.json" + ); fs.writeFileSync( rawReportPath, createGinkgoReport({ @@ -418,7 +439,10 @@ describe("cluster-report", () => { total: 4, successRate: 33.33, }); - expect(report.failedTests).toEqual(["[It] Suite fails & burns [Slow]", "[It] Other errors <loudly>"]); + expect(report.failedTests).toEqual([ + "[It] Suite fails & burns [Slow]", + "[It] Other errors <loudly>", + ]); expect(report.failedTestDetails).toEqual([ { name: "[It] Suite fails & burns [Slow]", @@ -481,7 +505,10 @@ describe("cluster-report", () => { test("captures suite-level failures from Ginkgo JSON", async () => inTempDir(async (tempDir) => { - const rawReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-15.json"); + const rawReportPath = path.join( + tempDir, + "e2e_report_replicated_2026-04-15.json" + ); fs.writeFileSync( rawReportPath, createGinkgoReport({ @@ -625,7 +652,9 @@ describe("cluster-report", () => { const detail = report.failedTestDetails[0]; expect(detail.name).toBe("[SynchronizedBeforeSuite]"); expect(detail.reason).toContain("Timed out after 300.001s."); - expect(detail.reason).toContain("object v12n-e2e-testdata-iso status.phase is Pending, expected Ready"); + expect(detail.reason).toContain( + "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready" + ); // The plain "[SynchronizedBeforeSuite]" header that follows the // "[FAILED] [307.964 seconds]" line must not leak into the reason. expect(detail.reason.split("\n")[0]).not.toBe("[SynchronizedBeforeSuite]"); @@ -633,8 +662,15 @@ describe("cluster-report", () => { test("fails when multiple matching Ginkgo JSON reports exist", async () => inTempDir(async (tempDir) => { - const firstReportPath = path.join(tempDir, "nested", "e2e_report_replicated_2026-04-15.json"); - const secondReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-16.json"); + const firstReportPath = path.join( + tempDir, + "nested", + "e2e_report_replicated_2026-04-15.json" + ); + const secondReportPath = path.join( + tempDir, + "e2e_report_replicated_2026-04-16.json" + ); fs.mkdirSync(path.dirname(firstReportPath), { recursive: true }); fs.writeFileSync( @@ -694,7 +730,9 @@ describe("cluster-report", () => { status: "missing", reason: "ginkgo-report-invalid", }); - expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Unable to parse Ginkgo JSON report")); + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining("Unable to parse Ginkgo JSON report") + ); })); test("throws a descriptive error when writing the cluster report fails", async () => @@ -716,7 +754,9 @@ describe("cluster-report", () => { context: createContext(), config, }) - ).rejects.toThrow(`Unable to write cluster report file ${reportFile}: disk full`); + ).rejects.toThrow( + `Unable to write cluster report file ${reportFile}: disk full` + ); } finally { writeSpy.mockRestore(); } @@ -742,7 +782,10 @@ describe("cluster-report", () => { specs.push( createSpecReport({ - containerHierarchyTexts: ["VirtualMachineOperationRestore", "restores a virtual machine from a snapshot"], + containerHierarchyTexts: [ + "VirtualMachineOperationRestore", + "restores a virtual machine from a snapshot", + ], containerHierarchyLabels: [["Slow"], []], leafNodeText: "BestEffort restore mode; automatic restart approval mode; manual run policy", state: "failed", diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index 00e9bc3633..f301c6f809 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -17,8 +17,14 @@ const { REPORT_FILE_PATTERN } = require("./shared/report-model"); const { getClusterChartFiles } = require("./messenger/chart-files"); const { makeThreadedReportInLoop } = require("./messenger/loop-client"); const { readMessengerConfigFromEnv } = require("./messenger/config"); -const { createMissingReport, getReportClusterKey } = require("./messenger/model"); -const { buildMainMessage, buildThreadMessages } = require("./messenger/markdown"); +const { + createMissingReport, + getReportClusterKey, +} = require("./messenger/model"); +const { + buildMainMessage, + buildThreadMessages, +} = require("./messenger/markdown"); /** * @typedef {Object} MessengerReportCore @@ -73,7 +79,9 @@ function readReports(reportsDir, configuredClusters, core) { } // Configured clusters first, in declared order; missing ones get synthetic reports. - const result = configuredClusters.map((name) => reportsByCluster.get(name) ?? createMissingReport(name)); + const result = configuredClusters.map( + (name) => reportsByCluster.get(name) ?? createMissingReport(name) + ); // Any extra clusters not in the configured list, sorted alphabetically. const configuredSet = new Set(configuredClusters); @@ -83,7 +91,9 @@ function readReports(reportsDir, configuredClusters, core) { extras.push(report); } } - extras.sort((a, b) => getReportClusterKey(a).localeCompare(getReportClusterKey(b))); + extras.sort((a, b) => + getReportClusterKey(a).localeCompare(getReportClusterKey(b)) + ); return [...result, ...extras]; } @@ -129,7 +139,10 @@ async function renderMessengerReport({ core, reportsDir }) { core.info(message); core.setOutput("message", message); - core.setOutput("thread_messages", JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message))); + core.setOutput( + "thread_messages", + JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message)) + ); if (config.loop) { try { diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index e62875a21c..69f93f13a3 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -381,7 +381,8 @@ describe("messenger-report", () => { files: [], }, { - message: "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + message: + "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", files: [], }, ]); @@ -478,7 +479,9 @@ describe("messenger-report", () => { expect(result.message).not.toContain("Branch: `main`"); expect(result.message).toContain("### Cluster failures"); - expect(result.message).toContain("- [replicated](https://example.invalid/replicated): ❌ CONFIGURE SDN FAILED"); + expect(result.message).toContain( + "- [replicated](https://example.invalid/replicated): ❌ CONFIGURE SDN FAILED" + ); expect(result.threadMessages).toEqual([]); })); @@ -701,7 +704,9 @@ describe("messenger-report", () => { // Empty body → no post id → thread replies cannot be sent → warning emitted. expect(global.fetch).toHaveBeenCalledTimes(1); - expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Loop API did not return a post id")); + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining("Loop API did not return a post id") + ); // Report outputs are still set because the message was built before sending. expect(core.setOutput).toHaveBeenCalledWith("thread_messages", "[]"); })); @@ -746,8 +751,12 @@ describe("messenger-report", () => { // Non-JSON body → parse warning → no post id → delivery warning. expect(global.fetch).toHaveBeenCalledTimes(1); - expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Loop API returned a non-JSON response body")); - expect(core.warning).toHaveBeenCalledWith(expect.stringContaining("Loop API did not return a post id")); + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining("Loop API returned a non-JSON response body") + ); + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining("Loop API did not return a post id") + ); // Report outputs are still set because the message was built before sending. expect(core.setOutput).toHaveBeenCalledWith("thread_messages", "[]"); })); diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 2db05e2109..afbefe7cdf 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -46,7 +46,9 @@ function parseLoopApiPayload(responseText, core) { try { return JSON.parse(responseText); } catch (error) { - core.warning(`Loop API returned a non-JSON response body: ${error.message}`); + core.warning( + `Loop API returned a non-JSON response body: ${error.message}` + ); return {}; } } @@ -62,7 +64,14 @@ function parseLoopApiPayload(responseText, core) { * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<Record<string, any>>} Parsed Loop API response. */ -async function postToLoopApi(loop, message, rootId, core, fileIds = [], { fetch: fetchFn = globalThis.fetch } = {}) { +async function postToLoopApi( + loop, + message, + rootId, + core, + fileIds = [], + { fetch: fetchFn = globalThis.fetch } = {} +) { const body = { channel_id: loop.channelId, message, @@ -81,7 +90,9 @@ async function postToLoopApi(loop, message, rootId, core, fileIds = [], { fetch: const responseText = await response.text(); if (!response.ok) { - throw new Error(`Loop API request failed with status ${response.status}: ${responseText}`); + throw new Error( + `Loop API request failed with status ${response.status}: ${responseText}` + ); } const payload = parseLoopApiPayload(responseText, core); @@ -104,7 +115,14 @@ function getFilesApiUrl(apiUrl) { * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<string>} Uploaded Loop file id. */ -async function uploadFileToLoop(loop, fileName, buffer, core, mimeType, { fetch: fetchFn = globalThis.fetch } = {}) { +async function uploadFileToLoop( + loop, + fileName, + buffer, + core, + mimeType, + { fetch: fetchFn = globalThis.fetch } = {} +) { const formData = new FormData(); formData.append("channel_id", loop.channelId); formData.append("files", new Blob([buffer], { type: mimeType }), fileName); @@ -119,7 +137,9 @@ async function uploadFileToLoop(loop, fileName, buffer, core, mimeType, { fetch: const responseText = await response.text(); if (!response.ok) { - throw new Error(`Loop file upload failed with status ${response.status}: ${responseText}`); + throw new Error( + `Loop file upload failed with status ${response.status}: ${responseText}` + ); } const payload = parseLoopApiPayload(responseText, core); @@ -140,12 +160,19 @@ async function uploadFileToLoop(loop, fileName, buffer, core, mimeType, { fetch: * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise<void>} */ -async function makeThreadedReportInLoop({ message, threadMessages, loop }, core, { - fetch: fetchFn = globalThis.fetch } = {}) { - const rootPost = await postToLoopApi(loop, message, undefined, core, [], { fetch: fetchFn }); +async function makeThreadedReportInLoop( + { message, threadMessages, loop }, + core, + { fetch: fetchFn = globalThis.fetch } = {} +) { + const rootPost = await postToLoopApi(loop, message, undefined, core, [], { + fetch: fetchFn, + }); if (!rootPost.id) { - throw new Error("Loop API did not return a post id; thread replies cannot be attached"); + throw new Error( + "Loop API did not return a post id; thread replies cannot be attached" + ); } for (const reply of threadMessages) { @@ -159,7 +186,9 @@ async function makeThreadedReportInLoop({ message, threadMessages, loop }, core, }) ) ); - fileIds = results.filter((result) => result.status === "fulfilled").map((result) => result.value); + fileIds = results + .filter((result) => result.status === "fulfilled") + .map((result) => result.value); const failures = results.filter((result) => result.status === "rejected"); for (const failure of failures) { diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 4454b92de2..14b57ae2e1 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -190,7 +190,9 @@ describe("loop-client", () => { expect(replyBody.file_ids).toEqual(["file-one"]); expect(core.warning).toHaveBeenCalledWith( - expect.stringContaining("Loop file upload failed for one attachment: Loop file upload failed with status 403") + expect.stringContaining( + "Loop file upload failed for one attachment: Loop file upload failed with status 403" + ) ); expect(core.warning).toHaveBeenCalledTimes(1); }); @@ -214,7 +216,9 @@ describe("loop-client", () => { text: async () => "permission denied", }, ]; - global.fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); await expect( makeThreadedReportInLoop( diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index c0f39cb01b..db7a696706 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -38,7 +38,9 @@ function formatRate(value) { function formatClusterLink(report) { const clusterName = sanitizeCell(report.cluster || report.storageType); - return report.workflowRunUrl ? `[${clusterName}](${report.workflowRunUrl})` : clusterName; + return report.workflowRunUrl + ? `[${clusterName}](${report.workflowRunUrl})` + : clusterName; } function splitReportsBySection(orderedReports) { @@ -47,14 +49,20 @@ function splitReportsBySection(orderedReports) { return { testsReports: reports.filter(isTestResultReport), stageFailureReports: reports.filter(isClusterFailureReport), - missingReports: reports.filter((report) => isMissingReport(report) && !isClusterFailureReport(report)), + missingReports: reports.filter( + (report) => isMissingReport(report) && !isClusterFailureReport(report) + ), }; } function renderBranchLine(orderedReports) { - const branches = Array.from(new Set(orderedReports.map((report) => report.branch).filter(Boolean))); + const branches = Array.from( + new Set(orderedReports.map((report) => report.branch).filter(Boolean)) + ); - return branches.length === 1 && branches[0] !== "main" ? [`Branch: \`${branches[0]}\``, ""] : []; + return branches.length === 1 && branches[0] !== "main" + ? [`Branch: \`${branches[0]}\``, ""] + : []; } /** @@ -138,7 +146,9 @@ function renderTestResultsSection(testsReports) { return []; } - const hasGinkgoErrors = testsReports.some((report) => Number((report.metrics || {}).errors || 0) > 0); + const hasGinkgoErrors = testsReports.some( + (report) => Number((report.metrics || {}).errors || 0) > 0 + ); const columns = buildTestResultsColumns(hasGinkgoErrors); const rows = [ buildMarkdownRow(columns.map((column) => column.header)), @@ -147,7 +157,9 @@ function renderTestResultsSection(testsReports) { for (const report of testsReports) { const metrics = report.metrics || {}; - rows.push(buildMarkdownRow(columns.map((column) => column.value(report, metrics)))); + rows.push( + buildMarkdownRow(columns.map((column) => column.value(report, metrics))) + ); } return ["### Test results", "", ...rows, ""]; @@ -170,7 +182,10 @@ function renderBulletSection(title, reports, getMessage) { return []; } - const bullets = reports.map((report) => `- ${formatClusterLink(report)}: ${sanitizeListItem(getMessage(report))}`); + const bullets = reports.map( + (report) => + `- ${formatClusterLink(report)}: ${sanitizeListItem(getMessage(report))}` + ); return [`### ${title}`, "", ...bullets, ""]; } @@ -201,13 +216,22 @@ function getMissingReportMessage(report) { */ function buildMainMessage(orderedReports) { const reportDate = getReportDate(orderedReports); - const { testsReports, stageFailureReports, missingReports } = splitReportsBySection(orderedReports); + const { testsReports, stageFailureReports, missingReports } = + splitReportsBySection(orderedReports); const lines = [ `## :dvp: DVP | E2E on nested clusters | ${reportDate}`, "", ...renderBranchLine(orderedReports), - ...renderBulletSection("Cluster failures", stageFailureReports, getClusterFailureMessage), - ...renderBulletSection("Missing reports", missingReports, getMissingReportMessage), + ...renderBulletSection( + "Cluster failures", + stageFailureReports, + getClusterFailureMessage + ), + ...renderBulletSection( + "Missing reports", + missingReports, + getMissingReportMessage + ), ...renderTestResultsSection(testsReports), ]; @@ -244,7 +268,10 @@ function getFailedTestGroupName(testName) { } function getFailedTestEntries(report) { - if (Array.isArray(report.failedTestDetails) && report.failedTestDetails.length > 0) { + if ( + Array.isArray(report.failedTestDetails) && + report.failedTestDetails.length > 0 + ) { return report.failedTestDetails.map((test) => ({ name: test.name, reason: test.reason, @@ -294,7 +321,9 @@ function renderFailedTestsThreadMessage(report) { lines.push("| Tests | Reason |"); lines.push("|---|---|"); for (const group of failedGroups) { - lines.push(`| ${sanitizeCell(group.name)} | ${sanitizeCell(group.reason)} |`); + lines.push( + `| ${sanitizeCell(group.name)} | ${sanitizeCell(group.reason)} |` + ); } } else { lines.push( @@ -358,7 +387,9 @@ async function buildThreadMessages(orderedReports, { getClusterChartFiles, core if (hasFailedTests(report)) { const clusterMessage = renderFailedTestsThreadMessage(report); messageParts.push( - renderedFailedTestsHeading ? clusterMessage : ["### Failed tests", clusterMessage].join("\n\n") + renderedFailedTestsHeading + ? clusterMessage + : ["### Failed tests", clusterMessage].join("\n\n") ); renderedFailedTestsHeading = true; } else { diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index b8afdecf96..9fe4a6a26b 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -79,11 +79,11 @@ function formatSpecName(specReport) { const labelSuffix = labels.map((label) => `[${label}]`).join(" "); const body = [...hierarchyParts, leafText].filter(Boolean).join(" "); - rreturn [`[${nodeType}]`, body, labelSuffix] - .filter(Boolean) - .join(" ") - .replace(/\s+/g, " ") - .trim(); + return [`[${nodeType}]`, body, labelSuffix] + .filter(Boolean) + .join(" ") + .replace(/\s+/g, " ") + .trim(); } function runtimeMs(value) { @@ -169,7 +169,8 @@ function parseGinkgoReport(jsonContent) { const failedTests = []; const failedTestDetails = []; const specTimings = []; - const startedAt = suites.find((suite) => suite && suite.StartTime)?.StartTime || null; + const startedAt = + suites.find((suite) => suite && suite.StartTime)?.StartTime || null; let suiteTotalMs = 0; for (const suite of suites) { @@ -205,7 +206,10 @@ function parseGinkgoReport(jsonContent) { group: hierarchyParts[0] || "Top-level Its", state: metricKey, runtimeMs: runtimeMs(specReport.RunTime), - labels: flattenLabels([...toArray(specReport.ContainerHierarchyLabels), ...toArray(specReport.LeafNodeLabels)]), + labels: flattenLabels([ + ...toArray(specReport.ContainerHierarchyLabels), + ...toArray(specReport.LeafNodeLabels), + ]), }); if (failureStates.has(metricKey)) { From a37a2ca966572a5aeb713cb9b0a1002e30607c65 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Fri, 22 May 2026 20:24:06 +0300 Subject: [PATCH 20/24] fix formatting 3 Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/cluster-report.test.js | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index ce322c3ecb..272b5c95bd 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -495,7 +495,10 @@ describe("cluster-report", () => { expect(core.setOutput).toHaveBeenCalledWith("report_kind", "tests"); expect(core.setOutput).toHaveBeenCalledWith("status", "failure"); expect(core.setOutput).toHaveBeenCalledWith("failed_stage", "e2e-test"); - expect(core.setOutput).toHaveBeenCalledWith("failed_stage_label", "E2E TEST"); + expect(core.setOutput).toHaveBeenCalledWith( + "failed_stage_label", + "E2E TEST" + ); expect(core.setOutput).toHaveBeenCalledWith( "workflow_run_url", "https://github.com/test/repo/actions/runs/12345" @@ -518,7 +521,8 @@ describe("cluster-report", () => { leafNodeType: "SynchronizedBeforeSuite", state: "failed", failure: { - Message: "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", + Message: + "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", }, }), ], @@ -549,14 +553,18 @@ describe("cluster-report", () => { expect(report.failedTestDetails).toEqual([ { name: "[SynchronizedBeforeSuite]", - reason: "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", + reason: + "object v12n-e2e-testdata-iso status.phase is Pending, expected Ready", }, ]); })); test("uses Ginkgo output log when JSON report is missing", async () => inTempDir(async (tempDir) => { - const outputPath = path.join(tempDir, "e2e_output_replicated_2026-04-15.log"); + const outputPath = path.join( + tempDir, + "e2e_output_replicated_2026-04-15.log" + ); fs.writeFileSync( outputPath, [ @@ -601,7 +609,10 @@ describe("cluster-report", () => { test("parses real Ginkgo BeforeSuite failure stdout", async () => inTempDir(async (tempDir) => { - const outputPath = path.join(tempDir, "e2e_output_replicated_2026-05-14.log"); + const outputPath = path.join( + tempDir, + "e2e_output_replicated_2026-05-14.log" + ); fs.writeFileSync( outputPath, [ @@ -611,7 +622,7 @@ describe("cluster-report", () => { "/home/runner/work/virtualization/virtualization/test/e2e/e2e_test.go:44", "PASS: all 94 specs have precheck labels", " STEP: Ensuring 12 precreated CVIs are available @ 05/14/26 13:57:36.142", - ' CVI "v12n-e2e-testdata-iso" exists but not ready (phase: Pending), waiting...', + " CVI \"v12n-e2e-testdata-iso\" exists but not ready (phase: Pending), waiting...", " [FAILED] in [SynchronizedBeforeSuite] - /home/runner/work/.../until.go:207 @ 05/14/26 14:02:37.61", "[SynchronizedBeforeSuite] [FAILED] [307.964 seconds]", "[SynchronizedBeforeSuite]", @@ -677,14 +688,18 @@ describe("cluster-report", () => { firstReportPath, createGinkgoReport({ startedAt: "2026-04-15T09:30:44Z", - specs: [createSpecReport({ leafNodeText: "old pass", state: "passed" })], + specs: [ + createSpecReport({ leafNodeText: "old pass", state: "passed" }), + ], }) ); fs.writeFileSync( secondReportPath, createGinkgoReport({ startedAt: "2026-04-16T09:30:44Z", - specs: [createSpecReport({ leafNodeText: "latest pass", state: "passed" })], + specs: [ + createSpecReport({ leafNodeText: "latest pass", state: "passed" }), + ], }) ); @@ -706,7 +721,10 @@ describe("cluster-report", () => { test("falls back to missing-report status when raw Ginkgo JSON is invalid", async () => inTempDir(async (tempDir) => { - const rawReportPath = path.join(tempDir, "e2e_report_replicated_2026-04-15.json"); + const rawReportPath = path.join( + tempDir, + "e2e_report_replicated_2026-04-15.json" + ); fs.writeFileSync(rawReportPath, "{not-valid-json"); const reportFile = path.join(tempDir, "report.json"); @@ -743,9 +761,11 @@ describe("cluster-report", () => { reportFile, }); - const writeSpy = jest.spyOn(fs, "writeFileSync").mockImplementation(() => { - throw new Error("disk full"); - }); + const writeSpy = jest + .spyOn(fs, "writeFileSync") + .mockImplementation(() => { + throw new Error("disk full"); + }); try { await expect( @@ -787,7 +807,8 @@ describe("cluster-report", () => { "restores a virtual machine from a snapshot", ], containerHierarchyLabels: [["Slow"], []], - leafNodeText: "BestEffort restore mode; automatic restart approval mode; manual run policy", + leafNodeText: + "BestEffort restore mode; automatic restart approval mode; manual run policy", state: "failed", }) ); From c8f8600f284c271ea4085b38bc4cec0e42af8453 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Mon, 25 May 2026 16:02:54 +0300 Subject: [PATCH 21/24] resovle comments Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .github/workflows/e2e-matrix.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 91f39d6591..e1d06b9013 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -480,7 +480,6 @@ jobs: - uses: actions/checkout@v4 - name: Download E2E report artifacts - # v8 exists upstream and keeps report artifact downloads on the current action major. uses: actions/download-artifact@v8 continue-on-error: true id: download-artifacts-pattern From 19c639c0ee709a9401e7daccf85c1554917c703a Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Tue, 26 May 2026 10:31:26 +0300 Subject: [PATCH 22/24] rm tmp folder Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- tmp/test-ci/report/run-local-report.js | 657 ------------------------- 1 file changed, 657 deletions(-) delete mode 100644 tmp/test-ci/report/run-local-report.js diff --git a/tmp/test-ci/report/run-local-report.js b/tmp/test-ci/report/run-local-report.js deleted file mode 100644 index 8f71dfca3b..0000000000 --- a/tmp/test-ci/report/run-local-report.js +++ /dev/null @@ -1,657 +0,0 @@ -#!/usr/bin/env node - -const fs = require("fs"); -const path = require("path"); -const { spawnSync } = require("child_process"); - -function printUsage() { - console.log(`Usage: - node tmp/test-ci/report/run-local-report.js --json /absolute/path/e2e_report_replicated_2026-05-15.json [options] - node tmp/test-ci/report/run-local-report.js --json /path/replicated.json --json /path/nfs.json [options] - node tmp/test-ci/report/run-local-report.js --json-dir /absolute/path/with/reports [options] - node tmp/test-ci/report/run-local-report.js --cluster replicated --stage configure-sdn [options] - node tmp/test-ci/report/run-local-report.js --spoiler-samples [options] - -Options: - --storage <name> Storage/cluster name for a single JSON without a CI-style file name - --cluster <name> Build a local report entry without JSON input - --branch <name> Branch label in the report. Default: local-test - --run-url <url> URL used in report links. Default: https://example.invalid/local-run - --loop-api-base-url <url> - Optional Loop base URL. Examples: - https://loop.flant.ru - https://loop.flant.ru/api/v4 - --channel-id <id> Optional Loop channel id for API delivery - --token <token> Optional Loop bot token for API delivery - --strict-loop-delivery Fail the local run when Loop delivery fails - --strict-loop-file-upload - Fail the local run when chart upload fails instead of posting without files - --json-dir <path> Load all *.json files from a directory - --xml <path> Backward-compatible alias for --json - --xml-dir <path> Backward-compatible alias for --json-dir - --stage <name> Stage result to emulate. Default: success - Allowed: success, bootstrap, configure-sdn, storage-setup, virtualization-setup, e2e-test - --result <value> Result for the selected stage. Default: failure - Allowed for non-success stages: failure, cancelled, skipped - --out-dir <path> Output directory. Default: tmp/test-ci/report/out - --charts-dir <path> Chart PNG output directory. Default: tmp/charts - --charts-manifest <path> Messenger chart manifest. Default: <work-dir>/messenger-charts/manifest.json - --spoiler-samples Render/send several spoiler/collapsible markdown samples instead of an E2E report - -Examples: - node tmp/test-ci/report/run-local-report.js --json tmp/test-ci/report/e2e_report_replicated_2026-05-15.json - node tmp/test-ci/report/run-local-report.js --json tmp/test-ci/report/e2e_report_replicated_2026-05-15.json --cluster nfs --stage configure-sdn - node tmp/test-ci/report/run-local-report.js --json-dir /tmp/e2e-reports - node tmp/test-ci/report/run-local-report.js --cluster replicated --stage e2e-test --result cancelled - node tmp/test-ci/report/run-local-report.js --json-dir /tmp/e2e-reports --loop-api-base-url "https://loop.flant.ru" --channel-id "$LOOP_CHANNEL_ID" --token "$LOOP_TOKEN" - node tmp/test-ci/report/run-local-report.js --json-dir /tmp/e2e-reports --loop-api-base-url "https://loop.flant.ru" --channel-id "$LOOP_CHANNEL_ID" --token "$LOOP_TOKEN" --strict-loop-delivery --strict-loop-file-upload - node tmp/test-ci/report/run-local-report.js --spoiler-samples --loop-api-base-url "https://loop.flant.ru" --channel-id "$LOOP_CHANNEL_ID" --token "$LOOP_TOKEN" -`); -} - -function parseArgs(argv) { - const args = {}; - - for (let i = 0; i < argv.length; i += 1) { - const token = argv[i]; - if (!token.startsWith("--")) { - continue; - } - - const key = token.slice(2); - const value = argv[i + 1]; - if (!value || value.startsWith("--")) { - args[key] = true; - continue; - } - - if (Object.prototype.hasOwnProperty.call(args, key)) { - if (Array.isArray(args[key])) { - args[key].push(value); - } else { - args[key] = [args[key], value]; - } - } else { - args[key] = value; - } - i += 1; - } - - return args; -} - -function toArray(value) { - if (typeof value === "undefined") { - return []; - } - - return Array.isArray(value) ? value : [value]; -} - -function ensureFileExists(filePath, label) { - if (!filePath || !fs.existsSync(filePath)) { - throw new Error(`${label} does not exist: ${filePath || "<empty>"}`); - } -} - -function mkdirp(dirPath) { - fs.mkdirSync(dirPath, { recursive: true }); -} - -function copyFile(sourcePath, targetPath) { - mkdirp(path.dirname(targetPath)); - fs.copyFileSync(sourcePath, targetPath); -} - -function resetDirectory(dirPath) { - fs.rmSync(dirPath, { recursive: true, force: true }); - mkdirp(dirPath); -} - -function createCore(outputs) { - return { - info(message) { - console.log(`[INFO] ${message}`); - }, - warning(message) { - console.warn(`[WARN] ${message}`); - }, - debug(message) { - console.debug(`[DEBUG] ${message}`); - }, - setOutput(name, value) { - outputs[name] = value; - console.log(`[OUTPUT] ${name}=${value}`); - }, - }; -} - -function buildStageResults(stage, stageResult) { - const stageResults = { - bootstrap: "success", - "configure-sdn": "success", - "storage-setup": "success", - "virtualization-setup": "success", - "e2e-test": "success", - }; - - if (!stage || stage === "success") { - return stageResults; - } - - if (!Object.prototype.hasOwnProperty.call(stageResults, stage)) { - throw new Error(`Unsupported --stage value: ${stage}`); - } - if (!["failure", "cancelled", "skipped"].includes(stageResult)) { - throw new Error( - `Unsupported --result value: ${stageResult}. Allowed: failure, cancelled, skipped` - ); - } - - stageResults[stage] = stageResult; - return stageResults; -} - -function collectJsonFilesFromDir(dirPath) { - if (!fs.existsSync(dirPath)) { - throw new Error(`JSON directory does not exist: ${dirPath}`); - } - - const entries = fs - .readdirSync(dirPath, { withFileTypes: true }) - .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) - .map((entry) => path.join(dirPath, entry.name)) - .sort((left, right) => left.localeCompare(right)); - - if (entries.length === 0) { - throw new Error(`No JSON files found in directory: ${dirPath}`); - } - - return entries; -} - -function deriveStorageType(reportPath, fallbackStorage) { - const baseName = path.basename(reportPath); - const datedMatch = baseName.match( - /^e2e_report_(.+)_(\d{4}-\d{2}-\d{2}.*)\.json$/ - ); - if (datedMatch) { - return datedMatch[1]; - } - - const genericMatch = baseName.match(/^e2e_report_(.+?)_.*\.json$/); - if (genericMatch) { - return genericMatch[1]; - } - - if (fallbackStorage) { - return fallbackStorage; - } - - throw new Error( - `Unable to derive storage type from file name "${baseName}". Use a CI-style name or pass --storage for a single JSON.` - ); -} - -function normalizeLoopApiBaseUrl(value) { - const trimmedValue = String(value || "") - .trim() - .replace(/\/+$/, ""); - - if (!trimmedValue) { - return ""; - } - - if (trimmedValue.endsWith("/api/v4/posts")) { - return trimmedValue; - } - - if (trimmedValue.endsWith("/api/v4")) { - return `${trimmedValue}/posts`; - } - - return `${trimmedValue}/api/v4/posts`; -} - -function buildLoopConfig( - loopApiBaseUrl, - loopChannelId, - loopToken, - options = {} -) { - const apiUrl = normalizeLoopApiBaseUrl(loopApiBaseUrl); - const channelId = String(loopChannelId || "").trim(); - const token = String(loopToken || "").trim(); - - if (!apiUrl && !channelId && !token) { - return null; - } - if (!apiUrl || !channelId || !token) { - throw new Error( - "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" - ); - } - - return { - apiUrl, - channelId, - token, - strictDelivery: Boolean(options.strictDelivery), - strictFileUploads: Boolean(options.strictFileUploads), - }; -} - -function buildSpoilerSampleMessages() { - const longReason = - "Timed out after 300.001s. object v12n-e2e-testdata-iso status.phase is Pending, expected Ready. Expected <string>: Pending to equal <string>: Ready."; - - return { - message: [ - "## :dvp: Spoiler/collapsible markdown test", - "", - "Root message. Replies contain different syntax variants for hiding long `Reason` text.", - ].join("\n"), - threadMessages: [ - { - message: [ - "**Variant 1: HTML `<details>` / `<summary>`**", - "", - "<details>", - "<summary>Show Reason</summary>", - "", - longReason, - "", - "</details>", - ].join("\n"), - files: [], - }, - { - message: [ - "**Variant 2: inline HTML `<details>` inside table cell**", - "", - "| Tests | Reason |", - "|---|---|", - `| SynchronizedBeforeSuite | <details><summary>Show</summary>${longReason}</details> |`, - ].join("\n"), - files: [], - }, - { - message: [ - "**Variant 3: StackExchange-style spoiler blockquote**", - "", - ">! " + longReason, - ].join("\n"), - files: [], - }, - { - message: [ - "**Variant 4: Discord/Telegram-style spoiler delimiters**", - "", - `||${longReason}||`, - ].join("\n"), - files: [], - }, - { - message: [ - "**Variant 5: Mattermost spoiler plugin command text**", - "", - "/spoiler " + longReason, - ].join("\n"), - files: [], - }, - ], - }; -} - -function normalizeThreadMessage(threadMessage) { - return typeof threadMessage === "string" - ? { message: threadMessage, files: [] } - : { - message: String(threadMessage.message || ""), - files: Array.isArray(threadMessage.files) ? threadMessage.files : [], - }; -} - -function writeThreadArtifacts(threadMessages, threadMessageFile, chartDir) { - const normalizedThreadMessages = threadMessages.map(normalizeThreadMessage); - if (normalizedThreadMessages.length === 0) { - return []; - } - - fs.writeFileSync( - threadMessageFile, - `${normalizedThreadMessages - .map((threadMessage) => threadMessage.message) - .join("\n\n---\n\n")}\n` - ); - - const writtenFiles = []; - for (const [ - messageIndex, - threadMessage, - ] of normalizedThreadMessages.entries()) { - for (const file of threadMessage.files) { - if (!file || !file.buffer || !file.name) { - continue; - } - - mkdirp(chartDir); - const targetPath = path.join( - chartDir, - `${messageIndex + 1}-${path.basename(file.name)}` - ); - fs.writeFileSync(targetPath, file.buffer); - writtenFiles.push(targetPath); - } - } - - return writtenFiles; -} - -function generateMessengerCharts({ - repoRoot, - reportsDir, - chartDir, - manifestPath, -}) { - const result = spawnSync( - process.env.PYTHON || "python3", - [ - path.join(repoRoot, ".github/scripts/python/e2e_report/charts.py"), - "messenger-all", - "--reports-dir", - reportsDir, - "--out-dir", - chartDir, - "--manifest", - manifestPath, - ], - { encoding: "utf8" } - ); - - if (result.error) { - throw result.error; - } - if (result.status !== 0) { - const output = (result.stderr || result.stdout || "").trim(); - throw new Error(`Chart rendering failed with status ${result.status}: ${output}`); - } - - return manifestPath; -} - -async function renderSpoilerSamples({ core, outDir, loop }) { - const { makeThreadedReportInLoop } = require(path.join( - path.resolve(__dirname, "../../.."), - ".github/scripts/js/e2e/report/messenger/loop-client" - )); - const { message, threadMessages } = buildSpoilerSampleMessages(); - - resetDirectory(outDir); - fs.writeFileSync( - path.join(outDir, "spoiler-samples-main.md"), - `${message}\n` - ); - fs.writeFileSync( - path.join(outDir, "spoiler-samples-thread.md"), - `${threadMessages - .map((threadMessage) => threadMessage.message) - .join("\n\n---\n\n")}\n` - ); - - core.info(message); - core.setOutput("message", message); - core.setOutput("thread_messages", JSON.stringify(threadMessages)); - - if (loop) { - await makeThreadedReportInLoop({ message, threadMessages, loop }, core); - } - - console.log(""); - console.log("Artifacts written:"); - console.log( - `- Main markdown: ${path.join(outDir, "spoiler-samples-main.md")}` - ); - console.log( - `- Thread markdown: ${path.join(outDir, "spoiler-samples-thread.md")}` - ); - if (loop) { - console.log("- Loop delivery: attempted"); - } -} - -function collectInputEntries(args) { - const jsonArgs = [ - ...toArray(args.json), - ...toArray(args.report), - ...toArray(args.xml), - ].map((value) => path.resolve(String(value))); - const jsonDirArgs = [ - ...toArray(args["json-dir"]), - ...toArray(args["report-dir"]), - ...toArray(args["xml-dir"]), - ].map((value) => path.resolve(String(value))); - const clusterArgs = toArray(args.cluster) - .map((value) => String(value).trim()) - .filter(Boolean); - const dirReports = jsonDirArgs.flatMap((dirPath) => - collectJsonFilesFromDir(dirPath) - ); - const allReports = Array.from(new Set([...jsonArgs, ...dirReports])); - - if (allReports.length === 0 && clusterArgs.length === 0) { - throw new Error( - "Pass at least one --json, one --json-dir, or one --cluster." - ); - } - - allReports.forEach((reportPath) => - ensureFileExists(reportPath, "Ginkgo JSON report") - ); - - const fallbackStorage = args.storage ? String(args.storage) : ""; - const reportEntries = allReports.map((reportPath) => { - const storageType = deriveStorageType( - reportPath, - allReports.length === 1 ? fallbackStorage : "" - ); - return { reportPath, storageType }; - }); - const clusterEntries = clusterArgs.map((storageType) => ({ - reportPath: null, - storageType, - })); - const entries = [...reportEntries, ...clusterEntries]; - - const uniqueStorageTypes = new Set(entries.map((entry) => entry.storageType)); - if (uniqueStorageTypes.size !== entries.length) { - throw new Error( - "Detected duplicate storage types across JSON or cluster inputs. Multiple entries for the same storage are not merged in the local runner." - ); - } - - return entries; -} - -async function main() { - const args = parseArgs(process.argv.slice(2)); - if (args.help) { - printUsage(); - process.exit(0); - } - - const hasInputs = - Boolean(args["spoiler-samples"]) || - toArray(args.json).length > 0 || - toArray(args.report).length > 0 || - toArray(args.xml).length > 0 || - toArray(args["json-dir"]).length > 0 || - toArray(args["report-dir"]).length > 0 || - toArray(args["xml-dir"]).length > 0 || - toArray(args.cluster).length > 0; - if (!hasInputs) { - console.log("No report inputs were provided."); - console.log("Pass at least one --json, one --json-dir, or one --cluster."); - console.log(""); - printUsage(); - return; - } - - const repoRoot = path.resolve(__dirname, "../../.."); - const clusterReport = require(path.join( - repoRoot, - ".github/scripts/js/e2e/report/cluster-report" - )); - const messengerReport = require(path.join( - repoRoot, - ".github/scripts/js/e2e/report/messenger-report" - )); - const branch = String(args.branch || "local-test"); - const runUrl = String(args["run-url"] || "https://example.invalid/local-run"); - const outDir = path.resolve( - String(args["out-dir"] || path.join(repoRoot, "tmp/test-ci/report/out")) - ); - const chartDir = path.resolve( - String(args["charts-dir"] || path.join(repoRoot, "tmp/charts")) - ); - const workDir = path.resolve( - String(args["work-dir"] || path.join(path.dirname(outDir), "work")) - ); - const messengerChartDir = path.join(workDir, "messenger-charts"); - const chartsManifest = path.resolve( - String( - args["charts-manifest"] || path.join(messengerChartDir, "manifest.json") - ) - ); - const messageFile = path.join(outDir, "messenger-report.md"); - const threadMessageFile = path.join(outDir, "messenger-thread.md"); - const loopApiBaseUrl = args["loop-api-base-url"] - ? String(args["loop-api-base-url"]) - : String(process.env.LOOP_API_BASE_URL || ""); - const loopChannelId = args["channel-id"] - ? String(args["channel-id"]) - : String(process.env.LOOP_CHANNEL_ID || ""); - const loopToken = args.token - ? String(args.token) - : String(process.env.LOOP_TOKEN || ""); - const stage = String(args.stage || "success"); - const stageResult = String(args.result || "failure"); - const strictLoopDelivery = Boolean(args["strict-loop-delivery"]); - const strictLoopFileUpload = Boolean(args["strict-loop-file-upload"]); - const loop = buildLoopConfig(loopApiBaseUrl, loopChannelId, loopToken, { - strictDelivery: strictLoopDelivery, - strictFileUploads: strictLoopFileUpload, - }); - - if (args["spoiler-samples"]) { - const outputs = {}; - await renderSpoilerSamples({ core: createCore(outputs), outDir, loop }); - return; - } - - const inputEntries = collectInputEntries(args); - const stageResults = buildStageResults(stage, stageResult); - - resetDirectory(outDir); - resetDirectory(chartDir); - resetDirectory(workDir); - - process.env.REPORTS_DIR = outDir; - process.env.EXPECTED_STORAGE_TYPES = JSON.stringify( - inputEntries.map((entry) => entry.storageType) - ); - process.env.LOOP_API_BASE_URL = loopApiBaseUrl; - process.env.LOOP_CHANNEL_ID = loopChannelId; - process.env.LOOP_TOKEN = loopToken; - process.env.LOOP_STRICT_DELIVERY = strictLoopDelivery ? "1" : ""; - process.env.LOOP_STRICT_FILE_UPLOAD = strictLoopFileUpload ? "1" : ""; - process.env.CHARTS_MANIFEST = chartsManifest; - - const outputs = {}; - const core = createCore(outputs); - const context = { - serverUrl: "https://github.com", - repo: { owner: "local", repo: "virtualization2" }, - runId: "local-test", - ref: `refs/heads/${branch}`, - }; - - const generatedReports = []; - for (const entry of inputEntries) { - const reportFile = path.join( - outDir, - `e2e_report_${entry.storageType}.json` - ); - const rawReportPath = entry.reportPath - ? path.join(workDir, `e2e_report_${entry.storageType}_local.json`) - : null; - - if (entry.reportPath) { - copyFile(entry.reportPath, rawReportPath); - } - - await clusterReport({ - core, - context, - config: { - storageType: entry.storageType, - pipelineJobName: `Local E2E (${entry.storageType})`, - reportsDir: workDir, - reportFile, - stageResults, - stageJobUrls: { - bootstrap: runUrl, - "configure-sdn": runUrl, - "storage-setup": runUrl, - "virtualization-setup": runUrl, - "e2e-test": runUrl, - }, - }, - }); - generatedReports.push(reportFile); - } - - generateMessengerCharts({ - repoRoot, - reportsDir: outDir, - chartDir: messengerChartDir, - manifestPath: chartsManifest, - }); - - const renderedMessages = await messengerReport({ core }); - const message = renderedMessages.message; - const threadMessages = renderedMessages.threadMessages || []; - - fs.writeFileSync(messageFile, `${message}\n`); - const chartFiles = writeThreadArtifacts( - threadMessages, - threadMessageFile, - chartDir - ); - - console.log(""); - console.log("Artifacts written:"); - generatedReports.forEach((reportFile) => { - console.log(`- JSON: ${reportFile}`); - }); - console.log(`- Main markdown: ${messageFile}`); - if (threadMessages.length > 0) { - console.log(`- Thread markdown: ${threadMessageFile}`); - } - console.log(`- Chart manifest: ${chartsManifest}`); - chartFiles.forEach((chartFile) => { - console.log(`- Chart: ${chartFile}`); - }); - - if (loopApiBaseUrl || loopChannelId || loopToken) { - console.log("- Loop delivery: attempted"); - } -} - -main().catch((error) => { - console.error(`[ERROR] ${error.message}`); - if (process.env.DEBUG_LOCAL_REPORT === "1" && error.stack) { - console.error(error.stack); - } - process.exit(1); -}); From 1d6d2422908227ad5d1576e950769d58ecf11b53 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Tue, 26 May 2026 10:44:21 +0300 Subject: [PATCH 23/24] resolve comments Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .github/scripts/js/e2e/report/messenger/loop-client.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index afbefe7cdf..04db49f0ea 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -191,13 +191,17 @@ async function makeThreadedReportInLoop( .map((result) => result.value); const failures = results.filter((result) => result.status === "rejected"); - for (const failure of failures) { + const failureDetails = failures.map((failure) => { const reason = failure.reason; - const details = reason && reason.message ? reason.message : reason; + return reason && reason.message ? reason.message : String(reason); + }); + for (const details of failureDetails) { core.warning(`Loop file upload failed for one attachment: ${details}`); } if (loop.strictFileUploads && failures.length > 0) { - throw new Error("Strict file uploads enabled; at least one attachment failed"); + throw new Error( + `Strict file uploads enabled; at least one attachment failed: ${failureDetails.join("; ")}` + ); } } await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds, { From 46f806ff0b32ad1747c4db96cac51be44bff9095 Mon Sep 17 00:00:00 2001 From: Nikita Korolev <nikita.korolev@flant.com> Date: Tue, 26 May 2026 13:38:10 +0300 Subject: [PATCH 24/24] fix cunstruct url Signed-off-by: Nikita Korolev <nikita.korolev@flant.com> --- .../js/e2e/report/messenger-report.test.js | 3 +- .../scripts/js/e2e/report/messenger/config.js | 66 +++++++++++++------ .../js/e2e/report/messenger/loop-client.js | 11 ++-- .../e2e/report/messenger/loop-client.test.js | 49 ++++++-------- 4 files changed, 72 insertions(+), 57 deletions(-) diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 69f93f13a3..4b7de87e1c 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -50,7 +50,8 @@ describe("messenger-report", () => { reportsDir: "custom-reports", configuredClusters: ["replicated", "nfs"], loop: { - apiUrl: "https://loop.example.invalid/api/v4/posts", + postsApiUrl: "https://loop.example.invalid/api/v4/posts", + filesApiUrl: "https://loop.example.invalid/api/v4/files", channelId: "channel-id", token: "token", strictDelivery: false, diff --git a/.github/scripts/js/e2e/report/messenger/config.js b/.github/scripts/js/e2e/report/messenger/config.js index bc163d329c..491cb024a3 100644 --- a/.github/scripts/js/e2e/report/messenger/config.js +++ b/.github/scripts/js/e2e/report/messenger/config.js @@ -10,30 +10,55 @@ // See the License for the specific language governing permissions and // limitations under the License. +const loopApiRootSegments = ["api", "v4"]; + +function getUrlPathSegments(url) { + return url.pathname.split("/").filter(Boolean); +} + +function buildUrlPath(segments) { + return `/${segments.join("/")}`; +} + +function findLoopApiRootIndex(segments) { + return segments.findIndex( + (segment, index) => + segment === loopApiRootSegments[0] && + segments[index + 1] === loopApiRootSegments[1] + ); +} + +function buildLoopEndpointUrl(apiBaseUrl, endpoint) { + const url = new URL(apiBaseUrl); + url.pathname = buildUrlPath([...getUrlPathSegments(url), endpoint]); + return url.toString(); +} + /** - * Normalizes the configured Loop API base URL to the `/api/v4/posts` endpoint. + * Normalizes the configured Loop API base URL to the `/api/v4` root. * * @param {string} value Raw Loop API base URL. - * @returns {string} Normalized posts endpoint URL or an empty string. + * @returns {string} Normalized API root URL or an empty string. */ function normalizeLoopApiBaseUrl(value) { - const trimmedValue = String(value || "") - .trim() - .replace(/\/+$/, ""); + const rawValue = String(value || "").trim(); - if (!trimmedValue) { + if (!rawValue) { return ""; } - if (trimmedValue.endsWith("/api/v4/posts")) { - return trimmedValue; - } - - if (trimmedValue.endsWith("/api/v4")) { - return `${trimmedValue}/posts`; - } + const url = new URL(rawValue); + const pathSegments = getUrlPathSegments(url); + const apiRootIndex = findLoopApiRootIndex(pathSegments); + const apiRootSegments = + apiRootIndex === -1 + ? [...pathSegments, ...loopApiRootSegments] + : pathSegments.slice(0, apiRootIndex + loopApiRootSegments.length); - return `${trimmedValue}/api/v4/posts`; + url.pathname = buildUrlPath(apiRootSegments); + url.search = ""; + url.hash = ""; + return url.toString(); } // Fallback used only when EXPECTED_STORAGE_TYPES is not set (e.g. local runs or tests). @@ -70,21 +95,22 @@ function parseBooleanEnv(value) { * mistake and should surface as an error rather than a silent no-op. * * @param {NodeJS.ProcessEnv} [env=process.env] Environment variables source. - * @returns {{ apiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null} + * @returns {{ postsApiUrl: string, filesApiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null} */ function readLoopConfig(env = process.env) { - const apiUrl = normalizeLoopApiBaseUrl(env.LOOP_API_BASE_URL); + const apiBaseUrl = normalizeLoopApiBaseUrl(env.LOOP_API_BASE_URL); const channelId = String(env.LOOP_CHANNEL_ID || "").trim(); const token = String(env.LOOP_TOKEN || "").trim(); - if (!apiUrl && !channelId && !token) { + if (!apiBaseUrl && !channelId && !token) { return null; } - if (!apiUrl || !channelId || !token) { + if (!apiBaseUrl || !channelId || !token) { throw new Error("LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required"); } return { - apiUrl, + postsApiUrl: buildLoopEndpointUrl(apiBaseUrl, "posts"), + filesApiUrl: buildLoopEndpointUrl(apiBaseUrl, "files"), channelId, token, strictDelivery: parseBooleanEnv(env.LOOP_STRICT_DELIVERY), @@ -99,7 +125,7 @@ function readLoopConfig(env = process.env) { * @returns {{ * reportsDir: string, * configuredClusters: string[], - * loop: { apiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null + * loop: { postsApiUrl: string, filesApiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null * }} Normalized messenger configuration. */ function readMessengerConfigFromEnv(env = process.env) { diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 04db49f0ea..9e96c166c9 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -18,7 +18,8 @@ /** * @typedef {Object} LoopCredentials - * @property {string} apiUrl + * @property {string} postsApiUrl + * @property {string} filesApiUrl * @property {string} channelId * @property {string} token */ @@ -79,7 +80,7 @@ async function postToLoopApi( ...(fileIds.length > 0 ? { file_ids: fileIds } : {}), }; - const response = await fetchFn(loop.apiUrl, { + const response = await fetchFn(loop.postsApiUrl, { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, @@ -100,10 +101,6 @@ async function postToLoopApi( return payload; } -function getFilesApiUrl(apiUrl) { - return String(apiUrl || "").replace(/\/posts$/, "/files"); -} - /** * Uploads a single file to Loop and returns the created file id. * @@ -127,7 +124,7 @@ async function uploadFileToLoop( formData.append("channel_id", loop.channelId); formData.append("files", new Blob([buffer], { type: mimeType }), fileName); - const response = await fetchFn(getFilesApiUrl(loop.apiUrl), { + const response = await fetchFn(loop.filesApiUrl, { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js index 14b57ae2e1..5b3a9be878 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.test.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -13,6 +13,16 @@ const { uploadFileToLoop, makeThreadedReportInLoop } = require("./loop-client"); const { createCore } = require("../shared/test-utils"); +function createLoop(overrides = {}) { + return { + postsApiUrl: "https://loop.example.invalid/api/v4/posts", + filesApiUrl: "https://loop.example.invalid/api/v4/files", + channelId: "channel-id", + token: "loop-token", + ...overrides, + }; +} + describe("loop-client", () => { afterEach(() => { delete global.fetch; @@ -26,11 +36,7 @@ describe("loop-client", () => { }); const fileId = await uploadFileToLoop( - { - apiUrl: "https://loop.example.invalid/api/v4/posts", - channelId: "channel-id", - token: "loop-token", - }, + createLoop(), "chart.png", Buffer.from("image-bytes"), createCore(), @@ -55,11 +61,7 @@ describe("loop-client", () => { }); test("posts the reply with uploaded chart file ids", async () => { - const loop = { - apiUrl: "https://loop.example.invalid/api/v4/posts", - channelId: "channel-id", - token: "loop-token", - }; + const loop = createLoop(); const responses = [ { ok: true, @@ -119,11 +121,7 @@ describe("loop-client", () => { }); test("posts the reply with successful attachments when one upload fails", async () => { - const loop = { - apiUrl: "https://loop.example.invalid/api/v4/posts", - channelId: "channel-id", - token: "loop-token", - }; + const loop = createLoop(); const core = createCore(); const responses = [ { @@ -179,10 +177,10 @@ describe("loop-client", () => { ); expect(global.fetch).toHaveBeenCalledTimes(4); - expect(global.fetch.mock.calls[0][0]).toBe(loop.apiUrl); - expect(global.fetch.mock.calls[1][0]).toBe("https://loop.example.invalid/api/v4/files"); - expect(global.fetch.mock.calls[2][0]).toBe("https://loop.example.invalid/api/v4/files"); - expect(global.fetch.mock.calls[3][0]).toBe(loop.apiUrl); + expect(global.fetch.mock.calls[0][0]).toBe(loop.postsApiUrl); + expect(global.fetch.mock.calls[1][0]).toBe(loop.filesApiUrl); + expect(global.fetch.mock.calls[2][0]).toBe(loop.filesApiUrl); + expect(global.fetch.mock.calls[3][0]).toBe(loop.postsApiUrl); const replyBody = JSON.parse(global.fetch.mock.calls[3][1].body); expect(replyBody.root_id).toBe("root-post-id"); @@ -198,12 +196,9 @@ describe("loop-client", () => { }); test("fails when strict file upload mode is enabled", async () => { - const loop = { - apiUrl: "https://loop.example.invalid/api/v4/posts", - channelId: "channel-id", - token: "loop-token", + const loop = createLoop({ strictFileUploads: true, - }; + }); const responses = [ { ok: true, @@ -246,11 +241,7 @@ describe("loop-client", () => { test("uses injected fetch without touching the global fetch", async () => { const originalFetch = globalThis.fetch; - const loop = { - apiUrl: "https://loop.example.invalid/api/v4/posts", - channelId: "channel-id", - token: "loop-token", - }; + const loop = createLoop(); const responses = [ { ok: true,