diff --git a/.github/scripts/js/.prettierrc b/.github/scripts/js/.prettierrc new file mode 100644 index 0000000000..963354f231 --- /dev/null +++ b/.github/scripts/js/.prettierrc @@ -0,0 +1,3 @@ +{ + "printWidth": 120 +} diff --git a/.github/scripts/js/e2e/report/cluster-report.js b/.github/scripts/js/e2e/report/cluster-report.js index e1df27c7d3..8a357d13ac 100644 --- a/.github/scripts/js/e2e/report/cluster-report.js +++ b/.github/scripts/js/e2e/report/cluster-report.js @@ -178,6 +178,8 @@ async function readStageJobUrlsFromApi(github, context, config, core) { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: null, * source: string, * }} Empty parsed-report payload. @@ -187,6 +189,8 @@ function emptyParsedReport(source) { metrics: zeroMetrics(), failedTests: [], failedTestDetails: [], + specTimings: [], + suiteTotalMs: 0, startedAt: null, source, }; @@ -217,6 +221,8 @@ const ginkgoOutputSource = { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: string|null, * }} parse Parser function for the source content. * @property {function(string): RegExp} pattern Builds the file-name regex for the source. @@ -252,6 +258,8 @@ function findGinkgoSource(config, source) { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: string|null, * source: string, * }} Parsed report payload with a source tag. @@ -320,6 +328,8 @@ function buildReportPayload({ metrics: parsedReport.metrics, failedTests: parsedReport.failedTests, failedTestDetails: parsedReport.failedTestDetails, + specTimings: parsedReport.specTimings || [], + suiteTotalMs: parsedReport.suiteTotalMs || 0, sourceReport: sourcePath, reportSource: parsedReport.source, }; diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index c8bbd2b34f..272b5c95bd 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -454,11 +454,43 @@ describe("cluster-report", () => { reason: "timed out waiting for VM to become ready", }, ]); + expect(report.suiteTotalMs).toBe(1800000); + expect(report.specTimings).toEqual([ + { + name: "passes", + group: "Suite", + state: "passed", + runtimeMs: 60000, + labels: [], + }, + { + name: "fails & burns", + group: "Suite", + state: "failed", + runtimeMs: 60000, + labels: ["Slow"], + }, + { + name: "errors ", + group: "Other", + state: "errors", + runtimeMs: 60000, + labels: [], + }, + { + name: "skipped", + group: "Top-level Its", + state: "skipped", + runtimeMs: 60000, + labels: [], + }, + ]); expect(report.reportSource).toBe("ginkgo-json"); expect(report.sourceReport).toBe(rawReportPath); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).reportKind).toBe( - "tests" - ); + const persistedReport = JSON.parse(fs.readFileSync(reportFile, "utf8")); + expect(persistedReport.reportKind).toBe("tests"); + expect(persistedReport.specTimings).toEqual(report.specTimings); + expect(persistedReport.suiteTotalMs).toBe(1800000); expect(core.setOutput).toHaveBeenCalledWith("report_file", reportFile); expect(core.setOutput).toHaveBeenCalledWith("report_kind", "tests"); expect(core.setOutput).toHaveBeenCalledWith("status", "failure"); @@ -828,6 +860,8 @@ describe("cluster-report", () => { successRate: 92.78, }); expect(parsed.startedAt).toBe("2026-04-28T03:11:27.708387575Z"); + expect(parsed.specTimings).toHaveLength(131); + expect(parsed.suiteTotalMs).toBe(1800000); expect(parsed.failedTests).toHaveLength(7); expect(parsed.failedTests).toContain( "[It] VirtualMachineOperationRestore restores a virtual machine from a snapshot BestEffort restore mode; automatic restart approval mode; manual run policy [Slow]" diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index fb2c6906b2..f301c6f809 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -14,6 +14,7 @@ const fs = require("fs"); const { listMatchingFiles } = require("./shared/fs-utils"); const { REPORT_FILE_PATTERN } = require("./shared/report-model"); +const { getClusterChartFiles } = require("./messenger/chart-files"); const { makeThreadedReportInLoop } = require("./messenger/loop-client"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { @@ -103,12 +104,15 @@ function readReports(reportsDir, configuredClusters, core) { * @param {MessengerMessagesParams} params Message rendering inputs. * @returns {{ * message: string, - * threadMessages: string[] + * threadMessages: Array<{message: string, files: Array>}> * }} Rendered markdown payloads. */ -function buildMessengerMessages({ reportsDir, configuredClusters, core }) { +async function buildMessengerMessages({ reportsDir, configuredClusters, core }) { const orderedReports = readReports(reportsDir, configuredClusters, core); - const threadMessages = buildThreadMessages(orderedReports); + const threadMessages = await buildThreadMessages(orderedReports, { + getClusterChartFiles, + core, + }); return { message: buildMainMessage(orderedReports), threadMessages, @@ -122,12 +126,12 @@ function buildMessengerMessages({ reportsDir, configuredClusters, core }) { * @param {RenderMessengerReportParams} params GitHub script dependencies. * @returns {Promise<{ * message: string, - * threadMessages: string[] + * threadMessages: Array<{message: string, files: Array>}> * }>} Rendered messages. */ async function renderMessengerReport({ core, reportsDir }) { const config = readMessengerConfigFromEnv(); - const { message, threadMessages } = buildMessengerMessages({ + const { message, threadMessages } = await buildMessengerMessages({ reportsDir: reportsDir || config.reportsDir, configuredClusters: config.configuredClusters, core, @@ -135,13 +139,19 @@ async function renderMessengerReport({ core, reportsDir }) { core.info(message); core.setOutput("message", message); - core.setOutput("thread_messages", JSON.stringify(threadMessages)); + core.setOutput( + "thread_messages", + JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message)) + ); if (config.loop) { try { await makeThreadedReportInLoop({ message, threadMessages, loop: config.loop }, core); } catch (error) { core.warning(`Unable to deliver report to Loop API: ${error.message}`); + if (config.loop.strictDelivery) { + throw error; + } } } diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 429a7c4f58..4b7de87e1c 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -13,7 +13,12 @@ const fs = require("fs"); const path = require("path"); +jest.mock("./messenger/chart-files", () => ({ + getClusterChartFiles: jest.fn().mockResolvedValue([]), +})); + const renderMessengerReport = require("./messenger-report"); +const { getClusterChartFiles } = require("./messenger/chart-files"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { createCore, withTempDir } = require("./shared/test-utils"); @@ -26,7 +31,11 @@ describe("messenger-report", () => { delete process.env.LOOP_API_BASE_URL; delete process.env.LOOP_CHANNEL_ID; delete process.env.LOOP_TOKEN; + delete process.env.LOOP_STRICT_DELIVERY; + delete process.env.LOOP_STRICT_FILE_UPLOAD; delete global.fetch; + getClusterChartFiles.mockReset(); + getClusterChartFiles.mockResolvedValue([]); }); test("reads normalized messenger config from env", () => { @@ -41,9 +50,12 @@ describe("messenger-report", () => { reportsDir: "custom-reports", configuredClusters: ["replicated", "nfs"], loop: { - apiUrl: "https://loop.example.invalid/api/v4/posts", + postsApiUrl: "https://loop.example.invalid/api/v4/posts", + filesApiUrl: "https://loop.example.invalid/api/v4/files", channelId: "channel-id", token: "token", + strictDelivery: false, + strictFileUploads: false, }, }); }); @@ -60,9 +72,7 @@ describe("messenger-report", () => { LOOP_API_BASE_URL: "https://loop.example.invalid", // LOOP_CHANNEL_ID and LOOP_TOKEN intentionally absent }) - ).toThrow( - "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" - ); + ).toThrow("LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required"); }); test("uses default configured clusters when env override is absent", () => { @@ -133,20 +143,22 @@ describe("messenger-report", () => { ); expect(result.message).not.toContain("⚠️ Errors"); expect(result.message).toContain("### Cluster failures"); - expect(result.message).toContain( - "- [nfs](https://example.invalid/nfs): CONFIGURE SDN" - ); + expect(result.message).toContain("- [nfs](https://example.invalid/nfs): CONFIGURE SDN"); + expect(result.message).not.toContain("### Top slowest tests"); expect(result.message).not.toContain("### Failed tests"); expect(result.threadMessages).toEqual([ - [ - "### Failed tests", - "", - "**[replicated](https://example.invalid/replicated)**", - "", - "| Tests | Reason |", - "|---|---|", - "| fails | Unexpected error: command timed out occurred |", - ].join("\n"), + { + message: [ + "### Failed tests", + "", + "**[replicated](https://example.invalid/replicated)**", + "", + "| Tests | Reason |", + "|---|---|", + "| fails | Unexpected error: command timed out occurred |", + ].join("\n"), + files: [], + }, ]); })); @@ -158,12 +170,112 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.message).toContain("### Missing reports"); - expect(result.message).toContain( - "- replicated: ⚠️ E2E REPORT ARTIFACT NOT FOUND" - ); + expect(result.message).toContain("- replicated: ⚠️ E2E REPORT ARTIFACT NOT FOUND"); expect(result.threadMessages).toEqual([]); })); + test("attaches duration chart files to thread reply without a text caption", async () => + inTempDir(async (tempDir) => { + const chartFile = { + name: "replicated-feature-duration-status.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }; + getClusterChartFiles.mockResolvedValue([chartFile]); + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 3, + skipped: 0, + failed: 0, + errors: 0, + total: 3, + successRate: 100, + }, + failedTests: [], + specTimings: [ + { name: "fast", group: "VM", state: "passed", runtimeMs: 1000 }, + { + name: "slow | pipe", + group: "Disk", + state: "passed", + runtimeMs: 90000, + }, + { name: "medium", group: "VM", state: "passed", runtimeMs: 30000 }, + ], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + + const core = createCore(); + const result = await renderMessengerReport({ core }); + + expect(result.message).not.toContain("### Top slowest tests"); + expect(result.threadMessages).toEqual([ + { + message: "**[replicated](https://example.invalid/replicated)**", + files: [chartFile], + }, + ]); + expect(result.threadMessages[0].message).not.toContain("### Test durations"); + expect(result.threadMessages[0].message).not.toContain("Attached charts:"); + expect(core.setOutput).toHaveBeenCalledWith( + "thread_messages", + JSON.stringify([result.threadMessages[0].message]) + ); + })); + + test("warns and surfaces a placeholder when chart files are unavailable", async () => + inTempDir(async (tempDir) => { + getClusterChartFiles.mockRejectedValue(new Error("chart cli unavailable")); + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 1, + skipped: 0, + failed: 0, + errors: 0, + total: 1, + successRate: 100, + }, + failedTests: [], + specTimings: [{ name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + + const core = createCore(); + const result = await renderMessengerReport({ core }); + + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining("Unable to prepare duration chart files for cluster replicated") + ); + expect(result.threadMessages).toEqual([ + { + message: expect.stringContaining("Charts unavailable."), + files: [], + }, + ]); + })); + test("warns and skips report files that are missing storageType/cluster fields", async () => inTempDir(async (tempDir) => { fs.writeFileSync( @@ -264,8 +376,16 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.threadMessages).toEqual([ - "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| replicated | — |", - "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + { + message: + "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| replicated | — |", + files: [], + }, + { + message: + "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + files: [], + }, ]); })); @@ -306,16 +426,19 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.threadMessages).toEqual([ - [ - "### Failed tests", - "", - "**[nfs](https://example.invalid/nfs)**", - "", - "| Tests | Reason |", - "|---|---|", - "| VirtualMachineOperationRestore | — |", - "| VirtualMachineAdditionalNetworkInterfaces | — |", - ].join("\n"), + { + message: [ + "### Failed tests", + "", + "**[nfs](https://example.invalid/nfs)**", + "", + "| Tests | Reason |", + "|---|---|", + "| VirtualMachineOperationRestore | — |", + "| VirtualMachineAdditionalNetworkInterfaces | — |", + ].join("\n"), + files: [], + }, ]); })); @@ -338,8 +461,7 @@ describe("messenger-report", () => { testStatus: { status: "not-run", reason: "cluster-stage-failure", - message: - "E2E tests were not run because cluster setup did not finish", + message: "E2E tests were not run because cluster setup did not finish", }, metrics: { passed: 0, @@ -382,8 +504,7 @@ describe("messenger-report", () => { testStatus: { status: "not-run", reason: "cluster-stage-failure", - message: - "E2E tests were not run because cluster setup did not finish", + message: "E2E tests were not run because cluster setup did not finish", }, metrics: { passed: 0, @@ -448,6 +569,12 @@ describe("messenger-report", () => { test("posts main report and per-cluster failed tests thread via Loop API", async () => inTempDir(async (tempDir) => { + const chartFile = { + name: "replicated-feature-duration-status.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }; + getClusterChartFiles.mockResolvedValue([chartFile]); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ @@ -466,6 +593,7 @@ describe("messenger-report", () => { successRate: 83.33, }, failedTests: ["[It] fails"], + specTimings: [{ name: "slow", group: "VM", state: "failed", runtimeMs: 90000 }], }) ); @@ -482,6 +610,11 @@ describe("messenger-report", () => { status: 201, text: async () => JSON.stringify({ id: "root-post-id" }), }) + .mockResolvedValueOnce({ + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-id" }] }), + }) .mockResolvedValueOnce({ ok: true, status: 201, @@ -490,7 +623,7 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); - expect(global.fetch).toHaveBeenCalledTimes(2); + expect(global.fetch).toHaveBeenCalledTimes(3); expect(global.fetch).toHaveBeenNthCalledWith( 1, "https://loop.example.invalid/api/v4/posts", @@ -506,11 +639,29 @@ describe("messenger-report", () => { channel_id: "channel-id", message: result.message, }); - expect(JSON.parse(global.fetch.mock.calls[1][1].body)).toEqual({ + expect(global.fetch).toHaveBeenNthCalledWith( + 2, + "https://loop.example.invalid/api/v4/files", + expect.objectContaining({ + method: "POST", + headers: { + Authorization: "Bearer loop-token", + }, + }) + ); + expect(JSON.parse(global.fetch.mock.calls[2][1].body)).toEqual({ channel_id: "channel-id", - message: - "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| fails | — |", + message: [ + "### Failed tests", + "", + "**[replicated](https://example.invalid/replicated)**", + "", + "| Tests | Reason |", + "|---|---|", + "| fails | — |", + ].join("\n"), root_id: "root-post-id", + file_ids: ["file-id"], }); })); @@ -654,4 +805,45 @@ describe("messenger-report", () => { "Unable to deliver report to Loop API: Loop API request failed with status 500: server exploded" ); })); + + test("fails local delivery when strict Loop delivery mode is enabled", async () => + inTempDir(async (tempDir) => { + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 11, + skipped: 0, + failed: 0, + errors: 0, + total: 11, + successRate: 100, + }, + failedTests: [], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + process.env.LOOP_API_BASE_URL = "https://loop.example.invalid"; + process.env.LOOP_CHANNEL_ID = "channel-id"; + process.env.LOOP_TOKEN = "loop-token"; + process.env.LOOP_STRICT_DELIVERY = "1"; + + global.fetch = jest.fn().mockResolvedValue({ + ok: false, + status: 500, + text: async () => "server exploded", + }); + + await expect(renderMessengerReport({ core: createCore() })).rejects.toThrow( + "Loop API request failed with status 500: server exploded" + ); + })); }); diff --git a/.github/scripts/js/e2e/report/messenger/chart-files.js b/.github/scripts/js/e2e/report/messenger/chart-files.js new file mode 100644 index 0000000000..02ba14913c --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/chart-files.js @@ -0,0 +1,47 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); +const path = require("path"); + +const { getReportClusterKey } = require("./model"); + +const defaultManifestPath = "tmp/messenger-charts/manifest.json"; + +function readChartManifest(manifestPath) { + if (!fs.existsSync(manifestPath)) { + return { clusters: {} }; + } + + return JSON.parse(fs.readFileSync(manifestPath, "utf8")); +} + +function getClusterChartFiles(report) { + const clusterKey = getReportClusterKey(report); + if (!clusterKey) { + return []; + } + + const manifestPath = process.env.CHARTS_MANIFEST || defaultManifestPath; + const manifest = readChartManifest(manifestPath); + const files = ((manifest.clusters || {})[clusterKey] || []).map((file) => ({ + name: file.name, + buffer: fs.readFileSync(path.resolve(file.path)), + mimeType: file.mimeType || "image/png", + })); + + return files; +} + +module.exports = { + getClusterChartFiles, +}; diff --git a/.github/scripts/js/e2e/report/messenger/chart-files.test.js b/.github/scripts/js/e2e/report/messenger/chart-files.test.js new file mode 100644 index 0000000000..a68e366329 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/chart-files.test.js @@ -0,0 +1,55 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require("fs"); +const path = require("path"); + +const { getClusterChartFiles } = require("./chart-files"); +const { withTempDir } = require("../shared/test-utils"); + +describe("chart-files", () => { + afterEach(() => { + delete process.env.CHARTS_MANIFEST; + }); + + test("returns no files when the manifest is missing", () => { + expect(getClusterChartFiles({ cluster: "replicated" })).toEqual([]); + }); + + test("loads chart files listed in the Python manifest", async () => + withTempDir("chart-files", async (tempDir) => { + const chartPath = path.join(tempDir, "replicated-feature-duration-status.png"); + const manifestPath = path.join(tempDir, "manifest.json"); + fs.writeFileSync(chartPath, Buffer.from("png")); + fs.writeFileSync( + manifestPath, + JSON.stringify({ + clusters: { + replicated: [ + { + name: "replicated-feature-duration-status.png", + path: chartPath, + mimeType: "image/png", + }, + ], + }, + }) + ); + process.env.CHARTS_MANIFEST = manifestPath; + + const files = await getClusterChartFiles({ cluster: "replicated" }); + + expect(files.map(({ name }) => name)).toEqual(["replicated-feature-duration-status.png"]); + expect(files[0].buffer).toEqual(Buffer.from("png")); + expect(files[0].mimeType).toBe("image/png"); + })); +}); diff --git a/.github/scripts/js/e2e/report/messenger/config.js b/.github/scripts/js/e2e/report/messenger/config.js index 12fa7dc659..491cb024a3 100644 --- a/.github/scripts/js/e2e/report/messenger/config.js +++ b/.github/scripts/js/e2e/report/messenger/config.js @@ -10,30 +10,55 @@ // See the License for the specific language governing permissions and // limitations under the License. +const loopApiRootSegments = ["api", "v4"]; + +function getUrlPathSegments(url) { + return url.pathname.split("/").filter(Boolean); +} + +function buildUrlPath(segments) { + return `/${segments.join("/")}`; +} + +function findLoopApiRootIndex(segments) { + return segments.findIndex( + (segment, index) => + segment === loopApiRootSegments[0] && + segments[index + 1] === loopApiRootSegments[1] + ); +} + +function buildLoopEndpointUrl(apiBaseUrl, endpoint) { + const url = new URL(apiBaseUrl); + url.pathname = buildUrlPath([...getUrlPathSegments(url), endpoint]); + return url.toString(); +} + /** - * Normalizes the configured Loop API base URL to the `/api/v4/posts` endpoint. + * Normalizes the configured Loop API base URL to the `/api/v4` root. * * @param {string} value Raw Loop API base URL. - * @returns {string} Normalized posts endpoint URL or an empty string. + * @returns {string} Normalized API root URL or an empty string. */ function normalizeLoopApiBaseUrl(value) { - const trimmedValue = String(value || "") - .trim() - .replace(/\/+$/, ""); + const rawValue = String(value || "").trim(); - if (!trimmedValue) { + if (!rawValue) { return ""; } - if (trimmedValue.endsWith("/api/v4/posts")) { - return trimmedValue; - } - - if (trimmedValue.endsWith("/api/v4")) { - return `${trimmedValue}/posts`; - } + const url = new URL(rawValue); + const pathSegments = getUrlPathSegments(url); + const apiRootIndex = findLoopApiRootIndex(pathSegments); + const apiRootSegments = + apiRootIndex === -1 + ? [...pathSegments, ...loopApiRootSegments] + : pathSegments.slice(0, apiRootIndex + loopApiRootSegments.length); - return `${trimmedValue}/api/v4/posts`; + url.pathname = buildUrlPath(apiRootSegments); + url.search = ""; + url.hash = ""; + return url.toString(); } // Fallback used only when EXPECTED_STORAGE_TYPES is not set (e.g. local runs or tests). @@ -57,6 +82,10 @@ function parseConfiguredClusters(value) { } } +function parseBooleanEnv(value) { + return ["1", "true", "yes"].includes(String(value || "").toLowerCase()); +} + /** * Reads Loop credentials from the environment. * @@ -66,22 +95,27 @@ function parseConfiguredClusters(value) { * mistake and should surface as an error rather than a silent no-op. * * @param {NodeJS.ProcessEnv} [env=process.env] Environment variables source. - * @returns {{ apiUrl: string, channelId: string, token: string } | null} + * @returns {{ postsApiUrl: string, filesApiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null} */ function readLoopConfig(env = process.env) { - const apiUrl = normalizeLoopApiBaseUrl(env.LOOP_API_BASE_URL); + const apiBaseUrl = normalizeLoopApiBaseUrl(env.LOOP_API_BASE_URL); const channelId = String(env.LOOP_CHANNEL_ID || "").trim(); const token = String(env.LOOP_TOKEN || "").trim(); - if (!apiUrl && !channelId && !token) { + if (!apiBaseUrl && !channelId && !token) { return null; } - if (!apiUrl || !channelId || !token) { - throw new Error( - "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" - ); + if (!apiBaseUrl || !channelId || !token) { + throw new Error("LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required"); } - return { apiUrl, channelId, token }; + return { + postsApiUrl: buildLoopEndpointUrl(apiBaseUrl, "posts"), + filesApiUrl: buildLoopEndpointUrl(apiBaseUrl, "files"), + channelId, + token, + strictDelivery: parseBooleanEnv(env.LOOP_STRICT_DELIVERY), + strictFileUploads: parseBooleanEnv(env.LOOP_STRICT_FILE_UPLOAD), + }; } /** @@ -91,7 +125,7 @@ function readLoopConfig(env = process.env) { * @returns {{ * reportsDir: string, * configuredClusters: string[], - * loop: { apiUrl: string, channelId: string, token: string } | null + * loop: { postsApiUrl: string, filesApiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null * }} Normalized messenger configuration. */ function readMessengerConfigFromEnv(env = process.env) { diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 647651bea0..9e96c166c9 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -18,7 +18,8 @@ /** * @typedef {Object} LoopCredentials - * @property {string} apiUrl + * @property {string} postsApiUrl + * @property {string} filesApiUrl * @property {string} channelId * @property {string} token */ @@ -26,8 +27,8 @@ /** * @typedef {Object} LoopPublishParams * @property {string} message - * @property {string[]} threadMessages - * @property {LoopCredentials} loop + * @property {Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>} threadMessages + * @property {LoopCredentials & {strictFileUploads?: boolean}} loop */ /** @@ -60,20 +61,32 @@ function parseLoopApiPayload(responseText, core) { * @param {string} message Post body. * @param {string} [rootId] Optional thread root id for reply posts. * @param {LoopClientCore} core GitHub core API. + * @param {string[]} [fileIds] Uploaded Loop file ids to attach. + * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise>} Parsed Loop API response. */ -async function postToLoopApi(loop, message, rootId, core) { - const response = await fetch(loop.apiUrl, { +async function postToLoopApi( + loop, + message, + rootId, + core, + fileIds = [], + { fetch: fetchFn = globalThis.fetch } = {} +) { + const body = { + channel_id: loop.channelId, + message, + ...(rootId ? { root_id: rootId } : {}), + ...(fileIds.length > 0 ? { file_ids: fileIds } : {}), + }; + + const response = await fetchFn(loop.postsApiUrl, { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, "Content-Type": "application/json", }, - body: JSON.stringify({ - channel_id: loop.channelId, - message, - ...(rootId ? { root_id: rootId } : {}), - }), + body: JSON.stringify(body), }); const responseText = await response.text(); @@ -88,15 +101,70 @@ async function postToLoopApi(loop, message, rootId, core) { return payload; } +/** + * Uploads a single file to Loop and returns the created file id. + * + * @param {LoopCredentials} loop Loop API credentials. + * @param {string} fileName File name shown in Loop. + * @param {Buffer} buffer File content. + * @param {LoopClientCore} core GitHub core API. + * @param {string} mimeType File MIME type. + * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. + * @returns {Promise} Uploaded Loop file id. + */ +async function uploadFileToLoop( + loop, + fileName, + buffer, + core, + mimeType, + { fetch: fetchFn = globalThis.fetch } = {} +) { + const formData = new FormData(); + formData.append("channel_id", loop.channelId); + formData.append("files", new Blob([buffer], { type: mimeType }), fileName); + + const response = await fetchFn(loop.filesApiUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${loop.token}`, + }, + body: formData, + }); + const responseText = await response.text(); + + if (!response.ok) { + throw new Error( + `Loop file upload failed with status ${response.status}: ${responseText}` + ); + } + + const payload = parseLoopApiPayload(responseText, core); + const fileId = payload.file_infos && payload.file_infos[0] && payload.file_infos[0].id; + if (!fileId) { + throw new Error("Loop API did not return uploaded file id"); + } + + core.info(`Loop API accepted file ${fileName} with status ${response.status}`); + return fileId; +} + /** * Publishes the main report and optional failed-tests thread to Loop. * * @param {LoopPublishParams} params Message payload and Loop credentials. * @param {LoopClientCore} core GitHub core API. + * @param {{fetch?: typeof fetch}} [options] Optional HTTP client dependencies. * @returns {Promise} */ -async function makeThreadedReportInLoop({ message, threadMessages, loop }, core) { - const rootPost = await postToLoopApi(loop, message, undefined, core); +async function makeThreadedReportInLoop( + { message, threadMessages, loop }, + core, + { fetch: fetchFn = globalThis.fetch } = {} +) { + const rootPost = await postToLoopApi(loop, message, undefined, core, [], { + fetch: fetchFn, + }); if (!rootPost.id) { throw new Error( @@ -104,11 +172,42 @@ async function makeThreadedReportInLoop({ message, threadMessages, loop }, core) ); } - for (const replyMessage of threadMessages) { - await postToLoopApi(loop, replyMessage, rootPost.id, core); + for (const reply of threadMessages) { + const files = Array.isArray(reply.files) ? reply.files : []; + let fileIds = []; + if (files.length > 0) { + const results = await Promise.allSettled( + files.map((file) => + uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType, { + fetch: fetchFn, + }) + ) + ); + fileIds = results + .filter((result) => result.status === "fulfilled") + .map((result) => result.value); + + const failures = results.filter((result) => result.status === "rejected"); + const failureDetails = failures.map((failure) => { + const reason = failure.reason; + return reason && reason.message ? reason.message : String(reason); + }); + for (const details of failureDetails) { + core.warning(`Loop file upload failed for one attachment: ${details}`); + } + if (loop.strictFileUploads && failures.length > 0) { + throw new Error( + `Strict file uploads enabled; at least one attachment failed: ${failureDetails.join("; ")}` + ); + } + } + await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds, { + fetch: fetchFn, + }); } } module.exports = { makeThreadedReportInLoop, + uploadFileToLoop, }; diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js new file mode 100644 index 0000000000..5b3a9be878 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -0,0 +1,272 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { uploadFileToLoop, makeThreadedReportInLoop } = require("./loop-client"); +const { createCore } = require("../shared/test-utils"); + +function createLoop(overrides = {}) { + return { + postsApiUrl: "https://loop.example.invalid/api/v4/posts", + filesApiUrl: "https://loop.example.invalid/api/v4/files", + channelId: "channel-id", + token: "loop-token", + ...overrides, + }; +} + +describe("loop-client", () => { + afterEach(() => { + delete global.fetch; + }); + + test("uploads files to Loop multipart endpoint", async () => { + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-id" }] }), + }); + + const fileId = await uploadFileToLoop( + createLoop(), + "chart.png", + Buffer.from("image-bytes"), + createCore(), + "image/png" + ); + + expect(fileId).toBe("file-id"); + expect(global.fetch).toHaveBeenCalledWith( + "https://loop.example.invalid/api/v4/files", + expect.objectContaining({ + method: "POST", + headers: { + Authorization: "Bearer loop-token", + }, + }) + ); + + const body = global.fetch.mock.calls[0][1].body; + expect(body.get("channel_id")).toBe("channel-id"); + expect(body.get("files").name).toBe("chart.png"); + await expect(body.get("files").text()).resolves.toBe("image-bytes"); + }); + + test("posts the reply with uploaded chart file ids", async () => { + const loop = createLoop(); + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-one" }] }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-two" }] }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + global.fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "feature-duration-status.png", + buffer: Buffer.from("one"), + mimeType: "image/png", + }, + { + name: "feature-duration-status-2.png", + buffer: Buffer.from("two"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + createCore() + ); + + expect(global.fetch).toHaveBeenCalledTimes(4); + expect(JSON.parse(global.fetch.mock.calls[3][1].body)).toEqual({ + channel_id: "channel-id", + message: "reply", + root_id: "root-post-id", + file_ids: ["file-one", "file-two"], + }); + }); + + test("posts the reply with successful attachments when one upload fails", async () => { + const loop = createLoop(); + const core = createCore(); + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-one" }] }), + }, + { + ok: false, + status: 403, + text: async () => + JSON.stringify({ + id: "api.context.permissions.app_error", + message: "permission denied", + }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + global.fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "feature-duration-status.png", + buffer: Buffer.from("one"), + mimeType: "image/png", + }, + { + name: "feature-duration-status-2.png", + buffer: Buffer.from("two"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + core + ); + + expect(global.fetch).toHaveBeenCalledTimes(4); + expect(global.fetch.mock.calls[0][0]).toBe(loop.postsApiUrl); + expect(global.fetch.mock.calls[1][0]).toBe(loop.filesApiUrl); + expect(global.fetch.mock.calls[2][0]).toBe(loop.filesApiUrl); + expect(global.fetch.mock.calls[3][0]).toBe(loop.postsApiUrl); + + const replyBody = JSON.parse(global.fetch.mock.calls[3][1].body); + expect(replyBody.root_id).toBe("root-post-id"); + expect(replyBody.message).toBe("reply"); + expect(replyBody.file_ids).toEqual(["file-one"]); + + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining( + "Loop file upload failed for one attachment: Loop file upload failed with status 403" + ) + ); + expect(core.warning).toHaveBeenCalledTimes(1); + }); + + test("fails when strict file upload mode is enabled", async () => { + const loop = createLoop({ + strictFileUploads: true, + }); + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: false, + status: 403, + text: async () => "permission denied", + }, + ]; + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await expect( + makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "chart.png", + buffer: Buffer.from("image-bytes"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + createCore() + ) + ).rejects.toThrow("Strict file uploads enabled; at least one attachment failed"); + expect(global.fetch).toHaveBeenCalledTimes(2); + }); + + test("uses injected fetch without touching the global fetch", async () => { + const originalFetch = globalThis.fetch; + const loop = createLoop(); + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + const fetch = jest.fn().mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [{ message: "reply", files: [] }], + loop, + }, + createCore(), + { fetch } + ); + + expect(fetch).toHaveBeenCalledTimes(2); + expect(globalThis.fetch).toBe(originalFetch); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index 893b15e45a..db7a696706 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -337,28 +337,77 @@ function renderFailedTestsThreadMessage(report) { return lines.join("\n"); } +function hasSpecTimings(report) { + return Array.isArray(report.specTimings) && report.specTimings.length > 0; +} + +function buildChartCaption(_files, chartsUnavailable) { + return chartsUnavailable ? "Charts unavailable." : ""; +} + /** - * Builds optional failed-tests thread messages for clusters with failed tests. + * Builds optional per-cluster thread messages for failed tests and chart attachments. * * @param {Array>} orderedReports Cluster reports in display order. - * @returns {string[]} Markdown thread message bodies. + * @param {{ + * getClusterChartFiles?: function(Record): Promise>, + * core?: {warning?: function(string): void} + * }} [options] + * @returns {Promise}>>} Markdown thread payloads. */ -function buildThreadMessages(orderedReports) { - const testsReports = orderedReports.filter((report) => - isTestResultReport(report) - ); - const failedTestReports = testsReports.filter(hasFailedTests); +async function buildThreadMessages(orderedReports, { getClusterChartFiles, core } = {}) { + const testsReports = orderedReports.filter((report) => isTestResultReport(report)); + const threadMessages = []; + let renderedFailedTestsHeading = false; - if (failedTestReports.length === 0) { - return []; + for (const report of testsReports) { + const messageParts = []; + let files = []; + let chartsUnavailable = false; + + if (getClusterChartFiles && hasSpecTimings(report)) { + try { + files = await getClusterChartFiles(report); + } catch (error) { + chartsUnavailable = true; + if (core && typeof core.warning === "function") { + core.warning( + `Unable to prepare duration chart files for cluster ${getReportClusterKey(report) || "unknown"}: ${ + error.message + }` + ); + } + } + } + + if (!hasFailedTests(report) && files.length === 0 && !chartsUnavailable) { + continue; + } + + if (hasFailedTests(report)) { + const clusterMessage = renderFailedTestsThreadMessage(report); + messageParts.push( + renderedFailedTestsHeading + ? clusterMessage + : ["### Failed tests", clusterMessage].join("\n\n") + ); + renderedFailedTestsHeading = true; + } else { + messageParts.push(`**${formatClusterLink(report)}**`); + } + + const chartCaption = buildChartCaption(files, chartsUnavailable); + if (chartCaption) { + messageParts.push(chartCaption); + } + + threadMessages.push({ + message: messageParts.join("\n\n"), + files, + }); } - return failedTestReports.map((report, index) => { - const clusterMessage = renderFailedTestsThreadMessage(report); - return index === 0 - ? ["### Failed tests", clusterMessage].join("\n\n") - : clusterMessage; - }); + return threadMessages; } module.exports = { diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index 3d3f5b2940..9fe4a6a26b 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -86,6 +86,11 @@ function formatSpecName(specReport) { .trim(); } +function runtimeMs(value) { + const runtime = Number(value || 0); + return Number.isFinite(runtime) ? Math.round(runtime / 1_000_000) : 0; +} + /** * Maps a raw Ginkgo spec state into the metrics bucket used by the final * messenger report. @@ -153,6 +158,8 @@ function buildFailureDetail(specReport) { * metrics: GinkgoMetrics, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array<{name: string, group: string, state: string, runtimeMs: number, labels: string[]}>, + * suiteTotalMs: number, * startedAt: string|null * }} Parsed report payload. */ @@ -161,10 +168,14 @@ function parseGinkgoReport(jsonContent) { const metrics = zeroMetrics(); const failedTests = []; const failedTestDetails = []; + const specTimings = []; const startedAt = suites.find((suite) => suite && suite.StartTime)?.StartTime || null; + let suiteTotalMs = 0; for (const suite of suites) { + suiteTotalMs += runtimeMs(suite && suite.RunTime); + for (const specReport of toArray(suite && suite.SpecReports)) { if (isSuiteNodeFailure(specReport)) { const failureDetail = buildFailureDetail(specReport); @@ -186,6 +197,20 @@ function parseGinkgoReport(jsonContent) { metrics.total += 1; const metricKey = getMetricKeyForState(specReport.State); metrics[metricKey] += 1; + const hierarchyParts = toArray(specReport.ContainerHierarchyTexts) + .map((part) => String(part || "").trim()) + .filter(Boolean); + const leafText = String(specReport.LeafNodeText || "").trim(); + specTimings.push({ + name: leafText, + group: hierarchyParts[0] || "Top-level Its", + state: metricKey, + runtimeMs: runtimeMs(specReport.RunTime), + labels: flattenLabels([ + ...toArray(specReport.ContainerHierarchyLabels), + ...toArray(specReport.LeafNodeLabels), + ]), + }); if (failureStates.has(metricKey)) { const failureDetail = buildFailureDetail(specReport); @@ -214,6 +239,8 @@ function parseGinkgoReport(jsonContent) { ]) ).values() ), + specTimings, + suiteTotalMs, startedAt, }; } @@ -330,6 +357,8 @@ function extractFailureReasonFromOutput(output, suiteNodeType) { * metrics: GinkgoMetrics, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: [], + * suiteTotalMs: 0, * startedAt: null, * }} Parsed fallback payload. */ @@ -340,6 +369,8 @@ function parseGinkgoOutput(outputContent) { metrics: zeroMetrics(), failedTests: [], failedTestDetails: [], + specTimings: [], + suiteTotalMs: 0, startedAt: null, }; diff --git a/.github/scripts/python/e2e_report/__init__.py b/.github/scripts/python/e2e_report/__init__.py new file mode 100644 index 0000000000..c7e0518801 --- /dev/null +++ b/.github/scripts/python/e2e_report/__init__.py @@ -0,0 +1,11 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/.github/scripts/python/e2e_report/charts.py b/.github/scripts/python/e2e_report/charts.py new file mode 100644 index 0000000000..9d64e83edc --- /dev/null +++ b/.github/scripts/python/e2e_report/charts.py @@ -0,0 +1,576 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render E2E report charts for CI and local debugging. + +Subcommands: +- messenger-all renders feature-duration-status PNGs and writes a manifest. +- slowest renders one slowest-specs PNG for a selected Describe. +- top renders slowest-specs PNGs for the top-N slowest Describes. + +The manifest schema is {"clusters": {"cluster": [{"name", "path", "mimeType"}]}}. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +from matplotlib.ticker import FuncFormatter, MultipleLocator # noqa: E402 + + +StatusName = Literal["passed", "failed", "errors", "skipped"] + +STATUSES: tuple[StatusName, ...] = ("passed", "failed", "errors", "skipped") +STATUS_COLORS: dict[StatusName, str] = { + "passed": "#00b83f", + "failed": "#ff3333", + "errors": "#d9a300", + "skipped": "#8f9aa3", +} +DURATION_COLORS = { + "fast": "#7ee787", + "medium": "#3fb950", + "slow": "#238636", +} +SLOW_THRESHOLD_MS = 300_000 +MEDIUM_THRESHOLD_MS = 60_000 +DEFAULT_TOP_N = 15 +REPORT_FILE_PATTERN = re.compile(r"^e2e_report_.*\.json$") + + +@dataclass(frozen=True) +class Timing: + name: str + group: str + state: str + runtime_ms: float + + @property + def full_name(self) -> str: + return self.name if self.group == self.name else f"{self.group} / {self.name}" + + +def sanitize_filename_part(value: Any) -> str: + """Return a path-safe file name part for user-controlled chart labels.""" + safe = re.sub(r"[^a-zA-Z0-9_-]+", "_", str(value or "cluster")) + return safe or "cluster" + + +def to_seconds(ms: float) -> float: + """Convert milliseconds to rounded seconds for chart labels.""" + return round(float(ms) / 1000, 2) + + +def normalize_timing(raw: dict[str, Any] | None) -> Timing: + raw = raw or {} + state = str(raw.get("state") or "errors") + if state not in STATUSES: + state = "errors" + + runtime = raw.get("runtimeMs", 0) + try: + runtime_ms = float(runtime) + except (TypeError, ValueError): + runtime_ms = 0 + if not math.isfinite(runtime_ms) or runtime_ms < 0: + runtime_ms = 0 + + return Timing( + name=str(raw.get("name") or "Unnamed spec"), + group=str(raw.get("group") or "Top-level Its"), + state=state, + runtime_ms=runtime_ms, + ) + + +def aggregate(spec_timings: list[dict[str, Any]] | None) -> tuple[list[Timing], dict[str, dict[str, Any]]]: + timings: list[Timing] = [] + by_group: dict[str, dict[str, Any]] = {} + + for raw_timing in spec_timings or []: + timing = normalize_timing(raw_timing) + timings.append(timing) + group = by_group.setdefault( + timing.group, + { + "status_count": {status: 0 for status in STATUSES}, + "status_durations": {status: 0.0 for status in STATUSES}, + "total": 0.0, + }, + ) + group["status_count"][timing.state] += 1 + group["status_durations"][timing.state] += timing.runtime_ms + group["total"] += timing.runtime_ms + + return timings, by_group + + +def duration_bucket(timing: Timing) -> str: + if timing.runtime_ms > SLOW_THRESHOLD_MS: + return "slow" + if timing.runtime_ms >= MEDIUM_THRESHOLD_MS: + return "medium" + return "fast" + + +def format_seconds(seconds: float) -> str: + """Format seconds compactly for bar labels.""" + return f"{seconds:.0f}s" if seconds >= 10 else f"{seconds:.1f}s" + + +def format_axis_seconds(value: float, _position: int) -> str: + """Format axis ticks as integer seconds.""" + return f"{int(value):,}" + + +def next_tick(value: float, step: int) -> int: + """Round value up to the next axis tick.""" + if value <= 0: + return step + return int(math.ceil(value / step) * step) + + +def _cluster_key(report: dict[str, Any]) -> str: + return str(report.get("storageType") or report.get("cluster") or "").strip() + + +def _compute_feature_segments( + by_group: dict[str, dict[str, Any]], +) -> tuple[ + list[str], + list[tuple[StatusName, list[float], list[float]]], + list[float], +]: + entries = sorted( + by_group.items(), + key=lambda item: ( + -(item[1]["status_count"]["failed"] + item[1]["status_count"]["errors"]), + -item[1]["total"], + item[0], + ), + ) + labels = [name for name, _ in entries] + left = [0.0] * len(entries) + segments: list[tuple[StatusName, list[float], list[float]]] = [] + + for status in STATUSES: + values = [to_seconds(group["status_durations"][status]) for _, group in entries] + segments.append((status, values, left.copy())) + left = [current + value for current, value in zip(left, values)] + + return labels, segments, left + + +def _apply_axes_style(ax: Any, labels: list[str], x_limit: int) -> None: + ax.set_xlim(0, x_limit) + ax.set_title("Overall durations for Describes", fontsize=8, pad=12) + ax.set_xlabel("Duration, seconds", fontsize=7) + ax.invert_yaxis() + if labels: + ax.set_ylim(len(labels) - 0.6, -0.6) + ax.legend( + loc="upper center", + bbox_to_anchor=(0.5, 1.015), + ncol=len(STATUSES), + frameon=False, + fontsize=6, + handlelength=2.4, + columnspacing=0.8, + ) + ax.margins(y=0) + ax.xaxis.set_major_locator(MultipleLocator(60)) + ax.xaxis.set_major_formatter(FuncFormatter(format_axis_seconds)) + ax.grid(axis="x", color="#c8c8c8", alpha=0.45, linewidth=0.5) + ax.grid(axis="y", color="#d9d9d9", alpha=0.55, linewidth=0.5) + ax.set_axisbelow(True) + ax.tick_params(axis="both", labelsize=6, colors="#555555", length=0) + for spine in ax.spines.values(): + spine.set_color("#dddddd") + spine.set_linewidth(0.5) + + +def render_feature_duration_status(report: dict[str, Any], output_dir: Path) -> dict[str, str]: + _, by_group = aggregate(report.get("specTimings") or []) + labels, segments, left = _compute_feature_segments(by_group) + height = max(6.4, 0.75 + len(labels) * 0.285) + fig, ax = plt.subplots(figsize=(10.24, height), dpi=100) + + for status, values, segment_left in segments: + ax.barh(labels, values, left=segment_left, label=status, color=STATUS_COLORS[status], height=0.72) + for row, (offset, value) in enumerate(zip(segment_left, values)): + if value <= 0: + continue + ax.text( + offset + value / 2, + row, + format_seconds(value), + ha="center", + va="center", + fontsize=6, + color="#333333", + ) + + x_limit = next_tick(max(left, default=0), 60) + _apply_axes_style(ax, labels, x_limit) + + cluster_name = sanitize_filename_part(_cluster_key(report) or "cluster") + return save_figure(fig, output_dir / f"{cluster_name}-feature-duration-status.png") + + +def render_slowest_specs( + spec_timings: list[dict[str, Any]], + storage_name: str, + describe: str, + output_dir: Path, + top_n: int = DEFAULT_TOP_N, +) -> dict[str, str]: + timings, _ = aggregate(spec_timings) + top = sorted(timings, key=lambda timing: (-timing.runtime_ms, timing.full_name))[:top_n] + labels = [timing.full_name for timing in top] + values = [to_seconds(timing.runtime_ms) for timing in top] + colors = [DURATION_COLORS[duration_bucket(timing)] for timing in top] + edge_colors = [ + STATUS_COLORS[timing.state] if timing.state in {"failed", "errors"} else "none" + for timing in top + ] + line_widths = [2 if timing.state in {"failed", "errors"} else 0 for timing in top] + + height = max(4.0, 0.6 + len(labels) * 0.45) + fig, ax = plt.subplots(figsize=(20.48, height), dpi=100) + ax.barh(labels, values, color=colors, edgecolor=edge_colors, linewidth=line_widths) + ax.set_title("Top slowest successful specs and failed specs (It/Entry)") + ax.set_xlabel("Duration, seconds") + ax.invert_yaxis() + ax.grid(axis="x", alpha=0.2) + + for row, (seconds, timing) in enumerate(zip(values, top)): + suffix = f" [{timing.state}]" if timing.state in {"failed", "errors"} else "" + ax.text(seconds, row, f" {format_seconds(seconds)}{suffix}", va="center", fontsize=8) + + file_name = ( + f"{sanitize_filename_part(storage_name)}-" + f"{sanitize_filename_part(describe)}-slowest-specs.png" + ) + return save_figure(fig, output_dir / file_name) + + +def save_figure(fig: plt.Figure, target_path: Path) -> dict[str, str]: + target_path.parent.mkdir(parents=True, exist_ok=True) + fig.tight_layout() + fig.savefig(target_path, format="png") + plt.close(fig) + return { + "name": target_path.name, + "path": str(target_path), + "mimeType": "image/png", + } + + +def runtime_ns_to_ms(value: Any) -> int: + """Ginkgo SpecReport.RunTime is in nanoseconds; round to milliseconds.""" + try: + runtime = float(value or 0) + except (TypeError, ValueError): + return 0 + if not math.isfinite(runtime) or runtime < 0: + return 0 + return round(runtime / 1_000_000) + + +def metric_key_for_state(state: Any) -> StatusName: + normalized = str(state or "").strip().lower() + if normalized in {"passed", "failed"}: + return normalized + # Keep the same pending-to-skipped collapse as shared/report-model.js. + if normalized in {"skipped", "pending"}: + return "skipped" + return "errors" + + +def parse_ginkgo_report(payload: Any) -> list[dict[str, Any]]: + suites = payload if isinstance(payload, list) else [payload] + timings: list[dict[str, Any]] = [] + + for suite in suites: + if not isinstance(suite, dict): + continue + for spec_report in suite.get("SpecReports") or []: + if not isinstance(spec_report, dict) or spec_report.get("LeafNodeType") != "It": + continue + hierarchy = [ + str(part).strip() + for part in spec_report.get("ContainerHierarchyTexts") or [] + if str(part).strip() + ] + timings.append( + { + "name": str(spec_report.get("LeafNodeText") or "").strip(), + "group": hierarchy[0] if hierarchy else "Top-level Its", + "state": metric_key_for_state(spec_report.get("State")), + "runtimeMs": runtime_ns_to_ms(spec_report.get("RunTime")), + } + ) + + return timings + + +def read_report(json_path: str | Path) -> dict[str, Any]: + path = Path(json_path) + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, dict) and isinstance(payload.get("specTimings"), list): + return payload + return {"specTimings": parse_ginkgo_report(payload)} + + +def available_describes(spec_timings: list[dict[str, Any]]) -> list[str]: + return sorted( + { + str(timing.get("group") or "").strip() + for timing in spec_timings + if str(timing.get("group") or "").strip() + } + ) + + +def derive_storage_type(report_path: str | Path, fallback_storage: str | None = None) -> str: + base_name = Path(report_path).name + dated_match = re.match(r"^e2e_report_(.+)_(\d{4}-\d{2}-\d{2}.*)\.json$", base_name) + if dated_match: + return dated_match.group(1) + generic_match = re.match(r"^e2e_report_(.+?)_.*\.json$", base_name) + if generic_match: + return generic_match.group(1) + if fallback_storage: + return fallback_storage + raise ValueError(f'Unable to derive storage type from file name "{base_name}". Pass --storage.') + + +def report_cluster_key(report: dict[str, Any]) -> str: + return _cluster_key(report) + + +def top_describes(spec_timings: list[dict[str, Any]] | None, top_n: int = 5) -> list[str]: + totals: dict[str, float] = {} + for raw_timing in spec_timings or []: + group = str(raw_timing.get("group") or "").strip() + if not group: + continue + try: + runtime = float(raw_timing.get("runtimeMs") or 0) + except (TypeError, ValueError): + runtime = 0 + totals[group] = totals.get(group, 0) + runtime + + return [name for name, _ in sorted(totals.items(), key=lambda item: (-item[1], item[0]))[:top_n]] + + +def render_cluster_charts(report: dict[str, Any], output_dir: Path) -> list[dict[str, str]]: + if not report.get("specTimings"): + return [] + return [render_feature_duration_status(report, output_dir)] + + +def render_messenger_charts( + reports_dir: str | Path = "downloaded-artifacts", + out_dir: str | Path = "tmp/messenger-charts", + manifest_path: str | Path = "tmp/messenger-charts/manifest.json", +) -> dict[str, dict[str, list[dict[str, str]]]]: + output_dir = Path(out_dir) + clusters: dict[str, list[dict[str, str]]] = {} + + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + cluster_name = report_cluster_key(report) or derive_storage_type(report_file) + files = render_cluster_charts(report, output_dir) + if files: + clusters[cluster_name] = files + + manifest = {"clusters": clusters} + target_path = Path(manifest_path) + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + return manifest + + +def _render_slowest_for_report( + report: dict[str, Any], + storage_name: str, + describe: str, + output_dir: Path, +) -> dict[str, str]: + if not describe: + raise ValueError("--describe is required") + + spec_timings = report.get("specTimings") or [] + filtered_timings = [ + timing for timing in spec_timings if str(timing.get("group") or "") == describe + ] + if not filtered_timings: + describes = available_describes(spec_timings) + lines = [ + f'No specs found for Describe "{describe}".', + "Available Describes:", + *(f"- {name}" for name in describes or [""]), + ] + raise ValueError("\n".join(lines)) + + return render_slowest_specs(filtered_timings, storage_name, describe, output_dir) + + +def render_slowest_for_describe( + json_path: str | Path, + describe: str, + out_dir: str | Path = "tmp", + storage: str | None = None, +) -> dict[str, str]: + report = read_report(json_path) + storage_name = ( + storage + or _cluster_key(report) + or derive_storage_type(json_path) + ) + return _render_slowest_for_report(report, storage_name, describe, Path(out_dir)) + + +def list_report_files(reports_dir: str | Path) -> list[Path]: + root = Path(reports_dir) + if not root.exists(): + return [] + return sorted(path for path in root.rglob("*") if path.is_file() and REPORT_FILE_PATTERN.match(path.name)) + + +def render_top_describes( + reports_dir: str | Path = "downloaded-artifacts", + out_dir: str | Path = "tmp", + top_n: int = 5, +) -> list[dict[str, str]]: + rendered_files: list[dict[str, str]] = [] + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + storage_name = report_cluster_key(report) or derive_storage_type(report_file) + for describe in top_describes(report.get("specTimings") or [], top_n): + try: + rendered_files.append( + _render_slowest_for_report( + report, + storage_name, + describe, + Path(out_dir), + ) + ) + except Exception as error: + print( + f'Failed to render Describe "{describe}" from "{report_file}": {error}', + file=sys.stderr, + ) + return rendered_files + + +def collect_messenger_debug(reports_dir: str | Path) -> dict[str, dict[str, Any]]: + clusters: dict[str, dict[str, Any]] = {} + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + cluster_name = report_cluster_key(report) or derive_storage_type(report_file) + _, by_group = aggregate(report.get("specTimings") or []) + clusters[cluster_name] = by_group + return {"clusters": clusters} + + +def collect_top_debug(reports_dir: str | Path, top_n: int) -> dict[str, dict[str, Any]]: + clusters: dict[str, dict[str, Any]] = {} + for report_file in list_report_files(reports_dir): + report = read_report(report_file) + cluster_name = report_cluster_key(report) or derive_storage_type(report_file) + clusters[cluster_name] = { + "topDescribes": top_describes(report.get("specTimings") or [], top_n), + } + return {"clusters": clusters} + + +def write_debug_json(payload: Any, debug_json_path: str | Path) -> None: + target_path = Path(debug_json_path) + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Render E2E report charts") + subparsers = parser.add_subparsers(dest="command", required=True) + + messenger_all = subparsers.add_parser("messenger-all", help="Render messenger charts and write a manifest") + messenger_all.add_argument("--reports-dir", default="downloaded-artifacts") + messenger_all.add_argument( + "--out-dir", + default="tmp/messenger-charts", + help="Literal directory for rendered PNG files.", + ) + messenger_all.add_argument("--manifest", default="tmp/messenger-charts/manifest.json") + messenger_all.add_argument("--debug-json", help="Write aggregate debug data to this JSON path.") + + slowest = subparsers.add_parser("slowest", help="Render slowest specs for one Describe") + slowest.add_argument("--json", required=True) + slowest.add_argument("--describe", required=True) + slowest.add_argument( + "--out-dir", + default="tmp", + help="Literal directory for the rendered PNG file.", + ) + slowest.add_argument("--storage") + + top = subparsers.add_parser("top", help="Render slowest specs for top-N Describes") + top.add_argument("--reports-dir", default="downloaded-artifacts") + top.add_argument( + "--out-dir", + default="tmp", + help="Literal directory for rendered PNG files.", + ) + top.add_argument("--top-n", type=int, default=5) + top.add_argument("--debug-json", help="Write aggregate debug data to this JSON path.") + + args = parser.parse_args(argv) + + if args.command == "messenger-all": + if not list_report_files(args.reports_dir): + raise SystemExit(f'No report files found in "{args.reports_dir}".') + render_messenger_charts(args.reports_dir, args.out_dir, args.manifest) + if args.debug_json: + write_debug_json(collect_messenger_debug(args.reports_dir), args.debug_json) + print(args.manifest) + return + if args.command == "slowest": + file = render_slowest_for_describe(args.json, args.describe, args.out_dir, args.storage) + print(file["path"]) + return + if args.command == "top": + if not list_report_files(args.reports_dir): + raise SystemExit(f'No report files found in "{args.reports_dir}".') + files = render_top_describes(args.reports_dir, args.out_dir, args.top_n) + if args.debug_json: + write_debug_json(collect_top_debug(args.reports_dir, args.top_n), args.debug_json) + for file in files: + print(file["path"]) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/python/e2e_report/charts_test.py b/.github/scripts/python/e2e_report/charts_test.py new file mode 100644 index 0000000000..da058d97d6 --- /dev/null +++ b/.github/scripts/python/e2e_report/charts_test.py @@ -0,0 +1,360 @@ +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from e2e_report import charts + + +def write_json(path: Path, payload: object) -> None: + path.write_text(json.dumps(payload), encoding="utf-8") + + +class ChartsTest(unittest.TestCase): + def test_aggregate_normalizes_timings(self) -> None: + timings, by_group = charts.aggregate( + [ + {"name": "fast", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + {"name": "bad", "group": "VM", "state": "unknown", "runtimeMs": "bad"}, + ] + ) + + self.assertEqual([timing.state for timing in timings], ["passed", "errors"]) + self.assertEqual(timings[1].runtime_ms, 0) + self.assertEqual(by_group["VM"]["status_count"]["errors"], 1) + + def test_top_describes_uses_duration_desc_then_name(self) -> None: + self.assertEqual( + charts.top_describes( + [ + {"group": "VM", "runtimeMs": 30_000}, + {"group": "Disk", "runtimeMs": 20_000}, + {"group": "Network", "runtimeMs": 20_000}, + {"group": "VM", "runtimeMs": 5_000}, + ], + 2, + ), + ["VM", "Disk"], + ) + + def test_render_slowest_for_describe_writes_expected_artifact(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "e2e_report_nfs_2026-05-15.json" + write_json( + report_path, + { + "storageType": "nfs", + "specTimings": [ + {"name": "fast", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, + ], + }, + ) + + rendered = charts.render_slowest_for_describe(report_path, "VM", out_dir=temp_dir) + + self.assertEqual(rendered["name"], "nfs-VM-slowest-specs.png") + self.assertEqual(Path(rendered["path"]), Path(temp_dir) / rendered["name"]) + self.assertTrue(Path(rendered["path"]).is_file()) + + def test_render_slowest_for_describe_lists_available_describes(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "e2e_report_nfs_2026-05-15.json" + write_json( + report_path, + { + "specTimings": [ + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 30_000}, + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 10_000}, + ], + }, + ) + + with self.assertRaisesRegex(ValueError, "Available Describes:\\n- Disk\\n- VM"): + charts.render_slowest_for_describe(report_path, "Network", out_dir=temp_dir) + + def test_render_cluster_charts_writes_messenger_chart_only(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + files = charts.render_cluster_charts( + { + "cluster": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + Path(temp_dir), + ) + + self.assertEqual([file["name"] for file in files], ["replicated-feature-duration-status.png"]) + self.assertTrue(Path(files[0]["path"]).is_file()) + + def test_render_feature_duration_status_writes_non_empty_png(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + file = charts.render_feature_duration_status( + { + "storageType": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + Path(temp_dir), + ) + + self.assertGreater(Path(file["path"]).stat().st_size, 1000) + + def test_render_messenger_charts_writes_manifest(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + reports_dir = Path(temp_dir) / "reports" + out_dir = Path(temp_dir) / "charts" + manifest_path = out_dir / "manifest.json" + reports_dir.mkdir() + write_json( + reports_dir / "e2e_report_replicated.json", + { + "storageType": "replicated", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + ) + + manifest = charts.render_messenger_charts(reports_dir, out_dir, manifest_path) + + self.assertEqual( + [file["name"] for file in manifest["clusters"]["replicated"]], + ["replicated-feature-duration-status.png"], + ) + self.assertEqual(json.loads(manifest_path.read_text(encoding="utf-8")), manifest) + + def test_report_cluster_key_prefers_storage_type(self) -> None: + report = {"storageType": "storage-first", "cluster": "cluster-second"} + + self.assertEqual(charts.report_cluster_key(report), "storage-first") + + def test_render_messenger_charts_uses_storage_type_key(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + reports_dir = Path(temp_dir) / "reports" + out_dir = Path(temp_dir) / "charts" + manifest_path = out_dir / "manifest.json" + reports_dir.mkdir() + write_json( + reports_dir / "e2e_report_replicated.json", + { + "storageType": "storage-first", + "cluster": "cluster-second", + "specTimings": [ + {"name": "slow", "group": "VM", "state": "passed", "runtimeMs": 90_000}, + ], + }, + ) + + manifest = charts.render_messenger_charts(reports_dir, out_dir, manifest_path) + + self.assertIn("storage-first", manifest["clusters"]) + self.assertNotIn("cluster-second", manifest["clusters"]) + + +class RuntimeNsToMsTest(unittest.TestCase): + def test_runtime_ns_to_ms(self) -> None: + cases = [ + (1_500_000.0, 2), + (1_000_000, 1), + (-1, 0), + (float("nan"), 0), + (None, 0), + ("bad", 0), + ] + + for value, expected in cases: + with self.subTest(value=value): + self.assertEqual(charts.runtime_ns_to_ms(value), expected) + + +class ParseGinkgoReportTest(unittest.TestCase): + def test_parse_ginkgo_report_filters_non_it_and_uses_default_group(self) -> None: + payload = { + "SpecReports": [ + { + "LeafNodeType": "It", + "LeafNodeText": "creates vm", + "ContainerHierarchyTexts": ["VM", "Nested"], + "State": "passed", + "RunTime": 1_500_000, + }, + { + "LeafNodeType": "BeforeSuite", + "LeafNodeText": "setup", + "ContainerHierarchyTexts": ["Suite"], + "State": "failed", + "RunTime": 10_000_000, + }, + { + "LeafNodeType": "It", + "LeafNodeText": "top level", + "ContainerHierarchyTexts": [], + "State": "pending", + "RunTime": 2_000_000, + }, + ], + } + + self.assertEqual( + charts.parse_ginkgo_report(payload), + [ + {"name": "creates vm", "group": "VM", "state": "passed", "runtimeMs": 2}, + {"name": "top level", "group": "Top-level Its", "state": "skipped", "runtimeMs": 2}, + ], + ) + + +class DeriveStorageTypeTest(unittest.TestCase): + def test_derive_storage_type(self) -> None: + self.assertEqual( + charts.derive_storage_type("e2e_report_replicated_2026-05-15.json"), + "replicated", + ) + self.assertEqual(charts.derive_storage_type("e2e_report_nfs_local.json"), "nfs") + self.assertEqual(charts.derive_storage_type("custom.json", "fallback"), "fallback") + + with self.assertRaisesRegex(ValueError, "Unable to derive storage type"): + charts.derive_storage_type("custom.json") + + +class ListReportFilesTest(unittest.TestCase): + def test_list_report_files_returns_sorted_matches(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + nested = root / "nested" + nested.mkdir() + write_json(nested / "e2e_report_z.json", {}) + write_json(root / "e2e_report_a.json", {}) + write_json(root / "other.json", {}) + (root / "e2e_report_dir.json").mkdir() + + self.assertEqual( + [path.name for path in charts.list_report_files(root)], + ["e2e_report_a.json", "e2e_report_z.json"], + ) + + +class RenderTopDescribesTest(unittest.TestCase): + def test_render_top_describes_writes_expected_pngs(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + reports_dir = root / "reports" + out_dir = root / "charts" + reports_dir.mkdir() + write_json( + reports_dir / "e2e_report_nfs_2026-05-15.json", + { + "storageType": "nfs", + "specTimings": [ + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 30_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 20_000}, + {"name": "net", "group": "Network", "state": "passed", "runtimeMs": 10_000}, + ], + }, + ) + + files = charts.render_top_describes(reports_dir, out_dir, top_n=2) + + self.assertEqual( + [file["name"] for file in files], + ["nfs-VM-slowest-specs.png", "nfs-Disk-slowest-specs.png"], + ) + self.assertTrue((out_dir / "nfs-VM-slowest-specs.png").is_file()) + + +class MainDispatchTest(unittest.TestCase): + def test_main_dispatches_messenger_all_slowest_and_top(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + reports_dir = root / "reports" + messenger_out = root / "messenger" + chart_out = root / "charts" + manifest_path = messenger_out / "manifest.json" + debug_path = root / "debug.json" + report_path = reports_dir / "e2e_report_nfs_2026-05-15.json" + reports_dir.mkdir() + write_json( + report_path, + { + "storageType": "nfs", + "specTimings": [ + {"name": "vm", "group": "VM", "state": "passed", "runtimeMs": 30_000}, + {"name": "disk", "group": "Disk", "state": "passed", "runtimeMs": 20_000}, + ], + }, + ) + + with contextlib.redirect_stdout(io.StringIO()): + charts.main( + [ + "messenger-all", + "--reports-dir", + str(reports_dir), + "--out-dir", + str(messenger_out), + "--manifest", + str(manifest_path), + "--debug-json", + str(debug_path), + ] + ) + charts.main( + [ + "slowest", + "--json", + str(report_path), + "--describe", + "VM", + "--out-dir", + str(chart_out), + ] + ) + charts.main( + [ + "top", + "--reports-dir", + str(reports_dir), + "--out-dir", + str(chart_out), + "--top-n", + "1", + ] + ) + + self.assertTrue(manifest_path.is_file()) + self.assertTrue(debug_path.is_file()) + self.assertTrue((chart_out / "nfs-VM-slowest-specs.png").is_file()) + + def test_top_with_zero_matching_reports_fails_clearly(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex(SystemExit, "No report files found"): + charts.main(["top", "--reports-dir", temp_dir]) + + def test_messenger_all_with_zero_matching_reports_fails_clearly(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with self.assertRaisesRegex(SystemExit, "No report files found"): + charts.main(["messenger-all", "--reports-dir", temp_dir]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/python/requirements.txt b/.github/scripts/python/requirements.txt new file mode 100644 index 0000000000..8b6f0d08f1 --- /dev/null +++ b/.github/scripts/python/requirements.txt @@ -0,0 +1 @@ +matplotlib==3.10.* diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 9ba40daf57..e1d06b9013 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -488,11 +488,45 @@ jobs: path: downloaded-artifacts/ merge-multiple: false + - name: Set up Node.js for report rendering + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: .github/scripts/js/package.json + + - name: Set up Python for chart rendering + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: .github/scripts/python/requirements.txt + + - name: Install JS report deps + run: npm install + working-directory: .github/scripts/js + + - name: Install Python chart deps + run: python3 -m pip install -r .github/scripts/python/requirements.txt + + - name: Show installed Python tooling + run: | + python3 -V + python3 -m pip list + + - name: Generate messenger chart files + run: >- + python3 .github/scripts/python/e2e_report/charts.py messenger-all + --reports-dir downloaded-artifacts + --out-dir tmp/messenger-charts + --manifest tmp/messenger-charts/manifest.json + - name: Send results to channel id: render-report uses: actions/github-script@v7 env: EXPECTED_STORAGE_TYPES: '["replicated","nfs"]' + CHARTS_MANIFEST: tmp/messenger-charts/manifest.json LOOP_API_BASE_URL: ${{ secrets.LOOP_API_BASE_URL }} LOOP_CHANNEL_ID: ${{ secrets.LOOP_CHANNEL_ID }} LOOP_TOKEN: ${{ secrets.LOOP_TOKEN }} @@ -500,3 +534,17 @@ jobs: script: | const renderMessengerReport = require('./.github/scripts/js/e2e/report/messenger-report'); await renderMessengerReport({core}); + + - name: Render top-5 slowest Describes per cluster + run: >- + python3 .github/scripts/python/e2e_report/charts.py top + --reports-dir downloaded-artifacts + --out-dir tmp/charts + --top-n 5 + + - name: Upload top-5 slowest Describe charts + uses: actions/upload-artifact@v4 + with: + name: e2e-report-slowest-by-describe + path: tmp/charts/ + if-no-files-found: warn diff --git a/Taskfile.yaml b/Taskfile.yaml index 95f2670d15..e698b063ed 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -50,6 +50,36 @@ tasks: - which jq >/dev/null || (echo "jq not found."; exit 1) silent: true + report:render:slowest: + desc: "Render slowest-specs chart for one Describe (DESCRIBE=..., JSON=...)" + silent: true + vars: + JSON: '{{.JSON | default ""}}' + DESCRIBE: '{{.DESCRIBE | default ""}}' + OUT_DIR: '{{.OUT_DIR | default "tmp"}}' + cmds: + - test -n "{{.JSON}}" || (echo "JSON=... is required"; exit 1) + - test -n "{{.DESCRIBE}}" || (echo "DESCRIBE=... is required"; exit 1) + - >- + python3 .github/scripts/python/e2e_report/charts.py slowest + --json "{{.JSON}}" + --describe "{{.DESCRIBE}}" --out-dir "{{.OUT_DIR}}/charts" + + report:render:top-slowest: + desc: "Render slowest-specs charts for top-N slowest Describes" + silent: true + vars: + REPORTS_DIR: '{{.REPORTS_DIR | default "tmp/test-ci/report/out"}}' + OUT_DIR: '{{.OUT_DIR | default "tmp"}}' + TOP_N: "{{.TOP_N | default 5}}" + cmds: + - mkdir -p "{{.OUT_DIR}}" "{{.REPORTS_DIR}}" + - >- + python3 .github/scripts/python/e2e_report/charts.py top + --reports-dir "{{.REPORTS_DIR}}" + --out-dir "{{.OUT_DIR}}/charts" + --top-n "{{.TOP_N}}" + check-helm: cmds: - which helm >/dev/null || (echo "helm not found."; exit 1)