diff --git a/.github/scripts/js/e2e/report/cluster-report.js b/.github/scripts/js/e2e/report/cluster-report.js index e1df27c7d3..5cd550470c 100644 --- a/.github/scripts/js/e2e/report/cluster-report.js +++ b/.github/scripts/js/e2e/report/cluster-report.js @@ -69,11 +69,27 @@ const { */ const workflowStages = [ - { name: "bootstrap", displayName: "Bootstrap cluster", needsJobId: "bootstrap" }, - { name: "configure-sdn", displayName: "Configure SDN", needsJobId: "configure-sdn" }, - { name: "storage-setup", displayName: "Configure storage", needsJobId: "configure-storage" }, - { name: "virtualization-setup", displayName: "Configure Virtualization", needsJobId: "configure-virtualization" }, - { name: "e2e-test", displayName: "E2E test", needsJobId: "e2e-test" }, + { + name: "bootstrap", + displayName: "Bootstrap cluster", + needsJobId: "bootstrap", + }, + { + name: "configure-sdn", + displayName: "Configure SDN", + needsJobId: "configure-sdn", + }, + { + name: "storage-setup", + displayName: "Configure storage", + needsJobId: "configure-storage", + }, + { + name: "virtualization-setup", + displayName: "Configure Virtualization", + needsJobId: "configure-virtualization", + }, + { name: "e2e-test", displayName: "E2E test", needsJobId: "e2e-test" }, ]; function readClusterReportConfigFromEnv(env = process.env) { @@ -87,7 +103,11 @@ function readClusterReportConfigFromEnv(env = process.env) { }; } -const requiredClusterReportConfigKeys = ["storageType", "reportsDir", "reportFile"]; +const requiredClusterReportConfigKeys = [ + "storageType", + "reportsDir", + "reportFile", +]; function requireClusterReportConfig(config) { for (const key of requiredClusterReportConfigKeys) { @@ -128,7 +148,9 @@ async function listWorkflowRunJobs(github, context) { } function findWorkflowJob(jobs, pipelineJobName, jobName) { - const nestedJobName = pipelineJobName ? `${pipelineJobName} / ${jobName}` : ""; + const nestedJobName = pipelineJobName + ? `${pipelineJobName} / ${jobName}` + : ""; return ( jobs.find((job) => job.name === nestedJobName) || @@ -147,7 +169,8 @@ function readStageResultsFromEnv(env = process.env) { const stageResults = {}; for (const { name, needsJobId } of workflowStages) { - stageResults[name] = String((needs[needsJobId] || {}).result || "").trim() || "skipped"; + stageResults[name] = + String((needs[needsJobId] || {}).result || "").trim() || "skipped"; } return stageResults; } @@ -161,7 +184,9 @@ async function readStageJobUrlsFromApi(github, context, config, core) { if (job) { stageJobUrls[name] = job.html_url || ""; } else { - core.warning(`Unable to find workflow job "${displayName}" for E2E report`); + core.warning( + `Unable to find workflow job "${displayName}" for E2E report` + ); } } @@ -178,6 +203,8 @@ async function readStageJobUrlsFromApi(github, context, config, core) { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: null, * source: string, * }} Empty parsed-report payload. @@ -187,6 +214,8 @@ function emptyParsedReport(source) { metrics: zeroMetrics(), failedTests: [], failedTestDetails: [], + specTimings: [], + suiteTotalMs: 0, startedAt: null, source, }; @@ -217,6 +246,8 @@ const ginkgoOutputSource = { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: string|null, * }} parse Parser function for the source content. * @property {function(string): RegExp} pattern Builds the file-name regex for the source. @@ -252,6 +283,8 @@ function findGinkgoSource(config, source) { * metrics: ReturnType, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array>, + * suiteTotalMs: number, * startedAt: string|null, * source: string, * }} Parsed report payload with a source tag. @@ -320,6 +353,8 @@ function buildReportPayload({ metrics: parsedReport.metrics, failedTests: parsedReport.failedTests, failedTestDetails: parsedReport.failedTestDetails, + specTimings: parsedReport.specTimings || [], + suiteTotalMs: parsedReport.suiteTotalMs || 0, sourceReport: sourcePath, reportSource: parsedReport.source, }; @@ -391,7 +426,9 @@ async function buildClusterReport({ core, context, github, config } = {}) { ? null : findGinkgoSource(resolvedConfig, ginkgoOutputSource); const sourcePath = rawReportPath || outputPath; - const sourceDescriptor = rawReportPath ? ginkgoJsonSource : ginkgoOutputSource; + const sourceDescriptor = rawReportPath + ? ginkgoJsonSource + : ginkgoOutputSource; if (!rawReportPath) { core.warning( diff --git a/.github/scripts/js/e2e/report/cluster-report.test.js b/.github/scripts/js/e2e/report/cluster-report.test.js index c8bbd2b34f..dc8486be1d 100644 --- a/.github/scripts/js/e2e/report/cluster-report.test.js +++ b/.github/scripts/js/e2e/report/cluster-report.test.js @@ -47,14 +47,12 @@ function createContext() { * @returns {Record} Mocked GitHub client. */ function createGithub(jobNames) { - const jobs = jobNames.map( - (name, index) => ({ - name, - html_url: `https://github.com/test/repo/actions/runs/12345/job/${ - index + 1 - }`, - }) - ); + const jobs = jobNames.map((name, index) => ({ + name, + html_url: `https://github.com/test/repo/actions/runs/12345/job/${ + index + 1 + }`, + })); return { rest: { @@ -238,11 +236,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "success" }, - "configure-storage": { result: "success" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "success" }, + "configure-storage": { result: "success" }, "configure-virtualization": { result: "success" }, - "e2e-test": { result: "success" }, + "e2e-test": { result: "success" }, }); expect(readClusterReportConfigFromEnv()).toMatchObject({ @@ -275,11 +273,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "failure" }, - "configure-storage": { result: "skipped" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "failure" }, + "configure-storage": { result: "skipped" }, "configure-virtualization": { result: "skipped" }, - "e2e-test": { result: "skipped" }, + "e2e-test": { result: "skipped" }, }); const report = await buildClusterReport({ @@ -305,11 +303,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "failure" }, - "configure-storage": { result: "skipped" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "failure" }, + "configure-storage": { result: "skipped" }, "configure-virtualization": { result: "skipped" }, - "e2e-test": { result: "skipped" }, + "e2e-test": { result: "skipped" }, }); const report = await buildClusterReport({ @@ -341,11 +339,11 @@ describe("cluster-report", () => { process.env.REPORTS_DIR = tempDir; process.env.REPORT_FILE = reportFile; process.env.NEEDS_CONTEXT = JSON.stringify({ - "bootstrap": { result: "success" }, - "configure-sdn": { result: "success" }, - "configure-storage": { result: "success" }, + bootstrap: { result: "success" }, + "configure-sdn": { result: "success" }, + "configure-storage": { result: "success" }, "configure-virtualization": { result: "success" }, - "e2e-test": { result: "success" }, + "e2e-test": { result: "success" }, }); const report = await buildClusterReport({ @@ -454,11 +452,43 @@ describe("cluster-report", () => { reason: "timed out waiting for VM to become ready", }, ]); + expect(report.suiteTotalMs).toBe(1800000); + expect(report.specTimings).toEqual([ + { + name: "passes", + group: "Suite", + state: "passed", + runtimeMs: 60000, + labels: [], + }, + { + name: "fails & burns", + group: "Suite", + state: "failed", + runtimeMs: 60000, + labels: ["Slow"], + }, + { + name: "errors ", + group: "Other", + state: "errors", + runtimeMs: 60000, + labels: [], + }, + { + name: "skipped", + group: "skipped", + state: "skipped", + runtimeMs: 60000, + labels: [], + }, + ]); expect(report.reportSource).toBe("ginkgo-json"); expect(report.sourceReport).toBe(rawReportPath); - expect(JSON.parse(fs.readFileSync(reportFile, "utf8")).reportKind).toBe( - "tests" - ); + const persistedReport = JSON.parse(fs.readFileSync(reportFile, "utf8")); + expect(persistedReport.reportKind).toBe("tests"); + expect(persistedReport.specTimings).toEqual(report.specTimings); + expect(persistedReport.suiteTotalMs).toBe(1800000); expect(core.setOutput).toHaveBeenCalledWith("report_file", reportFile); expect(core.setOutput).toHaveBeenCalledWith("report_kind", "tests"); expect(core.setOutput).toHaveBeenCalledWith("status", "failure"); @@ -590,7 +620,7 @@ describe("cluster-report", () => { "/home/runner/work/virtualization/virtualization/test/e2e/e2e_test.go:44", "PASS: all 94 specs have precheck labels", " STEP: Ensuring 12 precreated CVIs are available @ 05/14/26 13:57:36.142", - " CVI \"v12n-e2e-testdata-iso\" exists but not ready (phase: Pending), waiting...", + ' CVI "v12n-e2e-testdata-iso" exists but not ready (phase: Pending), waiting...', " [FAILED] in [SynchronizedBeforeSuite] - /home/runner/work/.../until.go:207 @ 05/14/26 14:02:37.61", "[SynchronizedBeforeSuite] [FAILED] [307.964 seconds]", "[SynchronizedBeforeSuite]", @@ -636,7 +666,9 @@ describe("cluster-report", () => { ); // The plain "[SynchronizedBeforeSuite]" header that follows the // "[FAILED] [307.964 seconds]" line must not leak into the reason. - expect(detail.reason.split("\n")[0]).not.toBe("[SynchronizedBeforeSuite]"); + expect(detail.reason.split("\n")[0]).not.toBe( + "[SynchronizedBeforeSuite]" + ); })); test("fails when multiple matching Ginkgo JSON reports exist", async () => @@ -828,6 +860,8 @@ describe("cluster-report", () => { successRate: 92.78, }); expect(parsed.startedAt).toBe("2026-04-28T03:11:27.708387575Z"); + expect(parsed.specTimings).toHaveLength(131); + expect(parsed.suiteTotalMs).toBe(1800000); expect(parsed.failedTests).toHaveLength(7); expect(parsed.failedTests).toContain( "[It] VirtualMachineOperationRestore restores a virtual machine from a snapshot BestEffort restore mode; automatic restart approval mode; manual run policy [Slow]" diff --git a/.github/scripts/js/e2e/report/messenger-report.js b/.github/scripts/js/e2e/report/messenger-report.js index fb2c6906b2..20eec223be 100644 --- a/.github/scripts/js/e2e/report/messenger-report.js +++ b/.github/scripts/js/e2e/report/messenger-report.js @@ -14,6 +14,7 @@ const fs = require("fs"); const { listMatchingFiles } = require("./shared/fs-utils"); const { REPORT_FILE_PATTERN } = require("./shared/report-model"); +const { renderClusterCharts } = require("./messenger/charts/chart-renderer"); const { makeThreadedReportInLoop } = require("./messenger/loop-client"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { @@ -103,12 +104,19 @@ function readReports(reportsDir, configuredClusters, core) { * @param {MessengerMessagesParams} params Message rendering inputs. * @returns {{ * message: string, - * threadMessages: string[] + * threadMessages: Array<{message: string, files: Array>}> * }} Rendered markdown payloads. */ -function buildMessengerMessages({ reportsDir, configuredClusters, core }) { +async function buildMessengerMessages({ + reportsDir, + configuredClusters, + core, +}) { const orderedReports = readReports(reportsDir, configuredClusters, core); - const threadMessages = buildThreadMessages(orderedReports); + const threadMessages = await buildThreadMessages(orderedReports, { + renderClusterCharts, + core, + }); return { message: buildMainMessage(orderedReports), threadMessages, @@ -122,12 +130,12 @@ function buildMessengerMessages({ reportsDir, configuredClusters, core }) { * @param {RenderMessengerReportParams} params GitHub script dependencies. * @returns {Promise<{ * message: string, - * threadMessages: string[] + * threadMessages: Array<{message: string, files: Array>}> * }>} Rendered messages. */ async function renderMessengerReport({ core, reportsDir }) { const config = readMessengerConfigFromEnv(); - const { message, threadMessages } = buildMessengerMessages({ + const { message, threadMessages } = await buildMessengerMessages({ reportsDir: reportsDir || config.reportsDir, configuredClusters: config.configuredClusters, core, @@ -135,13 +143,22 @@ async function renderMessengerReport({ core, reportsDir }) { core.info(message); core.setOutput("message", message); - core.setOutput("thread_messages", JSON.stringify(threadMessages)); + core.setOutput( + "thread_messages", + JSON.stringify(threadMessages.map((threadMessage) => threadMessage.message)) + ); if (config.loop) { try { - await makeThreadedReportInLoop({ message, threadMessages, loop: config.loop }, core); + await makeThreadedReportInLoop( + { message, threadMessages, loop: config.loop }, + core + ); } catch (error) { core.warning(`Unable to deliver report to Loop API: ${error.message}`); + if (config.loop.strictDelivery) { + throw error; + } } } diff --git a/.github/scripts/js/e2e/report/messenger-report.test.js b/.github/scripts/js/e2e/report/messenger-report.test.js index 429a7c4f58..1d96e047b7 100644 --- a/.github/scripts/js/e2e/report/messenger-report.test.js +++ b/.github/scripts/js/e2e/report/messenger-report.test.js @@ -13,7 +13,12 @@ const fs = require("fs"); const path = require("path"); +jest.mock("./messenger/charts/chart-renderer", () => ({ + renderClusterCharts: jest.fn().mockResolvedValue([]), +})); + const renderMessengerReport = require("./messenger-report"); +const { renderClusterCharts } = require("./messenger/charts/chart-renderer"); const { readMessengerConfigFromEnv } = require("./messenger/config"); const { createCore, withTempDir } = require("./shared/test-utils"); @@ -26,7 +31,11 @@ describe("messenger-report", () => { delete process.env.LOOP_API_BASE_URL; delete process.env.LOOP_CHANNEL_ID; delete process.env.LOOP_TOKEN; + delete process.env.LOOP_STRICT_DELIVERY; + delete process.env.LOOP_STRICT_FILE_UPLOAD; delete global.fetch; + renderClusterCharts.mockReset(); + renderClusterCharts.mockResolvedValue([]); }); test("reads normalized messenger config from env", () => { @@ -44,6 +53,8 @@ describe("messenger-report", () => { apiUrl: "https://loop.example.invalid/api/v4/posts", channelId: "channel-id", token: "token", + strictDelivery: false, + strictFileUploads: false, }, }); }); @@ -136,17 +147,21 @@ describe("messenger-report", () => { expect(result.message).toContain( "- [nfs](https://example.invalid/nfs): CONFIGURE SDN" ); + expect(result.message).not.toContain("### Top slowest tests"); expect(result.message).not.toContain("### Failed tests"); expect(result.threadMessages).toEqual([ - [ - "### Failed tests", - "", - "**[replicated](https://example.invalid/replicated)**", - "", - "| Tests | Reason |", - "|---|---|", - "| fails | Unexpected error: command timed out occurred |", - ].join("\n"), + { + message: [ + "### Failed tests", + "", + "**[replicated](https://example.invalid/replicated)**", + "", + "| Tests | Reason |", + "|---|---|", + "| fails | Unexpected error: command timed out occurred |", + ].join("\n"), + files: [], + }, ]); })); @@ -164,6 +179,116 @@ describe("messenger-report", () => { expect(result.threadMessages).toEqual([]); })); + test("attaches duration chart files to thread reply without a text caption", async () => + inTempDir(async (tempDir) => { + const chartFile = { + name: "replicated-top-slowest.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }; + renderClusterCharts.mockResolvedValue([chartFile]); + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 3, + skipped: 0, + failed: 0, + errors: 0, + total: 3, + successRate: 100, + }, + failedTests: [], + specTimings: [ + { name: "fast", group: "VM", state: "passed", runtimeMs: 1000 }, + { + name: "slow | pipe", + group: "Disk", + state: "passed", + runtimeMs: 90000, + }, + { name: "medium", group: "VM", state: "passed", runtimeMs: 30000 }, + ], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + + const core = createCore(); + const result = await renderMessengerReport({ core }); + + expect(result.message).not.toContain("### Top slowest tests"); + expect(result.threadMessages).toEqual([ + { + message: "**[replicated](https://example.invalid/replicated)**", + files: [chartFile], + }, + ]); + expect(result.threadMessages[0].message).not.toContain( + "### Test durations" + ); + expect(result.threadMessages[0].message).not.toContain( + "Attached charts:" + ); + expect(core.setOutput).toHaveBeenCalledWith( + "thread_messages", + JSON.stringify([result.threadMessages[0].message]) + ); + })); + + test("warns and surfaces a placeholder when chart rendering fails", async () => + inTempDir(async (tempDir) => { + renderClusterCharts.mockRejectedValue(new Error("canvas unavailable")); + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 1, + skipped: 0, + failed: 0, + errors: 0, + total: 1, + successRate: 100, + }, + failedTests: [], + specTimings: [ + { name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }, + ], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + + const core = createCore(); + const result = await renderMessengerReport({ core }); + + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining( + "Unable to render duration charts for cluster replicated" + ) + ); + expect(result.threadMessages).toEqual([ + { + message: expect.stringContaining("Charts unavailable."), + files: [], + }, + ]); + })); + test("warns and skips report files that are missing storageType/cluster fields", async () => inTempDir(async (tempDir) => { fs.writeFileSync( @@ -264,8 +389,16 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.threadMessages).toEqual([ - "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| replicated | — |", - "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + { + message: + "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| replicated | — |", + files: [], + }, + { + message: + "**[nfs](https://example.invalid/nfs)**\n\n| Tests | Reason |\n|---|---|\n| nfs | — |", + files: [], + }, ]); })); @@ -306,16 +439,19 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); expect(result.threadMessages).toEqual([ - [ - "### Failed tests", - "", - "**[nfs](https://example.invalid/nfs)**", - "", - "| Tests | Reason |", - "|---|---|", - "| VirtualMachineOperationRestore | — |", - "| VirtualMachineAdditionalNetworkInterfaces | — |", - ].join("\n"), + { + message: [ + "### Failed tests", + "", + "**[nfs](https://example.invalid/nfs)**", + "", + "| Tests | Reason |", + "|---|---|", + "| VirtualMachineOperationRestore | — |", + "| VirtualMachineAdditionalNetworkInterfaces | — |", + ].join("\n"), + files: [], + }, ]); })); @@ -448,6 +584,12 @@ describe("messenger-report", () => { test("posts main report and per-cluster failed tests thread via Loop API", async () => inTempDir(async (tempDir) => { + const chartFile = { + name: "replicated-top-slowest.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }; + renderClusterCharts.mockResolvedValue([chartFile]); fs.writeFileSync( path.join(tempDir, "e2e_report_replicated.json"), JSON.stringify({ @@ -466,6 +608,9 @@ describe("messenger-report", () => { successRate: 83.33, }, failedTests: ["[It] fails"], + specTimings: [ + { name: "slow", group: "VM", state: "failed", runtimeMs: 90000 }, + ], }) ); @@ -482,6 +627,11 @@ describe("messenger-report", () => { status: 201, text: async () => JSON.stringify({ id: "root-post-id" }), }) + .mockResolvedValueOnce({ + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-id" }] }), + }) .mockResolvedValueOnce({ ok: true, status: 201, @@ -490,7 +640,7 @@ describe("messenger-report", () => { const result = await renderMessengerReport({ core: createCore() }); - expect(global.fetch).toHaveBeenCalledTimes(2); + expect(global.fetch).toHaveBeenCalledTimes(3); expect(global.fetch).toHaveBeenNthCalledWith( 1, "https://loop.example.invalid/api/v4/posts", @@ -506,11 +656,29 @@ describe("messenger-report", () => { channel_id: "channel-id", message: result.message, }); - expect(JSON.parse(global.fetch.mock.calls[1][1].body)).toEqual({ + expect(global.fetch).toHaveBeenNthCalledWith( + 2, + "https://loop.example.invalid/api/v4/files", + expect.objectContaining({ + method: "POST", + headers: { + Authorization: "Bearer loop-token", + }, + }) + ); + expect(JSON.parse(global.fetch.mock.calls[2][1].body)).toEqual({ channel_id: "channel-id", - message: - "### Failed tests\n\n**[replicated](https://example.invalid/replicated)**\n\n| Tests | Reason |\n|---|---|\n| fails | — |", + message: [ + "### Failed tests", + "", + "**[replicated](https://example.invalid/replicated)**", + "", + "| Tests | Reason |", + "|---|---|", + "| fails | — |", + ].join("\n"), root_id: "root-post-id", + file_ids: ["file-id"], }); })); @@ -654,4 +822,47 @@ describe("messenger-report", () => { "Unable to deliver report to Loop API: Loop API request failed with status 500: server exploded" ); })); + + test("fails local delivery when strict Loop delivery mode is enabled", async () => + inTempDir(async (tempDir) => { + fs.writeFileSync( + path.join(tempDir, "e2e_report_replicated.json"), + JSON.stringify({ + cluster: "replicated", + storageType: "replicated", + reportKind: "tests", + branch: "main", + workflowRunUrl: "https://example.invalid/replicated", + startedAt: "2026-04-15T09:30:44", + metrics: { + passed: 11, + skipped: 0, + failed: 0, + errors: 0, + total: 11, + successRate: 100, + }, + failedTests: [], + }) + ); + + process.env.REPORTS_DIR = tempDir; + process.env.EXPECTED_STORAGE_TYPES = '["replicated"]'; + process.env.LOOP_API_BASE_URL = "https://loop.example.invalid"; + process.env.LOOP_CHANNEL_ID = "channel-id"; + process.env.LOOP_TOKEN = "loop-token"; + process.env.LOOP_STRICT_DELIVERY = "1"; + + global.fetch = jest.fn().mockResolvedValue({ + ok: false, + status: 500, + text: async () => "server exploded", + }); + + await expect( + renderMessengerReport({ core: createCore() }) + ).rejects.toThrow( + "Loop API request failed with status 500: server exploded" + ); + })); }); diff --git a/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap new file mode 100644 index 0000000000..2a9e9877e6 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/__snapshots__/chart-config.test.js.snap @@ -0,0 +1,220 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`chart-config builds deterministic duration histogram config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#a371f7", + "data": Array [ + 1, + 1, + 0, + 1, + 1, + ], + "label": "Specs", + }, + ], + "labels": Array [ + "0-30s", + "30-60s", + "60-300s", + "300-600s", + ">600s", + ], + }, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E spec duration distribution", + }, + }, + "responsive": false, + "scales": Object { + "y": Object { + "beginAtZero": true, + "ticks": Object { + "precision": 0, + }, + }, + }, + }, + "type": "bar", + }, + "name": "duration-histogram", +} +`; + +exports[`chart-config builds deterministic feature totals config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#f0883e", + "data": Array [ + 601, + 311, + 60, + ], + "label": "Total duration, seconds", + }, + ], + "labels": Array [ + "Network", + "VM", + "Disk", + ], + }, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E duration by feature", + }, + }, + "responsive": false, + "scales": Object { + "y": Object { + "beginAtZero": true, + }, + }, + }, + "type": "bar", + }, + "name": "feature-totals", +} +`; + +exports[`chart-config builds deterministic status stacked config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#3fb950", + "data": Array [ + 0, + 0, + 10, + ], + "label": "passed", + }, + Object { + "backgroundColor": "#f85149", + "data": Array [ + 0, + 0, + 301, + ], + "label": "failed", + }, + Object { + "backgroundColor": "#d29922", + "data": Array [ + 0, + 601, + 0, + ], + "label": "errors", + }, + Object { + "backgroundColor": "#8b949e", + "data": Array [ + 60, + 0, + 0, + ], + "label": "skipped", + }, + ], + "labels": Array [ + "Disk", + "Network", + "VM", + ], + }, + "options": Object { + "animation": false, + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "E2E duration by feature and status", + }, + }, + "responsive": false, + "scales": Object { + "x": Object { + "stacked": true, + }, + "y": Object { + "beginAtZero": true, + "stacked": true, + }, + }, + }, + "type": "bar", + }, + "name": "status-stacked", +} +`; + +exports[`chart-config builds deterministic top-N config 1`] = ` +Object { + "config": Object { + "data": Object { + "datasets": Array [ + Object { + "backgroundColor": "#58a6ff", + "data": Array [ + 601, + 301, + 60, + ], + "label": "Duration, seconds", + }, + ], + "labels": Array [ + "error", + "slow fail", + "medium skip", + ], + }, + "options": Object { + "animation": false, + "indexAxis": "y", + "plugins": Object { + "legend": Object { + "display": true, + }, + "title": Object { + "display": true, + "text": "Top slowest E2E specs", + }, + }, + "responsive": false, + "scales": Object { + "x": Object { + "beginAtZero": true, + }, + }, + }, + "type": "bar", + }, + "name": "top-slowest", +} +`; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js new file mode 100644 index 0000000000..7072697c05 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.js @@ -0,0 +1,222 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const statusColors = { + passed: "#3fb950", + failed: "#f85149", + errors: "#d29922", + skipped: "#8b949e", +}; + +function normalizeTiming(timing) { + return { + name: String(timing.name || "Unnamed spec"), + group: String(timing.group || timing.name || "Ungrouped"), + state: String(timing.state || "errors"), + runtimeMs: Math.max(0, Number(timing.runtimeMs || 0)), + }; +} + +function seconds(ms) { + return Number((ms / 1000).toFixed(2)); +} + +function baseOptions(title, extra = {}) { + return { + responsive: false, + animation: false, + plugins: { + title: { + display: true, + text: title, + }, + legend: { + display: true, + }, + }, + ...extra, + }; +} + +function sortByRuntimeDesc(left, right) { + return ( + right.runtimeMs - left.runtimeMs || left.name.localeCompare(right.name) + ); +} + +function groupTotals(specTimings) { + const totals = new Map(); + for (const rawTiming of specTimings || []) { + const timing = normalizeTiming(rawTiming); + if (!totals.has(timing.group)) { + totals.set(timing.group, 0); + } + totals.set(timing.group, totals.get(timing.group) + timing.runtimeMs); + } + + return Array.from(totals, ([group, runtimeMs]) => ({ + group, + runtimeMs, + })).sort( + (left, right) => + right.runtimeMs - left.runtimeMs || left.group.localeCompare(right.group) + ); +} + +function buildTopNConfig(specTimings, n = 15) { + const timings = (specTimings || []) + .map(normalizeTiming) + .sort(sortByRuntimeDesc) + .slice(0, n); + + return { + name: "top-slowest", + config: { + type: "bar", + data: { + labels: timings.map((timing) => timing.name), + datasets: [ + { + label: "Duration, seconds", + data: timings.map((timing) => seconds(timing.runtimeMs)), + backgroundColor: "#58a6ff", + }, + ], + }, + options: baseOptions("Top slowest E2E specs", { + indexAxis: "y", + scales: { + x: { beginAtZero: true }, + }, + }), + }, + }; +} + +function buildDurationHistogramConfig( + specTimings, + buckets = [30, 60, 300, 600, Infinity] +) { + const counts = buckets.map(() => 0); + for (const rawTiming of specTimings || []) { + const durationSeconds = + Number(normalizeTiming(rawTiming).runtimeMs || 0) / 1000; + const bucketIndex = buckets.findIndex( + (bucket) => durationSeconds <= bucket + ); + counts[bucketIndex >= 0 ? bucketIndex : buckets.length - 1] += 1; + } + + let previous = 0; + const labels = buckets.map((bucket) => { + const label = + bucket === Infinity ? `>${previous}s` : `${previous}-${bucket}s`; + previous = bucket; + return label; + }); + + return { + name: "duration-histogram", + config: { + type: "bar", + data: { + labels, + datasets: [ + { + label: "Specs", + data: counts, + backgroundColor: "#a371f7", + }, + ], + }, + options: baseOptions("E2E spec duration distribution", { + scales: { + y: { beginAtZero: true, ticks: { precision: 0 } }, + }, + }), + }, + }; +} + +function buildFeatureTotalsConfig(specTimings) { + const totals = groupTotals(specTimings); + + return { + name: "feature-totals", + config: { + type: "bar", + data: { + labels: totals.map((entry) => entry.group), + datasets: [ + { + label: "Total duration, seconds", + data: totals.map((entry) => seconds(entry.runtimeMs)), + backgroundColor: "#f0883e", + }, + ], + }, + options: baseOptions("E2E duration by feature", { + scales: { + y: { beginAtZero: true }, + }, + }), + }, + }; +} + +function buildStatusStackedConfig(specTimings) { + const groups = new Map(); + for (const rawTiming of specTimings || []) { + const timing = normalizeTiming(rawTiming); + if (!groups.has(timing.group)) { + groups.set(timing.group, { passed: 0, failed: 0, errors: 0, skipped: 0 }); + } + const status = Object.prototype.hasOwnProperty.call( + statusColors, + timing.state + ) + ? timing.state + : "errors"; + groups.get(timing.group)[status] += timing.runtimeMs; + } + + const labels = Array.from(groups.keys()).sort(); + const statuses = ["passed", "failed", "errors", "skipped"]; + + return { + name: "status-stacked", + config: { + type: "bar", + data: { + labels, + datasets: statuses.map((status) => ({ + label: status, + data: labels.map((label) => seconds(groups.get(label)[status])), + backgroundColor: statusColors[status], + })), + }, + options: baseOptions("E2E duration by feature and status", { + scales: { + x: { stacked: true }, + y: { beginAtZero: true, stacked: true }, + }, + }), + }, + }; +} + +module.exports = { + buildDurationHistogramConfig, + buildFeatureTotalsConfig, + buildStatusStackedConfig, + buildTopNConfig, +}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js new file mode 100644 index 0000000000..adc26ca8db --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-config.test.js @@ -0,0 +1,43 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + buildDurationHistogramConfig, + buildFeatureTotalsConfig, + buildStatusStackedConfig, + buildTopNConfig, +} = require("./chart-config"); + +const specTimings = [ + { name: "fast pass", group: "VM", state: "passed", runtimeMs: 10_000 }, + { name: "medium skip", group: "Disk", state: "skipped", runtimeMs: 60_000 }, + { name: "slow fail", group: "VM", state: "failed", runtimeMs: 301_000 }, + { name: "error", group: "Network", state: "errors", runtimeMs: 601_000 }, +]; + +describe("chart-config", () => { + test("builds deterministic top-N config", () => { + expect(buildTopNConfig(specTimings, 3)).toMatchSnapshot(); + }); + + test("builds deterministic duration histogram config", () => { + expect(buildDurationHistogramConfig(specTimings)).toMatchSnapshot(); + }); + + test("builds deterministic feature totals config", () => { + expect(buildFeatureTotalsConfig(specTimings)).toMatchSnapshot(); + }); + + test("builds deterministic status stacked config", () => { + expect(buildStatusStackedConfig(specTimings)).toMatchSnapshot(); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js new file mode 100644 index 0000000000..919a4d0db6 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.js @@ -0,0 +1,75 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { + buildDurationHistogramConfig, + buildFeatureTotalsConfig, + buildStatusStackedConfig, + buildTopNConfig, +} = require("./chart-config"); + +let canvasInstance; + +// Module-level singleton: ChartJSNodeCanvas startup (loading chart.js + setting +// up the cairo-backed canvas) is non-trivial, and the renderer is stateless +// between renderToBuffer calls. Reusing it across clusters keeps memory usage +// flat when the messenger report grows. +function loadChartRenderer() { + if (!canvasInstance) { + const { ChartJSNodeCanvas } = require("chartjs-node-canvas"); + canvasInstance = new ChartJSNodeCanvas({ + width: 1280, + height: 720, + backgroundColour: "#ffffff", + }); + } + + return canvasInstance; +} + +function sanitizeFilenamePart(value) { + const fallback = "cluster"; + const safe = String(value || fallback).replace(/[^a-zA-Z0-9_-]+/g, "_"); + return safe || fallback; +} + +async function renderClusterCharts(report) { + if ( + !Array.isArray(report && report.specTimings) || + report.specTimings.length === 0 + ) { + return []; + } + + const renderer = loadChartRenderer(); + const configs = [ + buildTopNConfig(report.specTimings), + buildDurationHistogramConfig(report.specTimings), + buildFeatureTotalsConfig(report.specTimings), + buildStatusStackedConfig(report.specTimings), + ]; + const clusterName = sanitizeFilenamePart( + report.cluster || report.storageType || "cluster" + ); + + return Promise.all( + configs.map(async ({ name, config }) => ({ + name: `${clusterName}-${name}.png`, + buffer: await renderer.renderToBuffer(config, "image/png"), + mimeType: "image/png", + })) + ); +} + +module.exports = { + renderClusterCharts, +}; diff --git a/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js new file mode 100644 index 0000000000..3551db51ec --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/charts/chart-renderer.test.js @@ -0,0 +1,57 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +jest.mock("chartjs-node-canvas", () => ({ + ChartJSNodeCanvas: jest.fn().mockImplementation(() => ({ + renderToBuffer: jest.fn().mockResolvedValue(Buffer.from("png")), + })), +})); + +const { renderClusterCharts } = require("./chart-renderer"); + +describe("chart-renderer", () => { + test("returns no files when spec timings are empty", async () => { + await expect(renderClusterCharts({ specTimings: [] })).resolves.toEqual([]); + }); + + test("renders four cluster chart images", async () => { + const files = await renderClusterCharts({ + cluster: "replicated", + specTimings: [ + { name: "slow", group: "VM", state: "passed", runtimeMs: 90000 }, + ], + }); + + expect(files).toEqual([ + { + name: "replicated-top-slowest.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + { + name: "replicated-duration-histogram.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + { + name: "replicated-feature-totals.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + { + name: "replicated-status-stacked.png", + buffer: Buffer.from("png"), + mimeType: "image/png", + }, + ]); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/config.js b/.github/scripts/js/e2e/report/messenger/config.js index 12fa7dc659..cb31abde04 100644 --- a/.github/scripts/js/e2e/report/messenger/config.js +++ b/.github/scripts/js/e2e/report/messenger/config.js @@ -57,6 +57,10 @@ function parseConfiguredClusters(value) { } } +function parseBooleanEnv(value) { + return ["1", "true", "yes"].includes(String(value || "").toLowerCase()); +} + /** * Reads Loop credentials from the environment. * @@ -66,7 +70,7 @@ function parseConfiguredClusters(value) { * mistake and should surface as an error rather than a silent no-op. * * @param {NodeJS.ProcessEnv} [env=process.env] Environment variables source. - * @returns {{ apiUrl: string, channelId: string, token: string } | null} + * @returns {{ apiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null} */ function readLoopConfig(env = process.env) { const apiUrl = normalizeLoopApiBaseUrl(env.LOOP_API_BASE_URL); @@ -81,7 +85,13 @@ function readLoopConfig(env = process.env) { "LOOP_CHANNEL_ID, LOOP_TOKEN, and LOOP_API_BASE_URL are required" ); } - return { apiUrl, channelId, token }; + return { + apiUrl, + channelId, + token, + strictDelivery: parseBooleanEnv(env.LOOP_STRICT_DELIVERY), + strictFileUploads: parseBooleanEnv(env.LOOP_STRICT_FILE_UPLOAD), + }; } /** @@ -91,7 +101,7 @@ function readLoopConfig(env = process.env) { * @returns {{ * reportsDir: string, * configuredClusters: string[], - * loop: { apiUrl: string, channelId: string, token: string } | null + * loop: { apiUrl: string, channelId: string, token: string, strictDelivery: boolean, strictFileUploads: boolean } | null * }} Normalized messenger configuration. */ function readMessengerConfigFromEnv(env = process.env) { diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.js b/.github/scripts/js/e2e/report/messenger/loop-client.js index 647651bea0..c6e369d131 100644 --- a/.github/scripts/js/e2e/report/messenger/loop-client.js +++ b/.github/scripts/js/e2e/report/messenger/loop-client.js @@ -26,8 +26,8 @@ /** * @typedef {Object} LoopPublishParams * @property {string} message - * @property {string[]} threadMessages - * @property {LoopCredentials} loop + * @property {Array<{message: string, files: Array<{name: string, buffer: Buffer, mimeType: string}>}>} threadMessages + * @property {LoopCredentials & {strictFileUploads?: boolean}} loop */ /** @@ -60,20 +60,24 @@ function parseLoopApiPayload(responseText, core) { * @param {string} message Post body. * @param {string} [rootId] Optional thread root id for reply posts. * @param {LoopClientCore} core GitHub core API. + * @param {string[]} [fileIds] Uploaded Loop file ids to attach. * @returns {Promise>} Parsed Loop API response. */ -async function postToLoopApi(loop, message, rootId, core) { +async function postToLoopApi(loop, message, rootId, core, fileIds = []) { + const body = { + channel_id: loop.channelId, + message, + ...(rootId ? { root_id: rootId } : {}), + ...(fileIds.length > 0 ? { file_ids: fileIds } : {}), + }; + const response = await fetch(loop.apiUrl, { method: "POST", headers: { Authorization: `Bearer ${loop.token}`, "Content-Type": "application/json", }, - body: JSON.stringify({ - channel_id: loop.channelId, - message, - ...(rootId ? { root_id: rootId } : {}), - }), + body: JSON.stringify(body), }); const responseText = await response.text(); @@ -88,6 +92,59 @@ async function postToLoopApi(loop, message, rootId, core) { return payload; } +function getFilesApiUrl(apiUrl) { + return String(apiUrl || "").replace(/\/posts$/, "/files"); +} + +/** + * Uploads a single file to Loop and returns the created file id. + * + * @param {LoopCredentials} loop Loop API credentials. + * @param {string} fileName File name shown in Loop. + * @param {Buffer} buffer File content. + * @param {LoopClientCore} core GitHub core API. + * @param {string} [mimeType] File MIME type. + * @returns {Promise} Uploaded Loop file id. + */ +async function uploadFileToLoop( + loop, + fileName, + buffer, + core, + mimeType = "image/png" +) { + const formData = new FormData(); + formData.append("channel_id", loop.channelId); + formData.append("files", new Blob([buffer], { type: mimeType }), fileName); + + const response = await fetch(getFilesApiUrl(loop.apiUrl), { + method: "POST", + headers: { + Authorization: `Bearer ${loop.token}`, + }, + body: formData, + }); + const responseText = await response.text(); + + if (!response.ok) { + throw new Error( + `Loop file upload failed with status ${response.status}: ${responseText}` + ); + } + + const payload = parseLoopApiPayload(responseText, core); + const fileId = + payload.file_infos && payload.file_infos[0] && payload.file_infos[0].id; + if (!fileId) { + throw new Error("Loop API did not return uploaded file id"); + } + + core.info( + `Loop API accepted file ${fileName} with status ${response.status}` + ); + return fileId; +} + /** * Publishes the main report and optional failed-tests thread to Loop. * @@ -95,7 +152,10 @@ async function postToLoopApi(loop, message, rootId, core) { * @param {LoopClientCore} core GitHub core API. * @returns {Promise} */ -async function makeThreadedReportInLoop({ message, threadMessages, loop }, core) { +async function makeThreadedReportInLoop( + { message, threadMessages, loop }, + core +) { const rootPost = await postToLoopApi(loop, message, undefined, core); if (!rootPost.id) { @@ -104,11 +164,36 @@ async function makeThreadedReportInLoop({ message, threadMessages, loop }, core) ); } - for (const replyMessage of threadMessages) { - await postToLoopApi(loop, replyMessage, rootPost.id, core); + for (const reply of threadMessages) { + const files = Array.isArray(reply.files) ? reply.files : []; + let fileIds = []; + if (files.length > 0) { + try { + fileIds = await Promise.all( + files.map((file) => + uploadFileToLoop(loop, file.name, file.buffer, core, file.mimeType) + ) + ); + } catch (error) { + if (loop.strictFileUploads) { + throw error; + } + + // Posting the reply without attachments is preferable to losing the + // whole thread (e.g. failed-tests table) when Loop rejects file + // uploads, typically with HTTP 403 when the bot token lacks the + // upload_file permission. + core.warning( + `Loop file upload failed; posting reply without attachments: ${error.message}` + ); + fileIds = []; + } + } + await postToLoopApi(loop, reply.message, rootPost.id, core, fileIds); } } module.exports = { makeThreadedReportInLoop, + uploadFileToLoop, }; diff --git a/.github/scripts/js/e2e/report/messenger/loop-client.test.js b/.github/scripts/js/e2e/report/messenger/loop-client.test.js new file mode 100644 index 0000000000..de64cf72a0 --- /dev/null +++ b/.github/scripts/js/e2e/report/messenger/loop-client.test.js @@ -0,0 +1,242 @@ +// Copyright 2026 Flant JSC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const { uploadFileToLoop, makeThreadedReportInLoop } = require("./loop-client"); +const { createCore } = require("../shared/test-utils"); + +describe("loop-client", () => { + afterEach(() => { + delete global.fetch; + }); + + test("uploads files to Loop multipart endpoint", async () => { + global.fetch = jest.fn().mockResolvedValue({ + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-id" }] }), + }); + + const fileId = await uploadFileToLoop( + { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }, + "chart.png", + Buffer.from("image-bytes"), + createCore(), + "image/png" + ); + + expect(fileId).toBe("file-id"); + expect(global.fetch).toHaveBeenCalledWith( + "https://loop.example.invalid/api/v4/files", + expect.objectContaining({ + method: "POST", + headers: { + Authorization: "Bearer loop-token", + }, + }) + ); + + const body = global.fetch.mock.calls[0][1].body; + expect(body.get("channel_id")).toBe("channel-id"); + expect(body.get("files").name).toBe("chart.png"); + await expect(body.get("files").text()).resolves.toBe("image-bytes"); + }); + + test("posts the reply with uploaded chart file ids", async () => { + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }; + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-one" }] }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ file_infos: [{ id: "file-two" }] }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "top-slowest.png", + buffer: Buffer.from("one"), + mimeType: "image/png", + }, + { + name: "status-stacked.png", + buffer: Buffer.from("two"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + createCore() + ); + + expect(global.fetch).toHaveBeenCalledTimes(4); + expect(JSON.parse(global.fetch.mock.calls[3][1].body)).toEqual({ + channel_id: "channel-id", + message: "reply", + root_id: "root-post-id", + file_ids: ["file-one", "file-two"], + }); + }); + + test("posts the reply without attachments when file upload fails", async () => { + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + }; + const core = createCore(); + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: false, + status: 403, + text: async () => + JSON.stringify({ + id: "api.context.permissions.app_error", + message: "permission denied", + }), + }, + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "reply-post-id" }), + }, + ]; + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "chart.png", + buffer: Buffer.from("image-bytes"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + core + ); + + expect(global.fetch).toHaveBeenCalledTimes(3); + expect(global.fetch.mock.calls[0][0]).toBe(loop.apiUrl); + expect(global.fetch.mock.calls[1][0]).toBe( + "https://loop.example.invalid/api/v4/files" + ); + expect(global.fetch.mock.calls[2][0]).toBe(loop.apiUrl); + + const replyBody = JSON.parse(global.fetch.mock.calls[2][1].body); + expect(replyBody.root_id).toBe("root-post-id"); + expect(replyBody.message).toBe("reply"); + expect(replyBody).not.toHaveProperty("file_ids"); + + expect(core.warning).toHaveBeenCalledWith( + expect.stringContaining( + "Loop file upload failed; posting reply without attachments" + ) + ); + }); + + test("fails when strict file upload mode is enabled", async () => { + const loop = { + apiUrl: "https://loop.example.invalid/api/v4/posts", + channelId: "channel-id", + token: "loop-token", + strictFileUploads: true, + }; + const responses = [ + { + ok: true, + status: 201, + text: async () => JSON.stringify({ id: "root-post-id" }), + }, + { + ok: false, + status: 403, + text: async () => "permission denied", + }, + ]; + global.fetch = jest + .fn() + .mockImplementation(() => Promise.resolve(responses.shift())); + + await expect( + makeThreadedReportInLoop( + { + message: "main", + threadMessages: [ + { + message: "reply", + files: [ + { + name: "chart.png", + buffer: Buffer.from("image-bytes"), + mimeType: "image/png", + }, + ], + }, + ], + loop, + }, + createCore() + ) + ).rejects.toThrow( + "Loop file upload failed with status 403: permission denied" + ); + expect(global.fetch).toHaveBeenCalledTimes(2); + }); +}); diff --git a/.github/scripts/js/e2e/report/messenger/markdown.js b/.github/scripts/js/e2e/report/messenger/markdown.js index 893b15e45a..29cc6c9948 100644 --- a/.github/scripts/js/e2e/report/messenger/markdown.js +++ b/.github/scripts/js/e2e/report/messenger/markdown.js @@ -337,28 +337,82 @@ function renderFailedTestsThreadMessage(report) { return lines.join("\n"); } +function hasSpecTimings(report) { + return Array.isArray(report.specTimings) && report.specTimings.length > 0; +} + +function renderChartCaption(_files, chartsUnavailable) { + return chartsUnavailable ? "Charts unavailable." : ""; +} + /** - * Builds optional failed-tests thread messages for clusters with failed tests. + * Builds optional per-cluster thread messages for failed tests and duration charts. * * @param {Array>} orderedReports Cluster reports in display order. - * @returns {string[]} Markdown thread message bodies. + * @param {{ + * renderClusterCharts?: function(Record): Promise>, + * core?: {warning?: function(string): void} + * }} [options] + * @returns {Promise}>>} Markdown thread payloads. */ -function buildThreadMessages(orderedReports) { +async function buildThreadMessages( + orderedReports, + { renderClusterCharts, core } = {} +) { const testsReports = orderedReports.filter((report) => isTestResultReport(report) ); - const failedTestReports = testsReports.filter(hasFailedTests); + const threadMessages = []; + let renderedFailedTestsHeading = false; - if (failedTestReports.length === 0) { - return []; + for (const report of testsReports) { + const messageParts = []; + let files = []; + let chartsUnavailable = false; + + if (renderClusterCharts && hasSpecTimings(report)) { + try { + files = await renderClusterCharts(report); + } catch (error) { + chartsUnavailable = true; + if (core && typeof core.warning === "function") { + core.warning( + `Unable to render duration charts for cluster ${ + getReportClusterKey(report) || "unknown" + }: ${error.message}` + ); + } + } + } + + if (!hasFailedTests(report) && files.length === 0 && !chartsUnavailable) { + continue; + } + + if (hasFailedTests(report)) { + const clusterMessage = renderFailedTestsThreadMessage(report); + messageParts.push( + renderedFailedTestsHeading + ? clusterMessage + : ["### Failed tests", clusterMessage].join("\n\n") + ); + renderedFailedTestsHeading = true; + } else { + messageParts.push(`**${formatClusterLink(report)}**`); + } + + const chartCaption = renderChartCaption(files, chartsUnavailable); + if (chartCaption) { + messageParts.push(chartCaption); + } + + threadMessages.push({ + message: messageParts.join("\n\n"), + files, + }); } - return failedTestReports.map((report, index) => { - const clusterMessage = renderFailedTestsThreadMessage(report); - return index === 0 - ? ["### Failed tests", clusterMessage].join("\n\n") - : clusterMessage; - }); + return threadMessages; } module.exports = { diff --git a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js index 3d3f5b2940..eca0925605 100644 --- a/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js +++ b/.github/scripts/js/e2e/report/shared/ginkgo-report-utils.js @@ -72,10 +72,12 @@ function formatSpecName(specReport) { .map((part) => String(part || "").trim()) .filter(Boolean); const leafText = String(specReport.LeafNodeText || "").trim(); - const labels = [...new Set([ - ...flattenLabels(specReport.ContainerHierarchyLabels), - ...flattenLabels(specReport.LeafNodeLabels), - ])]; + const labels = [ + ...new Set([ + ...flattenLabels(specReport.ContainerHierarchyLabels), + ...flattenLabels(specReport.LeafNodeLabels), + ]), + ]; const labelSuffix = labels.map((label) => `[${label}]`).join(" "); const body = [...hierarchyParts, leafText].filter(Boolean).join(" "); @@ -86,6 +88,11 @@ function formatSpecName(specReport) { .trim(); } +function runtimeMs(value) { + const runtime = Number(value || 0); + return Number.isFinite(runtime) ? Math.round(runtime / 1_000_000) : 0; +} + /** * Maps a raw Ginkgo spec state into the metrics bucket used by the final * messenger report. @@ -124,12 +131,16 @@ function formatFailureReason(specReport) { const failureStates = new Set(["failed", "errors"]); function isSuiteNodeFailure(specReport) { - const leafNodeType = String((specReport && specReport.LeafNodeType) || "").trim(); + const leafNodeType = String( + (specReport && specReport.LeafNodeType) || "" + ).trim(); if (!leafNodeType || leafNodeType === "It") { return false; } - return failureStates.has(getMetricKeyForState(specReport && specReport.State)); + return failureStates.has( + getMetricKeyForState(specReport && specReport.State) + ); } function buildFailureDetail(specReport) { @@ -153,6 +164,8 @@ function buildFailureDetail(specReport) { * metrics: GinkgoMetrics, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: Array<{name: string, group: string, state: string, runtimeMs: number, labels: string[]}>, + * suiteTotalMs: number, * startedAt: string|null * }} Parsed report payload. */ @@ -161,10 +174,14 @@ function parseGinkgoReport(jsonContent) { const metrics = zeroMetrics(); const failedTests = []; const failedTestDetails = []; + const specTimings = []; const startedAt = suites.find((suite) => suite && suite.StartTime)?.StartTime || null; + let suiteTotalMs = 0; for (const suite of suites) { + suiteTotalMs += runtimeMs(suite && suite.RunTime); + for (const specReport of toArray(suite && suite.SpecReports)) { if (isSuiteNodeFailure(specReport)) { const failureDetail = buildFailureDetail(specReport); @@ -186,6 +203,20 @@ function parseGinkgoReport(jsonContent) { metrics.total += 1; const metricKey = getMetricKeyForState(specReport.State); metrics[metricKey] += 1; + const hierarchyParts = toArray(specReport.ContainerHierarchyTexts) + .map((part) => String(part || "").trim()) + .filter(Boolean); + const leafText = String(specReport.LeafNodeText || "").trim(); + specTimings.push({ + name: leafText, + group: hierarchyParts[0] || leafText, + state: metricKey, + runtimeMs: runtimeMs(specReport.RunTime), + labels: flattenLabels([ + ...toArray(specReport.ContainerHierarchyLabels), + ...toArray(specReport.LeafNodeLabels), + ]), + }); if (failureStates.has(metricKey)) { const failureDetail = buildFailureDetail(specReport); @@ -214,6 +245,8 @@ function parseGinkgoReport(jsonContent) { ]) ).values() ), + specTimings, + suiteTotalMs, startedAt, }; } @@ -330,6 +363,8 @@ function extractFailureReasonFromOutput(output, suiteNodeType) { * metrics: GinkgoMetrics, * failedTests: string[], * failedTestDetails: Array<{name: string, reason: string}>, + * specTimings: [], + * suiteTotalMs: 0, * startedAt: null, * }} Parsed fallback payload. */ @@ -340,6 +375,8 @@ function parseGinkgoOutput(outputContent) { metrics: zeroMetrics(), failedTests: [], failedTestDetails: [], + specTimings: [], + suiteTotalMs: 0, startedAt: null, }; diff --git a/.github/scripts/js/e2e/report/shared/report-model.js b/.github/scripts/js/e2e/report/shared/report-model.js index 3e8523e9e0..19fedf5cf2 100644 --- a/.github/scripts/js/e2e/report/shared/report-model.js +++ b/.github/scripts/js/e2e/report/shared/report-model.js @@ -54,12 +54,12 @@ function ginkgoOutputPattern(storageType) { } const stageMessage = { - "bootstrap": "BOOTSTRAP CLUSTER", + bootstrap: "BOOTSTRAP CLUSTER", "configure-sdn": "CONFIGURE SDN", "storage-setup": "STORAGE SETUP", "virtualization-setup": "VIRTUALIZATION SETUP", "e2e-test": "E2E TEST", - "ready": "CLUSTER READY", + ready: "CLUSTER READY", "artifact-missing": "TEST REPORTS NOT FOUND", }; @@ -94,7 +94,8 @@ const statusMessageTemplates = { }; function buildStatusMessage(status, stageLabel) { - const template = statusMessageTemplates[status] || statusMessageTemplates.failure; + const template = + statusMessageTemplates[status] || statusMessageTemplates.failure; return template.replace("%s", stageLabel); } diff --git a/.github/scripts/js/package.json b/.github/scripts/js/package.json index 7d57cc285c..b038118441 100644 --- a/.github/scripts/js/package.json +++ b/.github/scripts/js/package.json @@ -19,5 +19,9 @@ "eslint": "^10.2.1", "jest": "28.1.2", "prettier": "^2.5.0" + }, + "dependencies": { + "chart.js": "^4.5.1", + "chartjs-node-canvas": "^5.0.0" } } diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml index 9ba40daf57..814a70dda8 100644 --- a/.github/workflows/e2e-matrix.yml +++ b/.github/workflows/e2e-matrix.yml @@ -488,6 +488,21 @@ jobs: path: downloaded-artifacts/ merge-multiple: false + # Node 20 is required because actions/github-script@v7 ships with Node 20 + # and chartjs-node-canvas pulls a native `canvas` binding whose ABI must + # match the runtime that ultimately requires it. Bump together with + # actions/github-script when v8 (Node 22) is available. + - name: Set up Node.js for report rendering + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + cache-dependency-path: .github/scripts/js/package-lock.json + + - name: Install E2E report dependencies + run: npm ci + working-directory: .github/scripts/js + - name: Send results to channel id: render-report uses: actions/github-script@v7