From c70c9d612a19092dcb03a1c8fae8afe3f23f4b25 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 08:11:43 +0000 Subject: [PATCH 1/7] feat(imports): publish versioned workbench findings through the shared Cloud transport --- docs/scanner-imports/schema-guide.md | 2 + .../normalized-import/guardian-response.json | 32 ++ .../normalized-import/guardian-selection.json | 13 + examples/normalized-import/trivy-report.json | 33 ++ .../codex-security/mcp-app/helpers-main.ts | 10 +- .../mcp-app/src/workbench/commands.ts | 23 +- .../external-finding-publications.ts | 105 +++++ .../src/workbench/external-findings.ts | 267 ++++++++--- .../mcp-app/src/workbench/migrations.ts | 5 +- .../mcp-app/tests/test_external_findings.ts | 180 ++++++- .../mcp-app/tests/test_workbench_database.ts | 9 +- .../mcp-app/tests/test_workbench_findings.ts | 5 +- .../shared/workbench-migrations.json | 11 + .../codex-security/tests/test_workbench_db.py | 3 + .../test_workbench_setup_and_migrations.py | 9 +- sdk/typescript/docs/sdk.md | 62 +++ sdk/typescript/scripts/package-dist-files.mjs | 3 + sdk/typescript/src/external-findings-local.ts | 439 ++++++++++++++++-- .../src/external-findings-publish.ts | 8 +- sdk/typescript/src/external-findings.ts | 6 +- .../src/external-publication-review.ts | 34 ++ sdk/typescript/src/index.ts | 13 + sdk/typescript/src/opengrep-findings.ts | 78 +++- sdk/typescript/src/runtime.ts | 5 +- .../src/semgrep-platform-findings.ts | 35 ++ sdk/typescript/src/sonar-findings.ts | 14 +- sdk/typescript/src/sonar-import.ts | 69 ++- .../src/stored-finding-projection.ts | 138 ++++++ .../src/stored-findings-publication.ts | 318 +++++++++++++ sdk/typescript/src/tenable-findings.ts | 2 +- sdk/typescript/src/veracode-findings.ts | 16 +- .../external-findings-publish.test.ts | 369 ++++++++++++++- .../inspector-repository-import.test.ts | 2 +- .../tests-ts/local-finding-report.test.ts | 160 +++++++ .../tests-ts/opengrep-findings.test.ts | 40 ++ .../semgrep-guardian-findings.test.ts | 13 + .../semgrep-platform-collector.test.ts | 24 +- .../semgrep-platform-selection-file.test.ts | 21 +- sdk/typescript/tests-ts/sonar-import.test.ts | 22 +- .../stored-finding-projection.test.ts | 85 ++++ .../veracode-repository-import.test.ts | 6 +- 41 files changed, 2461 insertions(+), 228 deletions(-) create mode 100644 examples/normalized-import/guardian-response.json create mode 100644 examples/normalized-import/guardian-selection.json create mode 100644 examples/normalized-import/trivy-report.json create mode 100644 plugins/codex-security/mcp-app/src/workbench/external-finding-publications.ts create mode 100644 sdk/typescript/src/external-publication-review.ts create mode 100644 sdk/typescript/src/stored-finding-projection.ts create mode 100644 sdk/typescript/src/stored-findings-publication.ts create mode 100644 sdk/typescript/tests-ts/local-finding-report.test.ts create mode 100644 sdk/typescript/tests-ts/stored-finding-projection.test.ts diff --git a/docs/scanner-imports/schema-guide.md b/docs/scanner-imports/schema-guide.md index 7767d62ac1..67dd6015e0 100644 --- a/docs/scanner-imports/schema-guide.md +++ b/docs/scanner-imports/schema-guide.md @@ -143,6 +143,8 @@ With the SDK, repeat the preparation call that created the submission, using the same options. For `prepareExternalPublication()`, supply the original export file. For `prepareNormalizedExternalPublication()`, supply the original `ExternalFindings`. Check `preview.resumed` before calling `publish()`. +For stored findings, reprepare the interrupted source group with its saved selection +and review ID. Completed groups already have their own receipts. **Upgrading from an earlier import client.** Finish pending Cloud submissions with the CLI or SDK build that created their checkpoints. Keep the original input and diff --git a/examples/normalized-import/guardian-response.json b/examples/normalized-import/guardian-response.json new file mode 100644 index 0000000000..63c1f0f4a4 --- /dev/null +++ b/examples/normalized-import/guardian-response.json @@ -0,0 +1,32 @@ +{ + "isError": false, + "content": [], + "structuredContent": { + "findings": [ + { + "id": "9007199254740992", + "repository": { "name": "example/project" }, + "ref": "refs/heads/main", + "issueType": "sast", + "rulePath": "demo.other-rule", + "message": "Another finding in the tool response", + "severity": "LOW", + "status": "open", + "filePath": "src/other.ts", + "line": 3 + }, + { + "id": "9007199254740993", + "repository": { "name": "example/project" }, + "ref": "refs/heads/main", + "issueType": "sast", + "rulePath": "demo.unchecked-path", + "message": "A user-supplied path reaches the file reader", + "severity": "HIGH", + "status": "open", + "filePath": "src/read.ts", + "line": 12 + } + ] + } +} diff --git a/examples/normalized-import/guardian-selection.json b/examples/normalized-import/guardian-selection.json new file mode 100644 index 0000000000..90e7d3c2de --- /dev/null +++ b/examples/normalized-import/guardian-selection.json @@ -0,0 +1,13 @@ +{ + "sourceKey": "demo/guardian/example/project/main/sast", + "sourceRepositoryUrl": "https://github.com/example/project", + "request": { + "name": "get_semgrep_sast_findings", + "arguments": { + "deployment_name": "demo", + "repository_name": "example/project", + "ref": "refs/heads/main" + } + }, + "findingIds": ["9007199254740993"] +} diff --git a/examples/normalized-import/trivy-report.json b/examples/normalized-import/trivy-report.json new file mode 100644 index 0000000000..0d096433d8 --- /dev/null +++ b/examples/normalized-import/trivy-report.json @@ -0,0 +1,33 @@ +{ + "SchemaVersion": 2, + "Trivy": { "Version": "0.76.1" }, + "ReportID": "00000000-0000-7000-8000-000000000001", + "CreatedAt": "2026-01-01T00:00:00Z", + "ArtifactType": "repository", + "ArtifactName": "https://github.com/example/project", + "Metadata": { + "RepoURL": "https://github.com/example/project", + "Branch": "main", + "Commit": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "Results": [ + { + "Target": "package-lock.json", + "Class": "lang-pkgs", + "Type": "npm", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2099-0001", + "PkgName": "@example/parser", + "PkgID": "@example/parser@1.0.0", + "InstalledVersion": "1.0.0", + "FixedVersion": "1.0.1", + "Title": "Example parser accepts an invalid length", + "Description": "A crafted length can cause excessive allocation.", + "Severity": "HIGH", + "PrimaryURL": "https://advisories.example.invalid/CVE-2099-0001" + } + ] + } + ] +} diff --git a/plugins/codex-security/mcp-app/helpers-main.ts b/plugins/codex-security/mcp-app/helpers-main.ts index c2e2d4b1b0..337e38f8dd 100644 --- a/plugins/codex-security/mcp-app/helpers-main.ts +++ b/plugins/codex-security/mcp-app/helpers-main.ts @@ -80,8 +80,14 @@ function runHelper(): void { "Usage: list-stored-findings\nReads a JSON object from stdin with an absolute stateDirectory, positive payload.limit and non-negative payload.offset.", "store-external-findings": "Usage: store-external-findings\nReads a JSON object from stdin with an absolute stateDirectory and payload containing repository, provider, sourceKey and findings with source_finding_id and evidence.", - "list-external-findings": - "Usage: list-external-findings\nReads a JSON object from stdin with an absolute stateDirectory and payload.repository; payload.id optionally selects one external finding.", + "read-external-findings": + "Usage: read-external-findings\nReads a JSON object from stdin with an absolute stateDirectory and payload.repository. Select an id, an importId, or a list of findings with id and version.", + "list-external-imports": + "Usage: list-external-imports\nReads a JSON object from stdin with an absolute stateDirectory and payload.repository.", + "record-external-finding-publications": + "Usage: record-external-finding-publications\nReads a JSON object from stdin with an absolute stateDirectory and publication receipts for stored finding versions.", + "list-external-finding-publications": + "Usage: list-external-finding-publications\nReads a JSON object from stdin with an absolute stateDirectory and payload.repository. Select an optional finding id.", "find-potential-duplicates": "Usage: find-potential-duplicates\nReads a JSON object from stdin with an absolute stateDirectory, payload.findingId and payload.scope containing either repositoryId or allRepositories: true.", "store-dedupe-groups": diff --git a/plugins/codex-security/mcp-app/src/workbench/commands.ts b/plugins/codex-security/mcp-app/src/workbench/commands.ts index a9f5b9bc1e..e26527e9ca 100644 --- a/plugins/codex-security/mcp-app/src/workbench/commands.ts +++ b/plugins/codex-security/mcp-app/src/workbench/commands.ts @@ -13,9 +13,16 @@ import { } from "./duplicates"; import { listExternalFindings, + listExternalImports, storeExternalFindings, + type ExternalFindingSelection, type ExternalFindingsImport, } from "./external-findings"; +import { + listExternalFindingPublications, + recordExternalFindingPublications, + type ExternalPublicationLinks, +} from "./external-finding-publications"; import { readSeverityClassification, @@ -84,6 +91,7 @@ export async function workbenchCommand( command !== "local-dedupe" && command !== "store-findings" && command !== "store-external-findings" && + command !== "record-external-finding-publications" && command !== "store-dedupe-groups" && command !== "severity-classification", }, @@ -101,8 +109,21 @@ export async function workbenchCommand( payload as ExternalFindingsImport, new Date().toISOString(), ); - if (command === "list-external-findings") + if (command === "read-external-findings") return listExternalFindings( + database, + payload as ExternalFindingSelection, + ); + if (command === "list-external-imports") + return listExternalImports(database, payload as { repository: string }); + if (command === "record-external-finding-publications") + return recordExternalFindingPublications( + database, + payload as ExternalPublicationLinks, + new Date().toISOString(), + ); + if (command === "list-external-finding-publications") + return listExternalFindingPublications( database, payload as { repository: string; id?: string }, ); diff --git a/plugins/codex-security/mcp-app/src/workbench/external-finding-publications.ts b/plugins/codex-security/mcp-app/src/workbench/external-finding-publications.ts new file mode 100644 index 0000000000..026c26ce23 --- /dev/null +++ b/plugins/codex-security/mcp-app/src/workbench/external-finding-publications.ts @@ -0,0 +1,105 @@ +import type { DatabaseSync } from "node:sqlite"; +import { requireRepository, requireText } from "./external-findings"; +import { transaction } from "./transaction"; + +export interface ExternalPublicationLinks { + repository: string; + cloudApiUrl: string; + accountId: string; + destinationId: string; + entries: { + id: string; + version: number; + receiptId: string; + outcome: "created" | "updated" | "unchanged" | "private_job"; + cloudFindingId?: string; + }[]; +} + +export function recordExternalFindingPublications( + database: DatabaseSync, + input: ExternalPublicationLinks, + timestamp: string, +) { + requireRepository(input?.repository); + requireText(input.cloudApiUrl, "cloudApiUrl"); + requireText(input.accountId, "accountId"); + requireText(input.destinationId, "destinationId"); + if (!Array.isArray(input.entries)) + throw new TypeError("entries must be an array."); + return transaction(database, "BEGIN IMMEDIATE", () => { + // The reviewed version may have been superseded during upload. Every + // positive revision through the current version existed in this repository. + const lookup = database.prepare( + "SELECT 1 FROM external_source_reports WHERE id = ? AND repository = ? AND version >= ?", + ); + const insert = database.prepare( + `INSERT INTO external_finding_publications + (finding_id, version, cloud_api_url, account_id, destination_id, receipt_id, outcome, cloud_finding_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING`, + ); + let recorded = 0; + for (const { + id, + version, + receiptId, + outcome, + cloudFindingId, + } of input.entries) { + requireText(id, "id"); + requireText(receiptId, "receiptId"); + if (cloudFindingId !== undefined) + requireText(cloudFindingId, "cloudFindingId"); + if ( + !Number.isSafeInteger(version) || + version < 1 || + !lookup.get(id, input.repository, version) + ) + throw new Error( + `No finding ${id} at version ${version} in the selected repository.`, + ); + if (!["created", "updated", "unchanged", "private_job"].includes(outcome)) + throw new TypeError("Unknown external publication outcome."); + recorded += Number( + insert.run( + id, + version, + input.cloudApiUrl, + input.accountId, + input.destinationId, + receiptId, + outcome, + cloudFindingId ?? null, + timestamp, + ).changes, + ); + } + return { recorded }; + }); +} + +export function listExternalFindingPublications( + database: DatabaseSync, + input: { repository: string; id?: string }, +) { + requireRepository(input?.repository); + if (input.id !== undefined) requireText(input.id, "id"); + const query = database.prepare( + `SELECT p.finding_id AS id, p.version, p.cloud_api_url AS cloudApiUrl, p.account_id AS accountId, + p.destination_id AS destinationId, p.receipt_id AS receiptId, p.outcome, p.cloud_finding_id AS cloudFindingId, + p.created_at AS createdAt FROM external_finding_publications p + JOIN external_source_reports f ON f.id = p.finding_id WHERE f.repository = ? + ${input.id === undefined ? "" : "AND p.finding_id = ?"} + ORDER BY p.created_at DESC, p.receipt_id, p.finding_id`, + ); + const rows = + input.id === undefined + ? query.all(input.repository) + : query.all(input.repository, input.id); + return { + publications: rows.map(({ cloudFindingId, ...row }) => ({ + ...row, + ...(cloudFindingId == null ? {} : { cloudFindingId }), + })), + }; +} diff --git a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts index 5781296fcd..ab23710e53 100644 --- a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts +++ b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts @@ -1,8 +1,9 @@ -import { hash } from "node:crypto"; +import { hash, randomUUID } from "node:crypto"; import { isAbsolute } from "node:path"; -import type { DatabaseSync, StatementSync } from "node:sqlite"; +import type { DatabaseSync } from "node:sqlite"; import { isDeepStrictEqual } from "node:util"; import { + compareCodePoints, object, parseJson, requireFiniteNumbers, @@ -21,6 +22,20 @@ export interface ExternalFindingsImport { provider: string; sourceKey: string; findings: ExternalFindingInput[]; + provenance?: { + sourceRepositories?: { url: string; platform?: "github" | "gitlab" }[]; + inspectorProjectMap?: [string, string][]; + }; + read?: number; + excluded?: unknown[]; + report?: Record; +} + +export interface ExternalFindingSelection { + repository: string; + id?: string; + importId?: string; + findings?: Pick[]; } export interface StoredExternalFinding { @@ -36,13 +51,18 @@ export interface StoredExternalFinding { updatedAt: string; } -function requireText(value: unknown, name: string): asserts value is string { +export function requireText( + value: unknown, + name: string, +): asserts value is string { if (typeof value !== "string" || value.length === 0) throw new TypeError(`${name} must be a non-empty string.`); requireSqliteText([value]); } -function requireRepository(repository: unknown): asserts repository is string { +export function requireRepository( + repository: unknown, +): asserts repository is string { requireText(repository, "repository"); if (!isAbsolute(repository)) throw new TypeError("repository must be an absolute local path."); @@ -67,11 +87,41 @@ function fromRow(row: Record): StoredExternalFinding { }; } +// A change to the publication context advances the local revision. All imports +// of one finding revision therefore have equivalent provenance. +const findingWithProvenance = `SELECT f.*, ( + SELECT i.provenance_json FROM external_finding_import_members m + JOIN external_finding_imports i ON i.id = m.import_id + WHERE m.finding_id = f.id AND m.version = f.version LIMIT 1 +) AS provenance_json FROM external_source_reports f`; + +// A scanner can enumerate the same repository bindings in a different order. +function serializeProvenance(provenance: ExternalFindingsImport["provenance"]) { + const repositories = provenance?.sourceRepositories; + const projects = provenance?.inspectorProjectMap; + if (!repositories?.length && !projects?.length) return null; + return JSON.stringify({ + sourceRepositories: repositories?.length + ? repositories + .map(({ url, platform }) => ({ url, platform })) + .sort( + (a, b) => + compareCodePoints(a.url, b.url) || + compareCodePoints(a.platform ?? "", b.platform ?? ""), + ) + : undefined, + inspectorProjectMap: projects?.length + ? projects.toSorted(([a], [b]) => compareCodePoints(a, b)) + : undefined, + }); +} + export function storeExternalFindings( database: DatabaseSync, input: ExternalFindingsImport, timestamp: string, ): { + importId?: string; findings: StoredExternalFinding[]; counts: { created: number; updated: number; unchanged: number }; } { @@ -83,6 +133,14 @@ export function storeExternalFindings( throw new TypeError("findings must be an array."); const counts = { created: 0, updated: 0, unchanged: 0 }; if (input.findings.length === 0) return { findings: [], counts }; + const provenanceJson = serializeProvenance(input.provenance); + const selection = { + read: input.read ?? input.findings.length, + excluded: input.excluded ?? [], + report: input.report, + }; + requireFiniteNumbers(selection); + const selectionJson = stringifyJson(selection, 0); // Validate and serialize the complete batch before taking the write lock. const sourceIds = new Set(); @@ -111,74 +169,159 @@ export function storeExternalFindings( }); return transaction(database, "BEGIN IMMEDIATE", () => { - const stored = database.prepare( - "SELECT * FROM external_source_reports WHERE id = ?", + const importId = randomUUID(); + database + .prepare( + `INSERT INTO external_finding_imports + (id, repository, created_at, provider, source_key, provenance_json, selection_json) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + importId, + input.repository, + timestamp, + input.provider, + input.sourceKey, + provenanceJson, + selectionJson, + ); + const stored = database.prepare(`${findingWithProvenance} WHERE f.id = ?`); + const write = database.prepare( + `INSERT INTO external_source_reports + (id, repository, provider, source_key, source_finding_id, evidence_json, version, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?) + ON CONFLICT(id) DO UPDATE SET evidence_json = excluded.evidence_json, + version = external_source_reports.version + 1, updated_at = excluded.updated_at + RETURNING *`, ); - let insert: StatementSync | undefined; - let update: StatementSync | undefined; const findings = entries.map((entry) => { const previous = stored.get(entry.id); - if (!previous) { - insert ??= database.prepare( - `INSERT INTO external_source_reports - (id, repository, provider, source_key, source_finding_id, - evidence_json, version, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)`, - ); - insert.run( - entry.id, - input.repository, - input.provider, - input.sourceKey, - entry.sourceFindingId, - entry.evidenceJson, - timestamp, - timestamp, - ); - counts.created++; - return fromRow(stored.get(entry.id)!); - } - const finding = fromRow(previous); - if (previous.evidence_json !== entry.evidenceJson) { - const evidence = parseEvidence(entry.evidenceJson); - if (!isDeepStrictEqual(finding.evidence, evidence)) { - update ??= database.prepare( - `UPDATE external_source_reports - SET evidence_json = ?, version = version + 1, updated_at = ? - WHERE id = ? RETURNING version, updated_at`, - ); - const row = update.get(entry.evidenceJson, timestamp, entry.id)!; - counts.updated++; - return { - ...finding, - evidence, - version: Number(row.version), - updatedAt: String(row.updated_at), - }; + if (previous && previous.provenance_json === provenanceJson) { + const finding = fromRow(previous); + if ( + previous.evidence_json === entry.evidenceJson || + isDeepStrictEqual(finding.evidence, parseEvidence(entry.evidenceJson)) + ) { + counts.unchanged++; + return finding; } } - counts.unchanged++; - return finding; + const row = write.get( + entry.id, + input.repository, + input.provider, + input.sourceKey, + entry.sourceFindingId, + entry.evidenceJson, + timestamp, + timestamp, + )!; + counts[previous ? "updated" : "created"]++; + return fromRow(row); }); - return { findings, counts }; + const member = database.prepare( + "INSERT INTO external_finding_import_members (import_id, position, finding_id, version) VALUES (?, ?, ?, ?)", + ); + findings.forEach(({ id, version }, position) => + member.run(importId, position, id, version), + ); + return { importId, findings, counts }; }); } +export function listExternalImports( + database: DatabaseSync, + selection: { repository: string }, +) { + requireRepository(selection?.repository); + return { + imports: database + .prepare( + `SELECT i.id AS importId, i.created_at AS createdAt, i.provider, i.source_key AS sourceKey, + (SELECT count(*) FROM external_finding_import_members m WHERE m.import_id = i.id) AS count + FROM external_finding_imports i WHERE i.repository = ? + ORDER BY i.created_at DESC, i.rowid DESC`, + ) + .all(selection.repository), + }; +} + export function listExternalFindings( database: DatabaseSync, - selection: { repository: string; id?: string }, -): { findings: StoredExternalFinding[] } { + selection: ExternalFindingSelection, +) { requireRepository(selection?.repository); - if (selection.id !== undefined) requireText(selection.id, "id"); - const selectByRepository = - "SELECT * FROM external_source_reports WHERE repository = ?"; - const rows = - selection.id === undefined - ? database - .prepare(`${selectByRepository} ORDER BY created_at, id`) - .all(selection.repository) - : database - .prepare(`${selectByRepository} AND id = ?`) - .all(selection.repository, selection.id); - return { findings: rows.map(fromRow) }; + const { repository, id, importId, findings: versions } = selection; + if (importId === undefined && versions === undefined) { + if (id !== undefined) requireText(id, "id"); + const lookup = "SELECT * FROM external_source_reports WHERE repository = ?"; + const rows = + id === undefined + ? database.prepare(`${lookup} ORDER BY created_at, id`).all(repository) + : database.prepare(`${lookup} AND id = ?`).all(repository, id); + return { findings: rows.map(fromRow) }; + } + if (id !== undefined || (importId !== undefined && versions !== undefined)) + throw new TypeError( + "Select either an import or a list of finding versions.", + ); + return transaction(database, "BEGIN", () => { + let summary: Record = {}; + let pins: string, input: string; + if (importId !== undefined) { + requireText(importId, "importId"); + const batch = database + .prepare( + "SELECT selection_json FROM external_finding_imports WHERE repository = ? AND id = ?", + ) + .get(repository, importId); + if (!batch) + throw new Error("Import does not exist in the selected repository."); + summary = parseJson(String(batch.selection_json)) as Record< + string, + unknown + >; + pins = + "SELECT finding_id AS id, version, position FROM external_finding_import_members WHERE import_id = ?"; + input = importId; + } else { + if (!Array.isArray(versions) || versions.length === 0) + throw new TypeError("Select at least one finding version."); + const seen = new Set(); + input = JSON.stringify( + versions.map(({ id, version }) => { + requireText(id, "finding id"); + if (seen.has(id) || !Number.isSafeInteger(version) || version < 1) + throw new TypeError( + "Select each finding once with a positive integer version.", + ); + seen.add(id); + return [id, version]; + }), + ); + pins = + "SELECT value ->> 0 AS id, value ->> 1 AS version, key AS position FROM json_each(?)"; + } + // A left join retains missing and foreign pins so a partial selection can never be approved. + const rows = database + .prepare( + `SELECT p.id AS pinned_id, p.version AS pinned_version, f.* FROM (${pins}) p + LEFT JOIN (${findingWithProvenance}) f ON f.id = p.id AND f.repository = ? + ORDER BY p.position`, + ) + .iterate(input, repository); + const findings = Array.from(rows, (row) => { + if (row.version !== row.pinned_version) + throw new Error( + `Finding ${row.pinned_id} version ${row.pinned_version} is missing or has changed. Review the current selection.`, + ); + return { + ...fromRow(row), + ...(row.provenance_json != null && { + provenance: parseJson(String(row.provenance_json)), + }), + }; + }); + return { read: findings.length, excluded: [], ...summary, findings }; + }); } diff --git a/plugins/codex-security/mcp-app/src/workbench/migrations.ts b/plugins/codex-security/mcp-app/src/workbench/migrations.ts index 6235e7c0a8..03abed1167 100644 --- a/plugins/codex-security/mcp-app/src/workbench/migrations.ts +++ b/plugins/codex-security/mcp-app/src/workbench/migrations.ts @@ -14,9 +14,10 @@ export interface Migration { statements: readonly string[]; } -// Repository-dependent migrations run in Python before semantic workbench operations. +// Python owns the repository-dependent migrations 67–71. Migration 72 only uses +// the external source tables that both workbench runtimes already maintain. export const migrations: readonly Migration[] = history.filter( - (item) => item.version <= 66, + (item) => item.version <= 66 || item.version === 72, ); const migration = (version: number) => migrations.find((item) => item.version === version)!; diff --git a/plugins/codex-security/mcp-app/tests/test_external_findings.ts b/plugins/codex-security/mcp-app/tests/test_external_findings.ts index e2d3974338..10bcb3dd05 100644 --- a/plugins/codex-security/mcp-app/tests/test_external_findings.ts +++ b/plugins/codex-security/mcp-app/tests/test_external_findings.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; +import { hash } from "node:crypto"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { after, test, type TestContext } from "node:test"; @@ -8,14 +8,18 @@ import { fileURLToPath } from "node:url"; import { importSource } from "./import-module.ts"; import { createTemporaryDirectories } from "./support/temporary-directories.ts"; import type * as External from "../src/workbench/external-findings.ts"; +import type * as Publications from "../src/workbench/external-finding-publications.ts"; import type * as Migrations from "../src/workbench/migrations.ts"; -const { storeExternalFindings, listExternalFindings } = (await importSource( - "src/workbench/external-findings.ts", -)) as typeof External; +const { storeExternalFindings, listExternalFindings, listExternalImports } = + (await importSource("src/workbench/external-findings.ts")) as typeof External; const { applyMigrations, migrations } = (await importSource( "src/workbench/migrations.ts", )) as typeof Migrations; +const { recordExternalFindingPublications, listExternalFindingPublications } = + (await importSource( + "src/workbench/external-finding-publications.ts", + )) as typeof Publications; const temporary = createTemporaryDirectories(true); after(() => temporary.cleanup()); const repository = join(process.cwd(), "synthetic-repository"); @@ -101,7 +105,7 @@ test("external imports preserve incomplete evidence without creating assessed fi input.findings[index].source_finding_id, ]; assert.deepEqual(finding, { - id: `ext_${createHash("sha256").update(JSON.stringify(identity)).digest("hex")}`, + id: `ext_${hash("sha256", JSON.stringify(identity))}`, repository, provider: input.provider, sourceKey: input.sourceKey, @@ -144,7 +148,17 @@ test("repeat imports keep identities and unchanged timestamps; changed evidence ], }; const replay = storeExternalFindings(database, reordered, updated); + assert.notEqual(replay.importId, first.importId); + assert.deepEqual( + listExternalFindings(database, { repository, importId: replay.importId }), + { + read: 1, + excluded: [], + findings: [first.findings[0]], + }, + ); assert.deepEqual(replay, { + importId: replay.importId, findings: [first.findings[0]], counts: { created: 0, updated: 0, unchanged: 1 }, }); @@ -221,11 +235,87 @@ test("a large mixed import and its replay return every stored row in input order }); const changes = database.prepare("SELECT total_changes() AS writes"); const beforeReplay = changes.get()!.writes; - assert.deepEqual(storeExternalFindings(database, mixed, created), { + const replay = storeExternalFindings(database, mixed, created); + assert.deepEqual(replay, { + importId: replay.importId, findings: result.findings, counts: { created: 0, updated: 0, unchanged: 513 }, }); - assert.equal(changes.get()!.writes, beforeReplay); + for (const selection of [ + { importId: replay.importId }, + { findings: replay.findings }, + ]) + assert.deepEqual( + listExternalFindings(database, { repository, ...selection }).findings, + result.findings, + ); + // Only the import and its membership were written; finding evidence stays untouched. + assert.equal( + Number(changes.get()!.writes) - Number(beforeReplay), + result.findings.length + 1, + ); +}); + +test("stored selections retain run context, isolate repositories and reject stale evidence or provenance", (t) => { + const database = open(t); + const input = batch(); + input.provenance = { + sourceRepositories: [{ url: "https://example.com/team/repo" }], + }; + input.read = 3; + input.excluded = [ + { position: 3, source_finding_id: null, reason: "missing ID" }, + ]; + input.report = { run: "scanner/run/1" }; + const first = storeExternalFindings(database, input, created); + const expected = first.findings.map((f) => ({ + ...f, + provenance: input.provenance, + })); + const select = ( + choice: Omit, + checkout = repository, + ) => listExternalFindings(database, { repository: checkout, ...choice }); + const asImport = { importId: first.importId }; + const refs = first.findings + .toReversed() + .map(({ id, version }) => ({ id, version })); + const asVersions = { findings: refs }; + assert.deepEqual(select(asImport), { + read: 3, + excluded: input.excluded, + report: input.report, + findings: expected, + }); + assert.deepEqual(select(asVersions).findings, expected.toReversed()); + assert.throws(() => select({ findings: [refs[0], refs[0]] }), /once/u); + const foreign = join(repository, "other"); + assert.throws(() => select(asImport, foreign), /selected repository/u); + assert.throws(() => select(asVersions, foreign), /missing/u); + assert.deepEqual( + listExternalImports(database, { repository: foreign }).imports, + [], + ); + const latest = storeExternalFindings( + database, + { ...input, provenance: undefined }, + updated, + ); + assert.equal(latest.counts.updated, 2); + assert.throws(() => select(asImport), /has changed/u); + assert.throws(() => select(asVersions), /has changed/u); + assert.deepEqual( + listExternalImports(database, { repository }).imports.map((row) => ({ + ...row, + })), + [latest, first].map((x) => ({ + importId: x.importId, + createdAt: x.findings[0].updatedAt, + provider: input.provider, + sourceKey: input.sourceKey, + count: 2, + })), + ); }); test("repository, provider and source namespaces keep distinct finding identities", (t) => { @@ -256,11 +346,39 @@ test("repository, provider and source namespaces keep distinct finding identitie ); }); +test("reordering a scope's repository bindings keeps the same revision", (t) => { + const database = open(t); + const input = batch(); + const refs = [ + { url: "https://example.com/A" }, + { url: "https://example.com/B" }, + ]; + const mappings: [string, string][] = [ + ["arn:A", refs[0].url], + ["arn:B", refs[1].url], + ]; + input.provenance = { + sourceRepositories: refs, + inspectorProjectMap: mappings, + }; + storeExternalFindings(database, input, created); + input.provenance = { + sourceRepositories: refs.toReversed(), + inspectorProjectMap: mappings.toReversed(), + }; + assert.deepEqual(storeExternalFindings(database, input, updated).counts, { + created: 0, + updated: 0, + unchanged: 2, + }); +}); + test("invalid batches and failed writes preserve all previously stored evidence", (t) => { const database = open(t); const input = batch(); storeExternalFindings(database, input, created); const before = listExternalFindings(database, { repository }); + const beforeImports = listExternalImports(database, { repository }); const duplicate = { ...input, findings: [ @@ -297,9 +415,57 @@ test("invalid batches and failed writes preserve all previously stored evidence" /synthetic write failure/u, ); assert.deepEqual(listExternalFindings(database, { repository }), before); + assert.deepEqual( + listExternalImports(database, { repository }), + beforeImports, + ); } }); +test("receipt links retain the actually published revision and replay idempotently", (t) => { + const database = open(t); + const imported = storeExternalFindings(database, batch(), created); + const { id, version } = imported.findings[0]; + const destination = { + cloudApiUrl: "https://service.example", + accountId: "account", + destinationId: "repo-id", + }; + const receipt: Publications.ExternalPublicationLinks["entries"][number] = { + id, + version, + receiptId: "receipt-1", + outcome: "created", + cloudFindingId: "remote-id", + }; + const record = ( + ...entries: Publications.ExternalPublicationLinks["entries"] + ) => + recordExternalFindingPublications( + database, + { repository, ...destination, entries }, + updated, + ).recorded; + const read = (checkout = repository) => + listExternalFindingPublications(database, { repository: checkout }) + .publications; + + const replacement = batch(); + replacement.findings[0].evidence = { title: "New revision while uploading" }; + storeExternalFindings(database, replacement, updated); + assert.throws( + () => record(receipt, { ...receipt, version: 99 }), + /No finding/u, + ); + assert.deepEqual(read(), []); + assert.equal(record(receipt), 1); + assert.equal(record(receipt), 0); + assert.deepEqual(read(), [ + { ...receipt, ...destination, createdAt: updated }, + ]); + assert.deepEqual(read(join(repository, "other")), []); +}); + test("a Python database at the preceding schema upgrades in Node and remains readable in Python", async () => { const directory = await temporary.create("external-migration-"); const path = join(directory, "workbench.sqlite3"); diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts index 41e080eda3..7b9791b5ba 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_database.ts +++ b/plugins/codex-security/mcp-app/tests/test_workbench_database.ts @@ -79,7 +79,7 @@ function assertMigrationNames(database: DatabaseSync, ...versions: number[]) { database .prepare("SELECT name FROM schema_migrations WHERE version = ?") .get(version)?.name, - migrations[version - 1].name, + workbenchMigrations[version - 1].name, ); } @@ -1051,7 +1051,7 @@ test("recorded additive migrations restore missing columns and configured error for (const version of [27, 28, 31, 32, 47, 54, 55, 58, 59, 60, 61]) database .prepare("INSERT INTO schema_migrations VALUES (?, ?, 'original')") - .run(version, migrations[version - 1].name); + .run(version, workbenchMigrations[version - 1].name); applyMigrations(database); const launch = database .prepare( @@ -1488,7 +1488,7 @@ test("Node initialization leaves repository migration semantics to the Python co pythonMigrations(controlPath); await databaseInfo(directory); const initialized = new DatabaseSync(databasePath); - assertMigrationNames(initialized, 48, 49, 53, 54, 55, 56, 60); + assertMigrationNames(initialized, 48, 49, 53, 54, 55, 56, 60, 72); assert.deepEqual( initialized.prepare("SELECT * FROM external_source_reports").all(), originalReports, @@ -1688,6 +1688,9 @@ with sqlite3.connect(sys.argv[2]) as connection: name !== "finding_issue_receipts" && name !== "deep_scan_worker_threads" && name !== "external_source_reports" && + name !== "external_finding_imports" && + name !== "external_finding_import_members" && + name !== "external_finding_publications" && name !== "sqlite_sequence", ) .map(([name, rows]) => { diff --git a/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts b/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts index 23f751da93..81f8c8bc49 100644 --- a/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts +++ b/plugins/codex-security/mcp-app/tests/test_workbench_findings.ts @@ -149,7 +149,10 @@ test("finding helper help exits without reading stdin", async () => { "store-findings", "list-stored-findings", "store-external-findings", - "list-external-findings", + "read-external-findings", + "list-external-imports", + "record-external-finding-publications", + "list-external-finding-publications", "find-potential-duplicates", "store-dedupe-groups", "list-dedupe-groups", diff --git a/plugins/codex-security/shared/workbench-migrations.json b/plugins/codex-security/shared/workbench-migrations.json index 1db8b4ce98..ae49e23c5d 100644 --- a/plugins/codex-security/shared/workbench-migrations.json +++ b/plugins/codex-security/shared/workbench-migrations.json @@ -577,5 +577,16 @@ "statements": [ "ALTER TABLE workspaces ADD COLUMN rejected_checkout_roots_json TEXT;" ] + }, + { + "version": 72, + "name": "retain external finding import selections and provenance", + "statements": [ + "CREATE TABLE external_finding_imports (\n id TEXT PRIMARY KEY,\n repository TEXT NOT NULL,\n created_at TEXT NOT NULL,\n provider TEXT NOT NULL,\n source_key TEXT NOT NULL,\n provenance_json TEXT,\n selection_json TEXT NOT NULL\n);", + "CREATE INDEX external_finding_imports_by_repository ON external_finding_imports (repository, created_at);", + "CREATE TABLE external_finding_import_members (\n import_id TEXT NOT NULL REFERENCES external_finding_imports(id),\n position INTEGER NOT NULL,\n finding_id TEXT NOT NULL REFERENCES external_source_reports(id),\n version INTEGER NOT NULL,\n PRIMARY KEY (import_id, position)\n) WITHOUT ROWID;", + "CREATE INDEX external_finding_members_by_version\nON external_finding_import_members (finding_id, version);", + "CREATE TABLE external_finding_publications (\n finding_id TEXT NOT NULL REFERENCES external_source_reports(id),\n version INTEGER NOT NULL,\n cloud_api_url TEXT NOT NULL,\n account_id TEXT NOT NULL,\n destination_id TEXT NOT NULL,\n receipt_id TEXT NOT NULL,\n outcome TEXT NOT NULL,\n cloud_finding_id TEXT,\n created_at TEXT NOT NULL,\n PRIMARY KEY (finding_id, version, cloud_api_url, account_id, destination_id, receipt_id)\n) WITHOUT ROWID;" + ] } ] diff --git a/plugins/codex-security/tests/test_workbench_db.py b/plugins/codex-security/tests/test_workbench_db.py index 43322dfa34..44829bf64b 100644 --- a/plugins/codex-security/tests/test_workbench_db.py +++ b/plugins/codex-security/tests/test_workbench_db.py @@ -73,6 +73,9 @@ "deep_scan_runs", "deep_scan_worker_threads", "deep_scan_workers", + "external_finding_import_members", + "external_finding_imports", + "external_finding_publications", "external_source_reports", "finding_decisions", "finding_dedupe_group_members", diff --git a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py index 64ea6f4fbb..f2c584b96b 100644 --- a/plugins/codex-security/tests/test_workbench_setup_and_migrations.py +++ b/plugins/codex-security/tests/test_workbench_setup_and_migrations.py @@ -119,6 +119,7 @@ (69, "distinguish unvalidated workspace targets"), (70, "retain checkout roots for Git discovery"), (71, "retain rejected workspace checkout boundaries"), + (72, "retain external finding import selections and provenance"), ] @@ -257,7 +258,8 @@ def test_repository_preview_history_keeps_original_application_times_after_budge assert [ tuple(row) for row in connection.execute( - "SELECT version, applied_at FROM schema_migrations WHERE version >= 67 ORDER BY version" + "SELECT version, applied_at FROM schema_migrations " + "WHERE version BETWEEN 67 AND 71 ORDER BY version" ) ] == [(version, f"preview-{version - 4}") for version in range(67, 72)] assert ( @@ -3279,7 +3281,10 @@ def test_identity_migrations_skip_unavailable_historical_git( (workspace_id, target_id, historical, timestamp, timestamp, timestamp), ) apply_migrations(connection) - assert connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] == 71 + assert ( + connection.execute("SELECT MAX(version) FROM schema_migrations").fetchone()[0] + == EXPECTED_MIGRATIONS[-1][0] + ) assert tuple( connection.execute( "SELECT id, current_path, repository_identity FROM security_targets" diff --git a/sdk/typescript/docs/sdk.md b/sdk/typescript/docs/sdk.md index c30e8b3d74..2e3cc46da3 100644 --- a/sdk/typescript/docs/sdk.md +++ b/sdk/typescript/docs/sdk.md @@ -667,6 +667,68 @@ For issue tracking, see [Linear publication](cli.md#publish-completed-scans-to-l ### Import external findings +For an agent-written parser, use the normalized import API. Pass an absolute +checkout path, a stable source namespace and the repository URL when known. +Count each input occurrence in `read` and supply its evidence or a numbered +exclusion. Keep the upstream occurrence ID as a string. + +```ts +import { prepareNormalizedLocalImport } from "@openai/codex-security"; + +const local = await prepareNormalizedLocalImport( + { + read: 1, + findings: [ + { + source_finding_id: "17", + evidence: { title: "Unsafe eval", severity: "high" }, + }, + ], + excluded: [], + }, + { + repository: checkout, + provider: "example-scanner", + sourceKey: "payments/main", + sourceRepositoryUrl: scmUrl, + }, +); +console.log(local.preview); +const { importId } = await local.import(); // After reviewing the preview. +``` + +A nonempty import returns an `importId`, including when every finding was +unchanged. Use `listLocalExternalImports({ repository })` to find an import. +`selectLocalExternalFindings({ repository, importId })` reads its members and +fails if a finding has since changed. To select individual findings across +imports, pass `{ repository, findings: [{ id, version }] }`. + +```ts +import { prepareStoredFindingsPublication } from "@openai/codex-security"; + +const { groups, excluded } = await prepareStoredFindingsPublication( + { repository: checkout, importId }, + { repository: cloudRepository }, +); +console.log( + groups.map(({ preview }) => preview), + excluded, +); +``` + +Show the user all destinations, findings and exclusions together. For each +approved group, retain its selection and review ID, then call +`group.publish(approvedReviewId)`. Check the returned status, receipts and +per-item errors before proceeding. If a submission is interrupted, reprepare +that group and subsequent unstarted groups from their saved selections and use +the original review IDs. Report a terminal partial result with its exclusions +and errors. If the approved ID was lost or the preview changed, show the new +preview for review. +`listLocalFindingPublications({ repository, id })` returns receipts for a +stored version, including completed private inbox job IDs. Custom provider +names and Tenable can be used locally. Cloud eligibility follows its provider +contract and the selected destination. + For a saved scanner report, call `prepareLocalExternalImport` for a checkout or `prepareExternalPublication` for a connected Cloud repository. Review the returned `preview`, then call `import()` to save locally or `publish()` to diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index 8d9fb29d2f..c975a49cb6 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -60,6 +60,9 @@ const distModules = [ "trivy-findings", "trivy-import", "external-findings-publish", + "external-publication-review", + "stored-finding-projection", + "stored-findings-publication", "private-finding-import", "external-import-contract", "external-import-models", diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts index e99f977ed3..be515d1a9e 100644 --- a/sdk/typescript/src/external-findings-local.ts +++ b/sdk/typescript/src/external-findings-local.ts @@ -1,4 +1,5 @@ import { resolve } from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { semgrepPlatformFindingBinding, semgrepPlatformReportScope, @@ -12,6 +13,7 @@ import { isPublicGitHost, gitTransportBytes, hostedRepositoryMatches, + normalizeHostedGitRepositoryUrl, } from "./hosted-git-repository.js"; import { checkRetainedTrivyProvider, @@ -27,6 +29,7 @@ import { externalImportDisposition, NoPublishableFindingsError, cloneImportPreview, + reportForJson, snapshotImportOptions, snapshotNormalizedFindings, snapshotSourceRepositories, @@ -44,9 +47,10 @@ import type { Provider, } from "./external-import-models.js"; import { + EXTERNAL_FINDINGS_PROVIDERS, + EXTERNAL_FINDINGS_READERS, localFindingRepositories, semgrepSarifRepositories, - type LocalFindingProvider, } from "./external-findings-read.js"; import { tenableReportFormat, @@ -108,7 +112,7 @@ import { export interface StoredExternalFinding { id: string; repository: string; - provider: LocalFindingProvider; + provider: string; sourceKey: string; sourceFindingId: string; evidence: ExternalFindingEvidence; @@ -116,12 +120,61 @@ export interface StoredExternalFinding { version: number; createdAt: string; updatedAt: string; + /** Reader context required to publish this version without the original file. */ + provenance?: ExternalFindingProvenance; +} + +export interface ExternalFindingProvenance { + sourceRepositories?: ExternalSourceRepository[]; + inspectorProjectMap?: [string, string][]; +} + +export type LocalExternalFindingSelection = { repository: string } & ( + | { importId: string; findings?: never } + | { + findings: readonly Pick[]; + importId?: never; + } +); + +export interface SelectedLocalExternalFindings extends Omit< + ExternalFindings, + "findings" +> { + findings: StoredExternalFinding[]; +} + +export interface StoredExternalImport { + importId: string; + createdAt: string; + provider: string; + sourceKey: string; + count: number; +} + +interface LocalFindingReceipt { + id: string; + version: number; + receiptId: string; + outcome: "created" | "updated" | "unchanged" | "private_job"; + cloudFindingId?: string; +} + +interface CloudFindingDestination { + cloudApiUrl: string; + accountId: string; + destinationId: string; +} + +export interface StoredFindingPublication + extends LocalFindingReceipt, CloudFindingDestination { + createdAt: string; } export interface NormalizedLocalImportOptions { /** Local checkout or source directory to associate with these records. */ repository: string; - provider: LocalFindingProvider; + provider: string; /** Stable scanner organization/project/branch namespace across imports. */ sourceKey: string; /** Verified Git repository URL for readers that bind a finding to a checkout. */ @@ -140,10 +193,15 @@ export interface LocalExternalImportDependencies { export interface LocalExternalImportPreview extends ExternalFindings { repository: string; - source: { provider: LocalFindingProvider; source_key: string }; + source: { + provider: string; + source_key: string; + }; } export interface LocalExternalImportResult { + /** Present when the import saved a nonempty selection, including unchanged rows. */ + importId?: string; repository: string; read: number; report?: ExternalFindings["report"]; @@ -168,6 +226,29 @@ function requireIdentity( throw new CodexSecurityError( `${field} must be nonempty and contain no NUL characters.`, ); + if (!value.isWellFormed()) + throw new CodexSecurityError(`${field} must contain valid Unicode.`); +} + +function validateFindingIds(current: ExternalFindings) { + const identities = new Set(); + for (const finding of current.findings) { + requireIdentity(finding.source_finding_id, "Source finding id"); + if (identities.has(finding.source_finding_id)) + throw new CodexSecurityError( + `Duplicate source finding id ${JSON.stringify(finding.source_finding_id)}.`, + ); + identities.add(finding.source_finding_id); + if (!isRecord(finding.evidence)) + throw new CodexSecurityError("Imported evidence must be an object."); + if ( + typeof finding.evidence.title !== "string" || + typeof finding.evidence.severity !== "string" + ) + throw new CodexSecurityError( + "Imported evidence requires a title and severity.", + ); + } } function checkStoredNumber(value: number): void { @@ -541,6 +622,21 @@ export async function configuredFetchRemotes( return remotes; } +async function hostedCheckoutRemotes( + repository: string, + protectedRoots: readonly string[], + environment: WorkbenchCommandOptions["environment"], + signal?: AbortSignal, +) { + if (!(await hasGitMetadata(repository, signal))) return undefined; + return readHostedGitRemotes( + repository, + [...protectedRoots, await canonicalConfigPath(process.cwd())], + environment, + signal, + ); +} + async function requireMatchingRepository( repository: string, provider: Provider, @@ -559,13 +655,9 @@ async function requireMatchingRepository( if (!snyk && sources.length === 0 && hostedSources.length === 0) return false; let allSourcesMatched = true; if (snyk || hostedSources.length) { - if (!(await hasGitMetadata(repository, signal))) return false; - const remotes = await readHostedGitRemotes( + const remotes = await hostedCheckoutRemotes( repository, - [ - ...protectedRoots, - await abortable(() => canonicalConfigPath(process.cwd()), signal), - ], + protectedRoots, environment, signal, ); @@ -663,6 +755,7 @@ export function sonarCheckoutMatchesSource( ); } +/** @internal Resolve and protect the local checkout and workbench state directory. */ async function localContext( selectedRepository: string, dependencies: LocalExternalImportDependencies, @@ -736,11 +829,105 @@ export async function prepareNormalizedLocalImport( options: NormalizedLocalImportOptions, dependencies: LocalExternalImportDependencies = {}, ): Promise { - dependencies.signal?.throwIfAborted(); - return prepareLocalImport( - input, - snapshotImportOptions(options), + const { signal } = dependencies; + signal?.throwIfAborted(); + const settings = snapshotImportOptions(options); + const reader = EXTERNAL_FINDINGS_READERS.find( + (name) => name === settings.provider, + ); + if (reader) + return prepareLocalImport( + input, + { ...settings, provider: EXTERNAL_FINDINGS_PROVIDERS[reader] }, + dependencies, + ); + + const references = snapshotSourceRepositories( + dependencies.sourceRepositories, + ); + requireIdentity(settings.provider, "Provider"); + requireIdentity(settings.sourceKey, "Source key"); + const current = snapshotNormalizedFindings(input, checkStoredNumber); + validateFindingIds(current); + if (current.report !== undefined && !isRecord(current.report)) + throw new CodexSecurityError("Report metadata must be an object."); + if (current.read !== current.findings.length + current.excluded.length) + throw new CodexSecurityError( + "Every source record must be accounted for by a finding or exclusion.", + ); + const positions = new Set(); + for (const { position, source_finding_id, reason } of current.excluded) { + if ( + !Number.isSafeInteger(position) || + position < 1 || + position > current.read || + positions.has(position) + ) + throw new CodexSecurityError( + "An excluded position must be unique and within the reported record count.", + ); + positions.add(position); + requireIdentity(reason, "Exclusion reason"); + if (source_finding_id !== null) + requireIdentity(source_finding_id, "Excluded source finding id"); + } + + const platform = references.find((ref) => ref.platform)?.platform; + const urls = new Set(references.map(({ url }) => url)); + if (settings.sourceRepositoryUrl !== undefined) + urls.add(settings.sourceRepositoryUrl); + for (const finding of current.findings) { + finding.evidence = validateExternalEvidence(finding.evidence, "local"); + const repository = finding.evidence.details?.repository; + if (repository) urls.add(repository.url); + } + const sources = Array.from(urls, (url) => ({ + url: declaredScannerRepositoryUrl(url, settings.provider, "local"), + platform, + })); + const [source] = sources; + const context = await localContext(settings.repository, dependencies); + let matched = false; + if (source) { + if ( + references.some((ref) => ref.platform && ref.platform !== platform) || + sources.some(({ url }) => !hostedRepositoryMatches(source, url)) + ) + throw new CodexSecurityError( + "The findings declare conflicting source repositories.", + ); + const remotes = await hostedCheckoutRemotes( + context.repository, + context.options.protectedRoot, + context.options.environment, + signal, + ); + matched = + remotes?.some((remote) => hostedRepositoryMatches(source, remote)) ?? + false; + if ( + !matched && + remotes?.some((remote) => + /^https?:\/\//iu.test( + normalizeHostedGitRepositoryUrl( + remote, + undefined, + platform !== undefined, + ), + ), + ) + ) + throw new CodexSecurityError( + `Imported repository ${JSON.stringify(source.url)} does not match a configured fetch remote of ${context.repository}.`, + ); + } + return preparedLocalStorage( + current, + settings, + context, dependencies, + matched, + { sourceRepositories: sources }, ); } @@ -767,11 +954,21 @@ export async function prepareLocalImport( readerSources = distinctSourceRepositories( snapshotSourceRepositories(dependencies.sourceRepositories), ), - parsedInput: readonly ExternalSourceRepository[] = [], + parsedInput?: readonly ExternalSourceRepository[], fetchRemotes?: readonly ConfiguredFetchRemote[], ): Promise { requireIdentity(options.sourceKey, "Source key"); - const parsedSources = distinctSourceRepositories(parsedInput); + const parsedSources = distinctSourceRepositories(parsedInput ?? []); + // File readers interpret their source option while parsing. For normalized + // input the caller declares one repository for the whole selection. + const normalizedSource = + parsedInput === undefined && options.sourceRepositoryUrl !== undefined + ? declaredScannerRepositoryUrl( + options.sourceRepositoryUrl, + options.provider, + "local", + ) + : undefined; const current = snapshotNormalizedFindings( input, checkStoredNumber, @@ -790,16 +987,8 @@ export async function prepareLocalImport( ); checkTrivyReportProvider(options.provider, current.report); let dependabotSource: string | undefined; - const identities = new Set(); + validateFindingIds(current); for (const finding of current.findings) { - requireIdentity(finding.source_finding_id, "Source finding id"); - if (identities.has(finding.source_finding_id)) - throw new CodexSecurityError( - `Duplicate source finding id ${JSON.stringify(finding.source_finding_id)}.`, - ); - identities.add(finding.source_finding_id); - if (!isRecord(finding.evidence)) - throw new CodexSecurityError("Imported evidence must be an object."); checkRetainedTrivyProvider(options.provider, finding, "local"); if (options.provider !== "github") continue; const native = dependabotRepository(finding); @@ -825,15 +1014,9 @@ export async function prepareLocalImport( options.sourceRepositoryUrl, [...readerSources, ...parsedSources], ); - const remotes = await readHostedGitRemotes( + const remotes = await hostedCheckoutRemotes( context.repository, - [ - ...context.options.protectedRoot, - await abortable( - () => canonicalConfigPath(process.cwd()), - dependencies.signal, - ), - ], + context.options.protectedRoot, context.options.environment, dependencies.signal, ); @@ -841,7 +1024,9 @@ export async function prepareLocalImport( throw new CodexSecurityError( "Tenable's source repository must match a configured Git fetch remote of the selected checkout.", ); - return preparedLocalStorage(current, options, context, dependencies, true); + return preparedLocalStorage(current, options, context, dependencies, true, { + sourceRepositories: [source], + }); } const { repository, options: workbenchOptions } = context; const sources = new Map(); @@ -866,7 +1051,7 @@ export async function prepareLocalImport( ); const mixed = projected.length > 0 && readerSources.length > 0; const nativeSources = new Map( - (mixed ? current.findings : projected) + (mixed || normalizedSource ? current.findings : projected) .filter( ({ evidence }) => options.provider !== "endor" || @@ -980,7 +1165,12 @@ export async function prepareLocalImport( const retained = bound.get(finding.source_finding_id) ?? checkmarxRepository(finding); const native = nativeSources.get(finding) ?? []; - const sarif = hasPlatformEvidence ? semgrepSarifRepositories(finding) : []; + const sarif = + hasPlatformEvidence || + (normalizedSource && options.provider === "semgrep") + ? semgrepSarifRepositories(finding) + : []; + if (normalizedSource) for (const source of sarif) known.add(source.url); const semgrep = semgrepPlatformFindingBinding( finding, options.provider, @@ -1309,6 +1499,19 @@ export async function prepareLocalImport( "The normalized repository does not match the Snyk reader's repository.", ); } + if (normalizedSource && !snyk) { + if ( + !sameRepositories([normalizedSource, ...sources.keys(), ...known]) || + hosted.some( + (source) => !semgrepReferenceMatches(source, normalizedSource), + ) + ) + throw new CodexSecurityError( + "The declared repository does not match the normalized finding source.", + ); + if (!hosted.length && !sources.size) + sources.set(normalizedSource, { url: normalizedSource }); + } const ordinarySources = [...sources.values()]; // A reader can identify the VCS for both a hosted project and an ordinary row. const loosePlatformFor = localRepositoryPlatform( @@ -1343,35 +1546,94 @@ export async function prepareLocalImport( ? (url) => checkmarxKeys.has(identify(url).key) : undefined, ); + let storedSources = [...references, ...ordinarySources, ...hosted]; + const [project] = hosted; + if ( + options.provider === "semgrep" && + !hasOrdinaryRows && + !ordinarySources.length && + hosted.length === 1 && + project && + references.every(({ url }) => url === project.url) + ) { + // A typed hint and a bare URL can be two witnesses for the same native + // project. Keep the hint if any stored occurrence needs it to verify its scope. + storedSources = [project]; + const native = semgrepPlatformRepositoryReference(project.url, []); + if (native.platform !== project.platform) { + try { + if ( + current.findings.every((finding) => { + const binding = semgrepPlatformFindingBinding(finding, "semgrep", [ + native, + ]); + return ( + binding?.url === native.url && + !binding.migration && + isDeepStrictEqual( + binding.sourceData, + finding.evidence.source_data, + ) + ); + }) + ) + storedSources = [native]; + } catch { + // A private SCM alias or SSH mapping can require the reader's platform. + } + } + } + if (snyk) + storedSources.push({ + url: snyk.url, + platform: + snyk.platform === "github" + ? "github" + : snyk.gitlab + ? "gitlab" + : undefined, + }); return preparedLocalStorage( current, options, context, dependencies, matchesRepositoryRemote, + { + sourceRepositories: distinctSourceRepositories(storedSources), + inspectorProjectMap: options.inspectorProjectMap + ? [...options.inspectorProjectMap] + : undefined, + }, ); } +/** @internal Prepare the transactional write after validating the report and repository. */ function preparedLocalStorage( current: ExternalFindings, - options: NormalizedLocalImportOptions, + { provider, sourceKey }: NormalizedLocalImportOptions, { repository, options: workbenchOptions, }: Awaited>, dependencies: LocalExternalImportDependencies, matchesRepositoryRemote: boolean, + provenance: ExternalFindingProvenance, ): PreparedLocalExternalImport { dependencies.signal?.throwIfAborted(); const preview = { ...cloneImportPreview(current), repository, - source: { provider: options.provider, source_key: options.sourceKey }, + source: { provider, source_key: sourceKey }, }; const payload = JSON.stringify({ repository, - provider: options.provider, - sourceKey: options.sourceKey, + provider, + sourceKey, + read: current.read, + excluded: current.excluded, + report: reportForJson(current.report), + provenance, findings: current.findings.map(({ source_finding_id, evidence }) => ({ source_finding_id, evidence, @@ -1436,6 +1698,7 @@ function preparedLocalStorage( }, dependencies.signal); return { ...summary, + importId: result["importId"] as string, findings: result["findings"] as unknown as StoredExternalFinding[], counts: result["counts"] as LocalExternalImportResult["counts"], }; @@ -1457,8 +1720,93 @@ export async function readLocalExternalFindings( ) { dependencies.signal?.throwIfAborted(); if (selection.id !== undefined) requireIdentity(selection.id, "Finding id"); - const { repository, selectedPath, options } = await localContext( + const { selectedPath, result } = await runLocalFindingWorkbench( selection.repository, + "read-external-findings", + { id: selection.id }, + dependencies, + ); + return { + repository: selectedPath, + findings: result["findings"] as unknown as StoredExternalFinding[], + }; +} + +/** Read the precise versions selected by the caller or captured by an earlier import. */ +export async function selectLocalExternalFindings( + selection: LocalExternalFindingSelection, + dependencies: LocalExternalImportDependencies = {}, +): Promise { + const { repository, importId, findings } = selection; + if ((importId === undefined) === (findings === undefined)) + throw new CodexSecurityError( + "Select one import ID or a list of versioned findings.", + ); + const { result } = await runLocalFindingWorkbench( + repository, + "read-external-findings", + importId !== undefined + ? { importId } + : { findings: findings!.map(({ id, version }) => ({ id, version })) }, + dependencies, + ); + return result as unknown as SelectedLocalExternalFindings; +} + +/** List this checkout's nonempty imports, most recent first. */ +export async function listLocalExternalImports( + { repository }: { repository: string }, + dependencies: LocalExternalImportDependencies = {}, +): Promise { + const { result } = await runLocalFindingWorkbench( + repository, + "list-external-imports", + {}, + dependencies, + ); + return result["imports"] as unknown as StoredExternalImport[]; +} + +/** @internal Attach Cloud acknowledgements to the versions that were submitted. */ +export async function recordExternalFindingPublications( + input: CloudFindingDestination & { + repository: string; + entries: LocalFindingReceipt[]; + }, + dependencies: LocalExternalImportDependencies = {}, +): Promise { + const { repository, ...receipt } = input; + await runLocalFindingWorkbench( + repository, + "record-external-finding-publications", + receipt, + dependencies, + ); +} + +/** Read the destination and receipt for each acknowledged finding version. */ +export async function listLocalFindingPublications( + { repository, id }: { repository: string; id?: string }, + dependencies: LocalExternalImportDependencies = {}, +): Promise { + const { result } = await runLocalFindingWorkbench( + repository, + "list-external-finding-publications", + { id }, + dependencies, + ); + return result["publications"] as unknown as StoredFindingPublication[]; +} + +/** @internal Run a command against the workbench for the selected checkout. */ +async function runLocalFindingWorkbench( + path: string, + command: string, + selection: Record, + dependencies: LocalExternalImportDependencies, +) { + const { repository, selectedPath, options } = await localContext( + path, dependencies, "read", ); @@ -1466,12 +1814,9 @@ export async function readLocalExternalFindings( dependencies.signal?.throwIfAborted(); return (dependencies.runWorkbench ?? runWorkbench)( options, - ["list-external-findings"], - JSON.stringify({ repository, id: selection.id }), + [command], + JSON.stringify({ repository, ...selection }), ); }, dependencies.signal); - return { - repository: selectedPath, - findings: result["findings"] as unknown as StoredExternalFinding[], - }; + return { selectedPath, result }; } diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 60ef3fc40a..8e23ad3611 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -225,7 +225,10 @@ export interface ExternalPublicationPreview extends ExternalFindings { export interface PreparedExternalPublication { preview: ExternalPublicationPreview; - publish(): Promise; + /** Persist acknowledged repository receipts before retiring their resumable request. */ + publish( + beforeRetire?: (result: ExternalPublicationResult) => Promise, + ): Promise; } // Limit read concurrency to avoid one network round trip per finding in series. @@ -1849,7 +1852,7 @@ async function preparePublicationInput( }; return { preview: cloneImportPreview(preview), - async publish() { + async publish(beforeRetire) { progress({ phase: "waiting", completed: 0 }); const result = ( acknowledged: FindingImportReceipt[], @@ -2055,6 +2058,7 @@ async function preparePublicationInput( ...completed, report: reportForJson(completed.report), }); + await beforeRetire?.(completed); await removePending(); return completed; } catch (error) { diff --git a/sdk/typescript/src/external-findings.ts b/sdk/typescript/src/external-findings.ts index 688d7b0b2c..c21d194f3f 100644 --- a/sdk/typescript/src/external-findings.ts +++ b/sdk/typescript/src/external-findings.ts @@ -222,16 +222,14 @@ export function cloneImportPreview(input: T): T { } /** Capture the caller's import settings before reading an export or resolving its destination. */ -export function snapshotImportOptions< - P extends LocalExternalImportOptions["provider"], ->({ +export function snapshotImportOptions

({ repository, provider, sourceKey, sourceRepositoryUrl, trivyScanRoot, inspectorProjectMap, -}: LocalExternalImportOptions & { provider: P }) { +}: Omit & { provider: P }) { if (provider === "trivy" && sourceRepositoryUrl !== undefined) throw new CodexSecurityError( "Trivy reads repository provenance from the report; do not supply --source-repository-url.", diff --git a/sdk/typescript/src/external-publication-review.ts b/sdk/typescript/src/external-publication-review.ts new file mode 100644 index 0000000000..806cb63280 --- /dev/null +++ b/sdk/typescript/src/external-publication-review.ts @@ -0,0 +1,34 @@ +import { hash } from "node:crypto"; +import type { ExternalPublicationPreview } from "./external-findings-publish.js"; +import { canonicalJson } from "./record.js"; + +/** Bind reviewed evidence to its account, repository, connector and destination version. */ +export function cloudReviewDigest( + preview: ExternalPublicationPreview, + checkout: string, + findings?: readonly { id: string; version: number }[], +) { + return hash( + "sha256", + canonicalJson({ + checkout, + api: preview.cloudApiUrl, + account: preview.accountId, + repository: preview.destination.id, + connector: preview.destination.repo_connector_id, + url: preview.destination.url, + reset: preview.destination.reset_marker, + source: preview.source, + privateImport: preview.privateImport, + ...(findings && { findings, excluded: preview.excluded }), + items: preview.requests.flatMap((request) => + request.items.map((item) => [ + item.source_finding_id, + item.expected_version, + request.repository.environment_id, + item.evidence, + ]), + ), + }), + ); +} diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index ec11147458..7ac3143b92 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -232,9 +232,17 @@ export type { } from "./external-findings-publish.js"; export { prepareNormalizedLocalImport, + listLocalExternalImports, listLocalExternalFindings, + listLocalFindingPublications, + selectLocalExternalFindings, } from "./external-findings-local.js"; export type { + ExternalFindingProvenance, + LocalExternalFindingSelection, + SelectedLocalExternalFindings, + StoredExternalImport, + StoredFindingPublication, StoredExternalFinding, NormalizedLocalImportOptions, LocalExternalImportDependencies, @@ -351,6 +359,11 @@ export { prepareLocalExternalImport, type LocalExternalImportOptions, } from "./external-findings-import.js"; +export { + prepareStoredFindingsPublication, + type StoredFindingsPublication, + type StoredFindingsPublicationGroup, +} from "./stored-findings-publication.js"; export { readTenableFindings } from "./tenable-findings.js"; export { normalizeOpenGrepFindings, diff --git a/sdk/typescript/src/opengrep-findings.ts b/sdk/typescript/src/opengrep-findings.ts index 96827c1d90..a82e1880ba 100644 --- a/sdk/typescript/src/opengrep-findings.ts +++ b/sdk/typescript/src/opengrep-findings.ts @@ -1,11 +1,12 @@ +import { isDeepStrictEqual } from "node:util"; import { validateExternalEvidenceJson, validateExternalReport, type ExternalImportTarget, } from "./external-import-contract.js"; -import { canonicalJson } from "./record.js"; +import { canonicalJson, isRecord } from "./record.js"; import { errorMessage } from "./errors.js"; -import type { ExternalFindings } from "./external-findings.js"; +import type { ExternalFinding, ExternalFindings } from "./external-findings.js"; import { messageText, nativeResultPath, @@ -16,7 +17,7 @@ import { import { appendFinding, array, - readScannerJson, + scannerJsonReader, record, requiredText, stableId, @@ -53,19 +54,70 @@ export function normalizeOpenGrepFindings( const values = array(results, "OpenGrep results"); const errors = array(report["errors"], "OpenGrep errors"); validateExternalReport(report); + return projectOpenGrepResults( + values, + repositoryUrl, + version, + errors.length, + target, + report, + ); +} + +/** Reproject a retained native occurrence without changing SDK- or agent-authored evidence. */ +export function projectStoredOpenGrepFinding( + finding: ExternalFinding, +): ExternalFindings | undefined { + const { evidence } = finding; + const native = evidence.source_data; + const traces = native?.["results"]; + const context = native?.["report"]; + const version = isRecord(context) && context["version"]; + const count = native?.["scan_error_count"]; + const repositoryUrl = evidence.details?.repository?.url; + if ( + evidence.details?.scanner_origin !== "OpenGrep" || + !repositoryUrl || + !Array.isArray(traces) || + typeof version !== "string" || + typeof count !== "number" || + !Number.isSafeInteger(count) || + count < 0 + ) + return undefined; + const project = (target: ExternalImportTarget) => + projectOpenGrepResults(traces, repositoryUrl, version, count, target); + const local = project("local"); + if ( + local.findings.length !== 1 || + local.excluded.length || + !isDeepStrictEqual(local.findings[0], finding) + ) + return undefined; + return project("cloud"); +} + +function projectOpenGrepResults( + values: unknown[], + repositoryUrl: string, + version: string, + scanErrorCount: number, + target: ExternalImportTarget, + report?: ExternalFindings["report"], +): ExternalFindings { const projection = { origin: "OpenGrep", repositoryUrl, severity: () => "unknown", reportContext: { version }, target, - scanErrorCount: errors.length, + scanErrorCount, }; const output: ExternalFindings = { read: values.length, findings: [], excluded: [], - report: structuredClone(report), + ...(report && { report: structuredClone(report) }), }; type Observation = { position: number; @@ -87,7 +139,7 @@ export function normalizeOpenGrepFindings( }); const sourceData = { report: { version }, - scan_error_count: errors.length, + scan_error_count: scanErrorCount, results, }; // Replacing [] with the same raw results adds the same wire bytes to every candidate. @@ -252,14 +304,6 @@ export function normalizeOpenGrepFindings( return output; } -export async function readOpenGrepFindings( - path: string, - options: OpenGrepImportOptions, - target: ExternalImportTarget = "cloud", -): Promise { - return normalizeOpenGrepFindings( - await readScannerJson(path), - options, - target, - ); -} +export const readOpenGrepFindings = scannerJsonReader( + normalizeOpenGrepFindings, +); diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 93d17c0e07..a9a036bda8 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -2028,7 +2028,10 @@ export async function runWorkbench( "store-findings", "list-stored-findings", "store-external-findings", - "list-external-findings", + "read-external-findings", + "list-external-imports", + "record-external-finding-publications", + "list-external-finding-publications", "find-potential-duplicates", "store-dedupe-groups", "list-dedupe-groups", diff --git a/sdk/typescript/src/semgrep-platform-findings.ts b/sdk/typescript/src/semgrep-platform-findings.ts index e839e9d2eb..53b5fc6b08 100644 --- a/sdk/typescript/src/semgrep-platform-findings.ts +++ b/sdk/typescript/src/semgrep-platform-findings.ts @@ -892,6 +892,41 @@ function nativeOccurrence( return { scope, aliases, repository, row }; } +/** Reproduce the native Cloud title fallback for an unmodified Platform message title. */ +export function storedSemgrepCloudEvidence( + finding: ExternalFinding, + references: readonly ExternalSourceRepository[] = [], +): ExternalFinding["evidence"] | undefined { + const { evidence } = finding; + if ( + evidence.title !== evidence.description || + Buffer.byteLength(evidence.title) <= DEFAULT_CLOUD_STRING_BYTES || + !evidence.source_data || + !isSemgrepPlatformEvidence(finding, "semgrep") + ) + return; + const { row, scope, aliases, repository } = nativeOccurrence( + evidence.source_data, + references, + "local", + ); + if (sourceId(scope, row) !== finding.source_finding_id) return; + const selectable = checkSemgrepPlatformRow(row, scope, aliases, repository); + const projectNames = + aliases && + projectAliases(scope.project.name, scope.project.url, references); + const title = (target: ExternalImportTarget) => + normalizePlatformRows( + [{ finding: row, selectable }], + scope, + target, + projectNames, + ).findings[0]?.evidence.title; + if (title("local") !== evidence.title) return; + const cloud = title("cloud"); + if (cloud && cloud !== evidence.title) return { ...evidence, title: cloud }; +} + /** Recheck the source repository when an SDK caller supplies normalized evidence directly. */ export function semgrepPlatformFindingRepository( finding: ExternalFinding, diff --git a/sdk/typescript/src/sonar-findings.ts b/sdk/typescript/src/sonar-findings.ts index 87e6368ea0..9eb4ee7549 100644 --- a/sdk/typescript/src/sonar-findings.ts +++ b/sdk/typescript/src/sonar-findings.ts @@ -11,7 +11,7 @@ import { isRecord } from "./record.js"; import { array, normalizeRecords, - readScannerJson, + scannerJsonReader, record, relativePath, requiredText, @@ -419,14 +419,4 @@ export function normalizeSonarFindings( ) as ExternalFindings & { report: typeof context }; } -export async function readSonarFindings( - path: string, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): Promise { - return normalizeSonarFindings( - await readScannerJson(path), - sourceRepositoryUrl, - target, - ); -} +export const readSonarFindings = scannerJsonReader(normalizeSonarFindings); diff --git a/sdk/typescript/src/sonar-import.ts b/sdk/typescript/src/sonar-import.ts index 324a4f8ac7..cd462b562c 100644 --- a/sdk/typescript/src/sonar-import.ts +++ b/sdk/typescript/src/sonar-import.ts @@ -9,6 +9,8 @@ import { type ConfiguredFetchRemote, prepareLocalImport, } from "./external-findings-local.js"; +import { prepareStoredFindingsPublication } from "./stored-findings-publication.js"; +import { cloudReviewDigest } from "./external-publication-review.js"; import { commandGitProtectionRoots } from "./local-git-metadata.js"; import { ExternalPublicationError, @@ -17,7 +19,6 @@ import { writeAtomicJson, type ExternalPublicationDependencies, type ExternalPublicationPreview, - type PreparedExternalPublication, } from "./external-findings-publish.js"; import { readScannerJson, stableId } from "./scanner-import.js"; import { declaredScannerRepositoryUrl } from "./scanner-repository-url.js"; @@ -52,34 +53,6 @@ interface SonarImportOptions { yes: boolean; } -function cloudReviewDigest( - preview: ExternalPublicationPreview, - checkout: string, -) { - return hash( - "sha256", - canonicalJson({ - checkout, - api: preview.cloudApiUrl, - account: preview.accountId, - repository: preview.destination.id, - connector: preview.destination.repo_connector_id, - url: preview.destination.url, - reset: preview.destination.reset_marker, - source: preview.source, - privateImport: preview.privateImport, - items: preview.requests.flatMap((request) => - request.items.map((item) => [ - item.source_finding_id, - item.expected_version, - request.repository.environment_id, - item.evidence, - ]), - ), - }), - ); -} - async function readCapture(input: string) { if (!(await stat(input)).isFile()) return { path: input, raw: await readScannerJson(input) }; @@ -313,8 +286,13 @@ export async function prepareSonarImport( [], fetchRemotes, ); - let cloud: PreparedExternalPublication | undefined; + let cloud: ExternalPublicationPreview | undefined; let cloudError: string | undefined; + const reviewedCapturePath = (preview: ExternalPublicationPreview) => + join( + root, + `${evidenceDigest}.${checkout}.${cloudReviewDigest(preview, repository)}.json`, + ); if (options.to === "cloud" && local.preview.read > 0) { const reviewedCheckout = retained?.[2] === checkout && Boolean(retained?.[3]); @@ -345,18 +323,14 @@ export async function prepareSonarImport( { repository: repositoryUrl, ...source }, dependencies, ); - const review = cloudReviewDigest(prepared.preview, repository); - const reviewedCapture = join( - root, - `${evidenceDigest}.${checkout}.${review}.json`, - ); + const reviewedCapture = reviewedCapturePath(prepared.preview); if (input !== reviewedCapture) { if (!confirmedRepository) throw new Error(bindingError); if (options.yes && !options.dryRun) throw new Error(reviewError); await writeAtomicJson(reviewedCapture, report); } capture = reviewedCapture; - cloud = prepared; + cloud = prepared.preview; } catch (error) { signal?.throwIfAborted(); cloudError = errorMessage(error); @@ -376,17 +350,32 @@ export async function prepareSonarImport( return { info, local: local.preview, - cloud: cloud?.preview, + cloud, cloudError, async import() { signal?.throwIfAborted(); - const { counts, ready, excluded } = await local.import(); - const saved = { ...info, local: { counts, ready, excluded } }; + const { counts, ready, excluded, importId } = await local.import(); + const saved = { ...info, local: { counts, ready, excluded, importId } }; if (!cloud) return { ...saved, cloudError, cloud: undefined }; try { + if (!importId) throw new Error("No stored Sonar import to publish."); + const { + groups: [publication], + } = await prepareStoredFindingsPublication( + { repository, importId }, + { repository: repositoryUrl }, + dependencies, + ); + if ( + !publication || + reviewedCapturePath(publication.preview) !== capture + ) + throw new Error( + "The stored findings or Cloud destination have changed. Preview the Sonar capture again with --to cloud --dry-run.", + ); return { ...saved, - cloud: await cloud.publish(), + cloud: await publication.publish(publication.reviewId), cloudError: undefined, }; } catch (error) { diff --git a/sdk/typescript/src/stored-finding-projection.ts b/sdk/typescript/src/stored-finding-projection.ts new file mode 100644 index 0000000000..0f1161ff1f --- /dev/null +++ b/sdk/typescript/src/stored-finding-projection.ts @@ -0,0 +1,138 @@ +import { isDeepStrictEqual } from "node:util"; +import { normalizeBlackDuckFindings } from "./blackduck-findings.js"; +import { normalizeContinuumFindings } from "./continuum-findings.js"; +import { normalizeEndorFindings } from "./endor-findings.js"; +import { + validateExternalEvidence, + type ExternalImportTarget, +} from "./external-import-contract.js"; +import type { StoredExternalFinding } from "./external-findings-local.js"; +import type { + ExternalFinding, + ExternalFindings, + ExternalSourceRepository, +} from "./external-findings.js"; +import { normalizeInspectorFindings } from "./inspector-findings.js"; +import { projectStoredOpenGrepFinding } from "./opengrep-findings.js"; +import { isRecord } from "./record.js"; +import { normalizeRecords } from "./scanner-import.js"; +import { storedSemgrepCloudEvidence } from "./semgrep-platform-findings.js"; +import { normalizeTrivyFindings } from "./trivy-findings.js"; + +type StoredRow = Pick< + StoredExternalFinding, + "provider" | "sourceFindingId" | "evidence" +>; +type Reader = (target: ExternalImportTarget) => ExternalFindings; + +/** Reconstruct only report envelopes for which a native row retains its inputs. */ +function* nativeReaders( + row: StoredRow, + inspectorMap?: ReadonlyMap, +): Generator { + const data = row.evidence.source_data; + if (!data) return; + if (row.provider === "endor" && data["uuid"] === row.sourceFindingId) + yield (target) => normalizeEndorFindings(data, target); + if ( + row.provider === "trivy" && + isRecord(data["report"]) && + isRecord(data["result"]) && + typeof data["scanRoot"] === "string" + ) { + const root = data["scanRoot"]; + const input = { + ...data["report"], + Results: [ + { + ...data["result"], + Vulnerabilities: [data["vulnerability"]], + Packages: data["package"] ? [data["package"]] : [], + }, + ], + }; + yield (target) => normalizeTrivyFindings(input, target, root); + } + const url = data["sourceRepositoryUrl"]; + if (typeof url !== "string") return; + if (row.provider === "inspector" && isRecord(data["finding"])) + yield (target) => + normalizeInspectorFindings( + { findings: [data["finding"]] }, + url, + target, + inspectorMap, + ); + if (row.provider === "continuum" && isRecord(data["finding"])) { + // The list and detail APIs can return the same raw record with different SAST projections. + yield (target) => normalizeContinuumFindings(data["finding"], url, target); + yield (target) => + normalizeContinuumFindings({ items: [data["finding"]] }, url, target); + } + if ( + row.provider === "blackduck" && + typeof data["projectVersionUrl"] === "string" + ) { + const input = { + totalCount: 1, + items: [data["row"]], + _meta: { + href: data["projectVersionUrl"] + "/vulnerable-bom-components", + links: [], + }, + }; + yield (target) => normalizeBlackDuckFindings(input, url, target); + } +} + +function cloudEvidence( + row: StoredRow, + finding: ExternalFinding, + sourceRepositories: readonly ExternalSourceRepository[], + inspectorMap?: ReadonlyMap, +) { + try { + return validateExternalEvidence(row.evidence, "cloud"); + } catch (error) { + if (row.provider === "semgrep") { + const projected = storedSemgrepCloudEvidence(finding, sourceRepositories); + if (projected) return validateExternalEvidence(projected, "cloud"); + } + // Enriched typed fields can exceed Cloud limits while the complete native + // record fits. Verify the local reader reproduced this reviewed row before + // allowing its Cloud reader to choose different optional fields. + for (const read of nativeReaders(row, inspectorMap)) { + let local: ExternalFindings; + try { + local = read("local"); + } catch { + continue; + } + if (!isDeepStrictEqual(local.findings[0], finding)) continue; + const cloud = read("cloud"); + if (cloud.findings.length) return cloud.findings[0]!.evidence; + } + throw error; + } +} + +export function projectStoredFinding( + row: StoredRow, + sourceRepositories: readonly ExternalSourceRepository[], + inspectorMap?: ReadonlyMap, +): ExternalFindings { + const finding = { + source_finding_id: row.sourceFindingId, + evidence: row.evidence, + }; + if (row.provider === "semgrep") { + const traces = projectStoredOpenGrepFinding(finding); + if (traces) return traces; + } + return normalizeRecords( + [row], + () => cloudEvidence(row, finding, sourceRepositories, inspectorMap), + () => row.sourceFindingId, + "cloud", + ); +} diff --git a/sdk/typescript/src/stored-findings-publication.ts b/sdk/typescript/src/stored-findings-publication.ts new file mode 100644 index 0000000000..34a533d033 --- /dev/null +++ b/sdk/typescript/src/stored-findings-publication.ts @@ -0,0 +1,318 @@ +import { isDeepStrictEqual } from "node:util"; +import cloudSchema from "../schemas/external-findings.schema.json" with { type: "json" }; +import { CodexSecurityError, errorMessage } from "./errors.js"; +import type { Provider } from "./external-import-models.js"; +import { + distinctSourceRepositories, + recordExternalFindingPublications, + selectLocalExternalFindings, + type LocalExternalFindingSelection, + type LocalExternalImportDependencies, + type StoredExternalFinding, +} from "./external-findings-local.js"; +import { + ExternalPublicationError, + prepareNormalizedExternalPublication, + type ExternalPublicationDependencies, + type ExternalPublicationPreview, + type ExternalPublicationResult, + type PreparedExternalPublication, +} from "./external-findings-publish.js"; +import { + NoPublishableFindingsError, + externalImportDisposition, + type ExternalFindings, + type ExternalSourceRepository, +} from "./external-findings.js"; +import { cloudReviewDigest } from "./external-publication-review.js"; +import { projectStoredFinding } from "./stored-finding-projection.js"; + +type Dependencies = ExternalPublicationDependencies & + LocalExternalImportDependencies; +type Reference = Pick< + StoredExternalFinding, + "id" | "version" | "provider" | "sourceKey" | "sourceFindingId" +>; +type Metadata = Pick; + +export interface StoredFindingsPublicationGroup { + /** Prepare this selection again to resume the same reviewed source in another process. */ + selection: LocalExternalFindingSelection; + preview: ExternalPublicationPreview; + reviewId: string; + publish(reviewId: string): Promise; +} + +export interface StoredFindingsPublication { + groups: StoredFindingsPublicationGroup[]; + excluded: (Reference & { reason: string })[]; +} + +function reference({ + id, + version, + provider, + sourceKey, + sourceFindingId, +}: StoredExternalFinding): Reference { + return { id, version, provider, sourceKey, sourceFindingId }; +} + +function cloudProvider(provider: string): provider is Provider { + return cloudSchema.$defs.ImportSource.properties.provider.enum.includes( + provider, + ); +} + +function cloudSelection( + rows: StoredExternalFinding[], + sourceRepositories: readonly ExternalSourceRepository[], + inspectorMap: ReadonlyMap | undefined, + metadata?: Metadata, +): ExternalFindings { + const selected: ExternalFindings = { + read: metadata?.excluded.length ?? 0, + findings: [], + excluded: [...(metadata?.excluded ?? [])], + report: metadata?.report, + }; + const skipped = new Set(selected.excluded.map((item) => item.position)); + let position = 0; + for (const row of rows) { + const projection = projectStoredFinding( + row, + sourceRepositories, + inspectorMap, + ); + selected.findings.push(...projection.findings); + selected.read += projection.read; + const positions = Array.from({ length: projection.read }, () => { + do { + position++; + } while (skipped.has(position)); + return position; + }); + for (const item of projection.excluded) + selected.excluded.push({ + ...item, + position: positions[item.position - 1]!, + source_finding_id: row.sourceFindingId, + ...(projection.read > 1 && { + reason: `Retained trace ${item.position} in ${JSON.stringify(row.sourceFindingId)} (position in stored selection order): ${item.reason}`, + }), + }); + } + // A native source may account for additional input observations, and an agent + // report may count a grouped occurrence once before its traces are expanded. + selected.read = Math.max(selected.read, metadata?.read ?? 0); + selected.excluded.sort((a, b) => a.position - b.position); + return selected; +} + +type ReceiptLink = Parameters< + typeof recordExternalFindingPublications +>[0]["entries"][number]; + +function receiptRecorder( + preview: ExternalPublicationPreview, + rows: StoredExternalFinding[], + dependencies: Dependencies, +) { + // Keep only the identifiers needed for receipts. The caller can edit its public preview. + const pins = new Map( + rows.map(({ id, version, sourceFindingId }) => [ + sourceFindingId, + { id, version }, + ]), + ); + const ready = preview.findings.map((f) => pins.get(f.source_finding_id)!); + const batches = preview.requests.map( + ({ items }) => + new Map( + items.map((item) => [ + item.client_id, + pins.get(item.source_finding_id)!, + ]), + ), + ); + const destination = { + repository: rows[0]!.repository, + cloudApiUrl: preview.cloudApiUrl, + accountId: preview.accountId, + destinationId: preview.destination.id, + }; + return async (result: ExternalPublicationResult) => { + const entries: ReceiptLink[] = []; + const job = result.privateJob; + // The private inbox provides job totals, with no per-item outcomes. + if ( + job?.status === "completed" && + !result.counts.error && + !result.unacknowledged + ) { + for (const pin of ready) + entries.push({ ...pin, receiptId: job.id, outcome: "private_job" }); + } + for (const [index, receipt] of result.receipts.entries()) { + for (const item of receipt.results) { + if (item.outcome !== "error") + entries.push({ + ...batches[index]!.get(item.client_id)!, + receiptId: receipt.id, + outcome: item.outcome, + cloudFindingId: item.canonical_finding_id ?? undefined, + }); + } + } + if (!entries.length) return; + try { + await recordExternalFindingPublications( + { ...destination, entries }, + dependencies, + ); + } catch (error) { + throw new ExternalPublicationError( + `Cloud returned receipts, but the workbench could not record them: ${errorMessage(error)}`, + result, + error, + ); + } + }; +} + +function prepareSource( + rows: StoredExternalFinding[], + options: { repository: string; provider: Provider; sourceKey: string }, + dependencies: Dependencies, + metadata?: Metadata, +) { + const references = [...(dependencies.sourceRepositories ?? [])]; + const inspectorProjectMap = new Map(); + for (const row of rows) { + references.push(...(row.provenance?.sourceRepositories ?? [])); + for (const [arn, url] of row.provenance?.inspectorProjectMap ?? []) { + if (inspectorProjectMap.has(arn) && inspectorProjectMap.get(arn) !== url) + throw new CodexSecurityError( + "The selected findings disagree on the Inspector project repository.", + ); + inspectorProjectMap.set(arn, url); + } + } + const sourceRepositories = distinctSourceRepositories(references); + return prepareNormalizedExternalPublication( + cloudSelection( + rows, + sourceRepositories, + inspectorProjectMap.size ? inspectorProjectMap : undefined, + metadata, + ), + { ...options, ...(inspectorProjectMap.size && { inspectorProjectMap }) }, + { ...dependencies, sourceRepositories }, + ); +} + +/** Preview each source once. Each group's approval can be resumed independently. */ +export async function prepareStoredFindingsPublication( + selection: LocalExternalFindingSelection, + options: { repository: string }, + dependencies: Dependencies = {}, +): Promise { + const { importId } = selection; + const stored = await selectLocalExternalFindings(selection, dependencies); + const groups: StoredFindingsPublicationGroup[] = []; + const excluded: StoredFindingsPublication["excluded"] = []; + const bySource = Map.groupBy(stored.findings, ({ provider, sourceKey }) => + JSON.stringify([provider, sourceKey]), + ); + for (const rows of bySource.values()) { + const { provider, sourceKey, repository } = rows[0]!; + if (!cloudProvider(provider)) { + const reason = `The Cloud import contract does not support provider ${JSON.stringify(provider)}.`; + for (const row of rows) excluded.push({ ...reference(row), reason }); + continue; + } + let prepared: PreparedExternalPublication; + try { + prepared = await prepareSource( + rows, + { ...options, provider, sourceKey }, + dependencies, + importId === undefined ? undefined : stored, + ); + } catch (error) { + if ( + !(error instanceof NoPublishableFindingsError) || + externalImportDisposition(error.selection) === "failed" + ) + throw error; + appendExclusions(rows, error.selection); + continue; + } + const { preview } = prepared; + appendExclusions(rows, preview); + const first = groups[0]?.preview; + if ( + first && + (first.accountId !== preview.accountId || + !isDeepStrictEqual(first.destination, preview.destination)) + ) + throw new CodexSecurityError( + "The Cloud account or repository changed during publication preparation.", + ); + const findings = rows.map(({ id, version }) => ({ id, version })); + const reviewId = cloudReviewDigest(preview, repository, findings); + const record = receiptRecorder(preview, rows, dependencies); + groups.push({ + selection: + importId === undefined + ? { repository, findings } + : { repository, importId }, + preview, + reviewId, + async publish(approvedReviewId) { + if (approvedReviewId !== reviewId) + throw new CodexSecurityError( + "The stored findings publication has changed. Review the new preview before publishing.", + ); + let recordingStarted = false; + async function recordOnce(result: ExternalPublicationResult) { + if (recordingStarted) return; + recordingStarted = true; + await record(result); + } + let result: ExternalPublicationResult; + try { + result = await prepared.publish(recordOnce); + } catch (error) { + if (error instanceof ExternalPublicationError) + await recordOnce(error.result); + throw error; + } + await recordOnce(result); + return result; + }, + }); + } + return { groups, excluded }; + + function appendExclusions( + rows: StoredExternalFinding[], + projected: ExternalFindings, + ) { + const accepted = new Set( + projected.findings.map((f) => f.source_finding_id), + ); + const reasons = new Map( + projected.excluded.map((f) => [f.source_finding_id, f.reason]), + ); + for (const row of rows) { + if (!accepted.has(row.sourceFindingId)) + excluded.push({ + ...reference(row), + reason: + reasons.get(row.sourceFindingId) ?? + "Excluded by the Cloud publisher.", + }); + } + } +} diff --git a/sdk/typescript/src/tenable-findings.ts b/sdk/typescript/src/tenable-findings.ts index 6570e16460..d9973d62c8 100644 --- a/sdk/typescript/src/tenable-findings.ts +++ b/sdk/typescript/src/tenable-findings.ts @@ -74,7 +74,7 @@ export function tenableSourceRepository( input: ExternalFindings, declared: string | undefined, references: readonly ExternalSourceRepository[], -) { +): ExternalSourceRepository { const url = declaredScannerRepositoryUrl(declared, "Tenable", "local"); const asset = asRecord(input.report?.["asset"]); if ( diff --git a/sdk/typescript/src/veracode-findings.ts b/sdk/typescript/src/veracode-findings.ts index 5b82d40daa..fdcfc161b5 100644 --- a/sdk/typescript/src/veracode-findings.ts +++ b/sdk/typescript/src/veracode-findings.ts @@ -10,7 +10,7 @@ import { isRecord } from "./record.js"; import { array, normalizeRecords, - readScannerJson, + scannerJsonReader, record, requiredText, stableId, @@ -555,14 +555,6 @@ export function normalizeVeracodeFindings( ); } -export async function readVeracodeFindings( - path: string, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): Promise { - return normalizeVeracodeFindings( - await readScannerJson(path), - sourceRepositoryUrl, - target, - ); -} +export const readVeracodeFindings = scannerJsonReader( + normalizeVeracodeFindings, +); diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index a16d2dfb7a..3b7e29987d 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -97,10 +97,21 @@ import { normalizeTrivyFindings } from "../src/trivy-findings.js"; import { normalizeInspectorFindings } from "../src/inspector-findings.js"; import { trivyReport, + trivyReportWithManyLocations, trivyRepositoryHeader, } from "./support/trivy-import-fixtures.js"; -import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; +import { + prepareNormalizedLocalImport, + type LocalExternalFindingSelection, +} from "../src/external-findings-local.js"; import { localFindingsFixture } from "./external-findings-local-helpers.js"; +import { + collectSelectedSemgrepGuardianFindings, + listLocalExternalFindings, + listLocalFindingPublications, + prepareLocalExternalImport, + prepareStoredFindingsPublication, +} from "../src/index.js"; import { inspectorCodeFinding, inspectorReport, @@ -731,6 +742,217 @@ async function fixture( }; } +test.each([0, 1])( + "stored Trivy and Guardian findings survive source deletion and a lost Cloud response in group %i", + async (index) => { + const cloud = await fixture(); + await using local = await storedPublicationFixture(cloud); + const { repository, deps } = local; + const report = trivyReport(); + report.Results.push({ + ...structuredClone(report.Results[0]!), + Target: "packages/other/package-lock.json", + }); + const trivyFile = cloud.file; + await writeFile(trivyFile, JSON.stringify(report)); + const prepareTrivy = () => + prepareLocalExternalImport( + trivyFile, + { + repository, + provider: "trivy", + sourceKey: "npm/main", + trivyScanRoot: ".", + }, + deps, + ); + expect((await (await prepareTrivy()).import()).counts.created).toBe(2); + report.Results[0]!.Vulnerabilities[0]!.Title = "Updated local evidence"; + await writeFile(trivyFile, JSON.stringify(report)); + expect((await (await prepareTrivy()).import()).counts).toEqual({ + created: 0, + updated: 1, + unchanged: 1, + }); + await rm(trivyFile); + + const examples = new URL( + "../../../examples/normalized-import/", + import.meta.url, + ); + const readExample = async (name: string) => + JSON.parse(await readFile(new URL(name, examples), "utf8")); + const guardian = await collectSelectedSemgrepGuardianFindings( + await readExample("guardian-selection.json"), + { callTool: () => readExample("guardian-response.json") }, + ); + const guardianImport = await local.store( + "semgrep", + normalizeSemgrepFindings(guardian, {}, "local"), + ); + expect(guardianImport.counts.created).toBe(1); + + const rows = await listLocalExternalFindings({ repository }, deps); + const selected = [ + rows.find((row) => row.provider === "trivy" && row.version === 2)!, + rows.find((row) => row.provider === "semgrep")!, + ]; + const selection = { + repository, + findings: selected.map(({ id, version }) => ({ id, version })), + }; + const { groups } = await local.prepare(selection); + expect(groups).toHaveLength(2); + const reviewedEnvironment = cloud.state.environmentId; + cloud.state.environmentId = "different-environment"; + const changed = await local.prepare(selection); + await expect( + changed.groups[0]!.publish(groups[0]!.reviewId), + ).rejects.toThrow(/review/i); + cloud.state.environmentId = reviewedEnvironment; + expect(cloud.posts).toHaveLength(0); + + // The caller saves the reviewed selections before sending through the existing transport. + for (const group of groups.slice(0, index)) + expect((await group.publish(group.reviewId)).status).toBe("complete"); + cloud.state.loseResponse = true; + const interrupted = groups[index]!; + await expect( + interrupted.publish(interrupted.reviewId), + ).rejects.toMatchObject({ + result: { status: "interrupted" }, + }); + expect(cloud.posts).toHaveLength(index + 1); + const lost = cloud.posts[index]!; + if (index === 1) { + const completed = (await local.prepare(groups[0]!.selection)).groups[0]!; + await expect(completed.publish(groups[0]!.reviewId)).rejects.toThrow( + /review/i, + ); + } + + for (const group of groups.slice(index)) { + const retry = await local.prepare(group.selection); + expect(retry.groups).toHaveLength(1); + const resumed = retry.groups[0]!; + // Changing the display copy cannot redirect the approved request or its receipt. + resumed.preview.findings[0]!.source_finding_id = "different-id"; + resumed.preview.destination.id = "different-destination"; + expect((await resumed.publish(group.reviewId)).status).toBe("complete"); + } + expect(cloud.posts).toHaveLength(3); + expect(cloud.posts.filter((body) => body === lost)).toHaveLength(2); + expect(cloud.reports.size).toBe(2); + const history = await listLocalFindingPublications({ repository }, deps); + expect(history).toHaveLength(2); + for (const row of selected) { + const remoteRow = cloud.reports.get(row.sourceFindingId)!; + expect(remoteRow).toMatchObject({ + source: { provider: row.provider, source_key: row.sourceKey }, + source_finding_id: row.sourceFindingId, + version: 1, + evidence: { title: row.evidence.title }, + }); + const link = history.find((item) => item.id === row.id)!; + expect(link).toMatchObject({ + version: row.version, + outcome: "created", + cloudFindingId: remoteRow.canonical_finding_id, + destinationId: options.repository, + }); + expect(cloud.receipts.has(link.receiptId)).toBe(true); + } + expect(await listLocalExternalFindings({ repository }, deps)).toEqual(rows); + }, +); + +test("stored publication retries a failed SQLite receipt write without another upload", async () => { + const cloud = await fixture(); + await using local = await storedPublicationFixture(cloud); + const imported = await local.store("wiz", { + read: 1, + findings: [localFinding("recover")], + excluded: [], + }); + const { repository } = local; + const [group] = ( + await local.prepare({ repository, importId: imported.importId! }) + ).groups; + using database = new Database( + join(local.stateDirectory, "workbench.sqlite3"), + ); + database.exec(`CREATE TRIGGER reject_receipts BEFORE INSERT ON external_finding_publications + BEGIN SELECT RAISE(ABORT, 'receipt write interrupted'); END`); + await expect(group!.publish(group!.reviewId)).rejects.toThrow( + /receipt write interrupted/, + ); + expect(cloud.posts).toHaveLength(1); + expect( + await listLocalFindingPublications({ repository }, local.deps), + ).toEqual([]); + database.exec("DROP TRIGGER reject_receipts"); + + const [retry] = (await local.prepare(group!.selection)).groups; + expect(retry!.reviewId).toBe(group!.reviewId); + const result = await retry!.publish(group!.reviewId); + expect(result.status).toBe("complete"); + expect(cloud.posts).toHaveLength(1); + expect( + await listLocalFindingPublications({ repository }, local.deps), + ).toMatchObject([ + { + id: imported.findings[0]!.id, + receiptId: result.receipts[0]!.id, + outcome: "created", + }, + ]); +}); + +test("stored findings recover a private inbox upload and retain its job receipt", async () => { + const { f, inbox, deps, uploads, jobs } = await privateUploadFixture( + [], + "wiz", + ); + await using local = await storedPublicationFixture(f, deps.fetch); + const { repository } = local; + const row = localFinding(); + const imported = await local.store("wiz", { + read: 1, + findings: [row], + excluded: [], + }); + const selection = { repository, importId: imported.importId! }; + const [first] = (await local.prepare(selection)).groups; + expect(first!.preview.privateImport).toBeDefined(); + expect(first!.selection).toEqual(selection); + await expect(first!.publish(first!.reviewId)).rejects.toMatchObject({ + result: { status: "interrupted" }, + }); + expect( + await listLocalFindingPublications({ repository }, local.deps), + ).toEqual([]); + inbox.recoverUpload = true; + const [retry] = (await local.prepare(first!.selection)).groups; + expect((await retry!.publish(first!.reviewId)).status).toBe("complete"); + expect(uploads).toHaveLength(1); + expect(JSON.parse(uploads[0]!.bytes).findings[0].source_finding_id).toBe( + row.source_finding_id, + ); + expect(jobs).toHaveLength(1); + const receipts = await listLocalFindingPublications( + { repository }, + local.deps, + ); + expect(receipts).toHaveLength(1); + expect(receipts[0]).toMatchObject({ + id: imported.findings[0]!.id, + version: 1, + outcome: "private_job", + destinationId: options.repository, + receiptId: "import_" + "b".repeat(32), + }); +}); + test.each([true, false])( "repository severity exclusions keep the approved requests on retry (initial advertisement %s)", async (advertised) => { @@ -8561,3 +8783,148 @@ test("a private structured report retains occurrence IDs, unknown severity and s privateReport(input, { ...limits, max_record_bytes: 32 }), ).toThrow("record limit"); }); + +function localFinding(id?: string, severity?: string) { + const row = normalized(id, severity); + return { ...row, evidence: validateExternalEvidence(row.evidence, "local") }; +} + +/** Use one disposable checkout and SQLite database for import, review and receipt lookup. */ +async function storedPublicationFixture( + cloud: Awaited>, + fetch = cloud.deps.fetch, +) { + const local = await localFindingsFixture({ + remote: cloud.state.repositoryUrl, + }); + const { repository } = local; + const deps = { + fetch, + environment: { + ...local.environment, + CODEX_HOME: cloud.environment.CODEX_HOME, + }, + }; + return { + ...local, + deps, + prepare: (selection: LocalExternalFindingSelection) => + prepareStoredFindingsPublication(selection, options, deps), + async store( + provider: string, + input: ExternalFindings, + sourceRepositoryUrl?: string, + ) { + const prepared = await prepareNormalizedLocalImport( + input, + { repository, provider, sourceKey: "selected", sourceRepositoryUrl }, + deps, + ); + return prepared.import(); + }, + }; +} + +test("stored publication reports unsupported providers and Cloud-only exclusions against their local versions", async () => { + const cloud = await fixture(); + await using local = await storedPublicationFixture(cloud); + const { repository, deps } = local; + cloud.state.catalogHeaders["Codex-Security-Repository-Import-Severities"] = + "high"; + const store = ( + provider: string, + ids: string[], + report?: Record, + ) => + local.store( + provider, + { + read: ids.length, + findings: ids.map((id) => + localFinding(id, id === "unscored" ? "unknown" : "high"), + ), + excluded: [], + report, + }, + cloud.state.repositoryUrl, + ); + await store("wiz", ["ready", "unscored", "too-long-" + "x".repeat(512)]); + await store("tenant-scanner", ["local-only"]); + const rows = await listLocalExternalFindings({ repository }, deps); + const selection = { + repository, + findings: rows.map(({ id, version }) => ({ id, version })), + }; + const excluded = rows.filter((row) => row.sourceFindingId !== "ready"); + const prepared = await local.prepare(selection); + const group = prepared.groups[0]!; + expect(group.preview.findings.map((f) => f.source_finding_id)).toEqual([ + "ready", + ]); + for (const { id, version, provider, sourceFindingId } of excluded) { + expect(prepared.excluded).toContainEqual( + expect.objectContaining({ + id, + version, + reason: expect.stringMatching( + provider !== "wiz" + ? /tenant-scanner/ + : sourceFindingId === "unscored" + ? /unknown/ + : /512/, + ), + }), + ); + } + expect((await group.publish(group.reviewId)).status).toBe("partial"); + expect(cloud.posts).toHaveLength(1); + expect( + (await listLocalFindingPublications({ repository }, deps)).map((r) => r.id), + ).toEqual([rows.find((r) => r.sourceFindingId === "ready")!.id]); + const failed = await store("wiz", ["z".repeat(513)], { + errors: ["scanner permission denied"], + }); + await expect( + local.prepare({ repository, importId: failed.importId! }), + ).rejects.toThrow(/scanner permission denied/); + cloud.state.repositoryUrl = "https://github.com/example/another"; + await expect(local.prepare(selection)).rejects.toThrow(/does not match/); + expect(cloud.posts).toHaveLength(1); +}); + +test.each(["trivy", "endor"] as const)( + "stored native %s rows use the file reader's Cloud projection when enriched fields exceed its limits", + async (provider) => { + const cloud = await fixture(); + await using local = await storedPublicationFixture(cloud); + const base = endorFinding(); + const { target_dependency_name: _, ...spec } = base.spec; + const raw = { + ...base, + spec: { ...spec, target_dependency_package_name: "x".repeat(513) }, + }; + const report = trivyReportWithManyLocations(); + const read = (target: "local" | "cloud") => + provider === "trivy" + ? normalizeTrivyFindings(report, target, ".") + : normalizeEndorFindings(raw, target); + const input = read("local"); + const expected = read("cloud"); + if (provider === "trivy") + expect( + input.findings.some((f) => f.evidence.locations!.length > 100), + ).toBe(true); + else + expect(input.findings[0]!.evidence.packages).not.toEqual( + expected.findings[0]!.evidence.packages, + ); + const result = await local.store(provider, input); + const prepared = await local.prepare({ + repository: local.repository, + importId: result.importId!, + }); + expect(prepared.groups[0]!.preview.findings).toEqual(expected.findings); + expect(prepared.excluded).toEqual([]); + expect(cloud.posts).toHaveLength(0); + }, +); diff --git a/sdk/typescript/tests-ts/inspector-repository-import.test.ts b/sdk/typescript/tests-ts/inspector-repository-import.test.ts index c2925d2199..af87ea896f 100644 --- a/sdk/typescript/tests-ts/inspector-repository-import.test.ts +++ b/sdk/typescript/tests-ts/inspector-repository-import.test.ts @@ -173,7 +173,7 @@ test.each( const prepare = (url: string) => prepareNormalizedLocalImport( normalized([typedRow(url)]), - f.options, + { ...f.options, sourceRepositoryUrl: undefined }, f.deps, ); await expect(prepare(other)).rejects.toThrow( diff --git a/sdk/typescript/tests-ts/local-finding-report.test.ts b/sdk/typescript/tests-ts/local-finding-report.test.ts new file mode 100644 index 0000000000..f559e6ca43 --- /dev/null +++ b/sdk/typescript/tests-ts/local-finding-report.test.ts @@ -0,0 +1,160 @@ +import { existsSync } from "node:fs"; +import { expect, test } from "bun:test"; +import { + listLocalExternalImports, + prepareNormalizedLocalImport, + selectLocalExternalFindings, + type ExternalFindings, +} from "../src/index.js"; +import { localFindingsFixture } from "./external-findings-local-helpers.js"; + +const source = { + provider: "example-scanner", + sourceKey: "project/main", + sourceRepositoryUrl: "https://github.com/example/project", +}; + +function report(): ExternalFindings { + return { + read: 2, + findings: [ + { + source_finding_id: "9007199254740993", + evidence: { title: "Unsafe subprocess", severity: "high" }, + }, + ], + excluded: [ + { position: 2, source_finding_id: null, reason: "Missing vendor ID" }, + ], + report: { job: "example-run" }, + }; +} + +test("a parser-defined provider stores its reviewed selection and can retrieve it by import ID", async () => { + await using fixture = await localFindingsFixture({ + remote: "git@github.com:example/project.git", + }); + const { repository, environment } = fixture; + const input = report(); + const prepare = () => + prepareNormalizedLocalImport( + input, + { ...source, repository }, + { environment }, + ); + const prepared = await prepare(); + expect(prepared.matchesRepositoryRemote).toBe(true); + expect(existsSync(fixture.stateDirectory)).toBe(false); + + prepared.preview.findings[0]!.evidence.title = "Edited preview"; + const first = await prepared.import(); + const repeat = await (await prepare()).import(); + expect(repeat.counts.unchanged).toBe(1); + const imports = await listLocalExternalImports( + { repository }, + { environment }, + ); + expect(imports.map((i) => i.importId)).toEqual([ + repeat.importId!, + first.importId!, + ]); + + const select = () => + selectLocalExternalFindings( + { repository, importId: repeat.importId! }, + { environment }, + ); + expect(await select()).toMatchObject({ + read: input.read, + report: input.report, + excluded: input.excluded, + findings: [ + { + sourceFindingId: "9007199254740993", + evidence: input.findings[0]!.evidence, + version: 1, + provenance: { + sourceRepositories: [{ url: source.sourceRepositoryUrl }], + }, + }, + ], + }); + input.findings[0]!.evidence.title = "New evidence"; + await (await prepare()).import(); + await expect(select()).rejects.toThrow(/changed/); +}); + +test("agent inputs account for every occurrence and validate identities and evidence", async () => { + await using fixture = await localFindingsFixture(); + const prepare = (input: ExternalFindings, options = source) => + prepareNormalizedLocalImport( + input, + { ...options, repository: fixture.repository }, + { environment: fixture.environment }, + ); + const input = report(); + const [row] = input.findings; + const [excluded] = input.excluded; + await expect(prepare(input, { ...source, provider: "" })).rejects.toThrow(); + await expect( + prepare(input, { ...source, sourceKey: String.fromCharCode(0xd800) }), + ).rejects.toThrow(); + for (const invalid of [ + { ...input, read: 3 }, + { ...input, findings: [row, row], read: 3 }, + { ...input, excluded: [{ ...excluded, position: 3 }] }, + { ...input, read: 3, excluded: [excluded, excluded] }, + ]) + await expect(prepare(invalid as ExternalFindings)).rejects.toThrow(); + const missingSeverity = { + ...input, + findings: [{ ...row, evidence: { title: "Missing severity" } }], + }; + for (const provider of [source.provider, "wiz"]) + await expect( + prepare(missingSeverity as ExternalFindings, { ...source, provider }), + ).rejects.toThrow(); +}); + +test("a custom parser binds all declared sources to one repository", async () => { + await using fixture = await localFindingsFixture({ + remote: source.sourceRepositoryUrl, + }); + const input = report(); + const prepare = (sourceRepositoryUrl?: string, provider = source.provider) => + prepareNormalizedLocalImport( + input, + { + ...source, + provider, + repository: fixture.repository, + sourceRepositoryUrl, + }, + { environment: fixture.environment }, + ); + expect((await prepare()).matchesRepositoryRemote).toBe(false); + const other = "https://github.com/example/other"; + await expect(prepare(other)).rejects.toThrow(/does not match/u); + for (const provider of ["wiz", "snyk"]) + await expect(prepare(other, provider)).rejects.toThrow(/does not match/u); + fixture.git("remote", "add", "upstream", other); + const details = { + kind: "sast" as const, + repository: { url: source.sourceRepositoryUrl }, + }; + input.findings[0]!.evidence.details = details; + await expect(prepare(other)).rejects.toThrow(/conflicting/u); + await expect(prepare(other, "wiz")).rejects.toThrow(/does not match/u); + delete input.findings[0]!.evidence.details; + input.findings[0]!.evidence.source_data = { + vulnerableAsset: { + nativeType: "github#repositoryBranch", + cloudProviderURL: source.sourceRepositoryUrl + "/tree/main", + }, + }; + await expect(prepare(other, "wiz")).rejects.toThrow(/does not match/u); + input.findings[0]!.evidence.details = details; + details.repository.url += "/../other"; + await expect(prepare()).rejects.toThrow(); + expect(existsSync(fixture.stateDirectory)).toBe(false); +}); diff --git a/sdk/typescript/tests-ts/opengrep-findings.test.ts b/sdk/typescript/tests-ts/opengrep-findings.test.ts index dd94e24e44..00c84b72f3 100644 --- a/sdk/typescript/tests-ts/opengrep-findings.test.ts +++ b/sdk/typescript/tests-ts/opengrep-findings.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { normalizeOpenGrepFindings as normalize, + projectStoredOpenGrepFinding, readOpenGrepFindings, } from "../src/opengrep-findings.js"; import { @@ -138,6 +139,45 @@ test("one source occurrence retains every trace observation in deterministic ord expect(input.results.reverse()).toEqual(original.results); }); +test("stored OpenGrep groups publish eligible sibling traces and identify every excluded retained trace", () => { + const input = opengrepTraceJson(); + const deep = structuredClone(input.results[0]!); + let trace: unknown = "leaf"; + for (let level = 0; level < 20; level++) trace = { child: trace }; + Object.assign(deep.extra.metadata, { trace }); + input.results.push(deep); + const stored = normalize(input, options, "local").findings[0]!; + const traces = stored.evidence.source_data![ + "results" + ] as typeof input.results; + expect(traces).toHaveLength(3); + const native = normalize({ ...input, results: traces }, options, "cloud"); + const projected = projectStoredOpenGrepFinding(stored)!; + expect(projected.read).toBe(3); + expect(projected.findings).toEqual(native.findings); + expect(projected.excluded).toEqual(native.excluded); + expect(projected.excluded).toEqual([ + { + position: + traces.findIndex((result) => "trace" in result.extra.metadata) + 1, + source_finding_id: stored.source_finding_id, + reason: expect.stringContaining("depth limit"), + }, + ]); + expect(projected.findings[0]!.evidence.source_data!["results"]).toHaveLength( + 2, + ); + expect(projected.findings[0]!.evidence.source_data!["scan_error_count"]).toBe( + input.errors.length, + ); + expect( + projectStoredOpenGrepFinding({ ...stored, source_finding_id: "custom-id" }), + ).toBeUndefined(); + const edited = structuredClone(stored); + edited.evidence.title = "Reviewed title"; + expect(projectStoredOpenGrepFinding(edited)).toBeUndefined(); +}); + test("an invalid observation does not hide a valid sibling at the same source range", () => { const input: any = opengrepJson(); const valid = structuredClone(input.results[0]); diff --git a/sdk/typescript/tests-ts/semgrep-guardian-findings.test.ts b/sdk/typescript/tests-ts/semgrep-guardian-findings.test.ts index 32e82e773e..8dccd7df30 100644 --- a/sdk/typescript/tests-ts/semgrep-guardian-findings.test.ts +++ b/sdk/typescript/tests-ts/semgrep-guardian-findings.test.ts @@ -70,6 +70,19 @@ test("a signed-in Guardian selection and Web API collection keep independent row }); expect(await fixture.read()).toHaveLength(0); expect((await preview.import()).counts.created).toBe(1); + const replay = await prepareNormalizedLocalImport( + preview.preview, + { + repository: fixture.repository, + provider: "semgrep", + sourceKey: fixture.sourceKey, + }, + { + environment: fixture.environment, + sourceRepositories: [{ url: sourceRepositoryUrl }], + }, + ); + expect((await replay.import()).counts.unchanged).toBe(1); // Guardian returns a string; the Web API can send the same digits as a JSON integer. const webId = (JSON as JSON & { rawJSON(value: string): unknown }).rawJSON( diff --git a/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts b/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts index bcf9d4c1d2..6370dd2303 100644 --- a/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts +++ b/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts @@ -11,7 +11,10 @@ import { semgrepPlatformFindingBinding, semgrepPlatformFindingRepository, } from "../src/semgrep-platform-findings.js"; -import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; +import { + prepareNormalizedLocalImport, + selectLocalExternalFindings, +} from "../src/external-findings-local.js"; import { hostedRepositoryUrlKey } from "../src/hosted-git-repository.js"; import { normalizeSemgrepFindings } from "../src/semgrep-findings.js"; import { dependencies } from "./cli-fixtures.js"; @@ -1659,6 +1662,11 @@ test("a verified alias remains replayable with or without a reader platform hint const read = (saved: typeof report) => normalizeSemgrepFindings(saved, { sourceRepositories }, "local"); expect(read(report).findings[0]).toEqual(row); + const changedSpelling = await prepareNormalizedLocalImport(input, settings, { + ...dependencies, + sourceRepositories: [{ url: remote.toLowerCase(), platform: "github" }], + }); + expect((await changedSpelling.import()).counts.updated).toBe(1); const withoutProof = read({ ...report, @@ -1671,6 +1679,20 @@ test("a verified alias remains replayable with or without a reader platform hint expect(() => semgrepPlatformFindingRepository(withoutProof, "semgrep"), ).toThrow(/witness/); + const dependentImport = await prepareNormalizedLocalImport( + { read: 1, findings: [withoutProof], excluded: [] }, + settings, + dependencies, + ); + const saved = await dependentImport.import(); + expect(saved.counts.updated).toBe(1); + const retained = await selectLocalExternalFindings( + { repository: settings.repository, importId: saved.importId! }, + dependencies, + ); + expect(retained.findings[0]?.provenance?.sourceRepositories).toEqual( + sourceRepositories, + ); for (const witness of [ { name: "other", id: "77" }, { ...repository, id: "99" }, diff --git a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts index e85e78eba4..80915f925e 100644 --- a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts +++ b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts @@ -17,6 +17,7 @@ import { normalizeSemgrepFindings } from "../src/semgrep-findings.js"; import { normalizeSemgrepPlatformFindings, selectSemgrepPlatformFindings, + storedSemgrepCloudEvidence, } from "../src/semgrep-platform-findings.js"; import { semgrepSarif } from "./support/semgrep-import-fixtures.js"; import { localReportFixture as reportFixture } from "./external-findings-local-helpers.js"; @@ -130,15 +131,23 @@ test("Semgrep message titles stay intact by default and fall back above 512 Clou ).findings[0]?.evidence.title, ).toBe(exact); const report = selectRow({ rule: { message } }); - expect( - normalizeSemgrepPlatformFindings(report).findings[0]?.evidence.title, - ).toBe(message); - expect( - normalizeSemgrepPlatformFindings(report, {}, "cloud").findings[0]?.evidence, - ).toMatchObject({ + const stored = normalizeSemgrepPlatformFindings(report).findings[0]!; + const cloud = normalizeSemgrepPlatformFindings(report, {}, "cloud") + .findings[0]!.evidence; + expect(stored.evidence.title).toBe(message); + expect(cloud).toMatchObject({ title: "Semgrep finding 8", description: message, }); + expect(storedSemgrepCloudEvidence(stored)).toEqual(cloud); + const amended = { + ...stored.evidence, + title: message + "!", + description: message + "!", + }; + expect( + storedSemgrepCloudEvidence({ ...stored, evidence: amended }), + ).toBeUndefined(); }); test("two selected Platform SAST IDs from separate pages replay unchanged in SQLite", async () => { diff --git a/sdk/typescript/tests-ts/sonar-import.test.ts b/sdk/typescript/tests-ts/sonar-import.test.ts index 36606a2e31..f3708fe63b 100644 --- a/sdk/typescript/tests-ts/sonar-import.test.ts +++ b/sdk/typescript/tests-ts/sonar-import.test.ts @@ -16,7 +16,10 @@ import { import { basename, dirname, join } from "node:path"; import { prepareSonarImport } from "../src/sonar-import.js"; import { prepareLocalExternalImport } from "../src/external-findings-import.js"; -import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; +import { + prepareNormalizedLocalImport, + selectLocalExternalFindings, +} from "../src/external-findings-local.js"; import { normalizeSonarFindings, parseSonarCapture, @@ -421,8 +424,23 @@ test("Cloud confirmation verifies the retained evidence, account and assigned en yes: true, dryRun: false, }; - expect((await prepareSonarImport(confirm, deps)).cloud).toBeDefined(); + const prepared = await prepareSonarImport(confirm, deps); + expect(prepared.cloud).toBeDefined(); reportVersion = 1; + const changedAfterPreview = await prepared.import(); + expect(changedAfterPreview.local.counts.created).toBe(1); + expect(changedAfterPreview.cloud).toBeUndefined(); + expect(changedAfterPreview.cloudError).toContain( + "Preview the Sonar capture again", + ); + const saved = await selectLocalExternalFindings( + { + repository: selection.cwd, + importId: changedAfterPreview.local.importId!, + }, + deps, + ); + expect(saved.findings.map((row) => row.sourceFindingId)).toEqual(["issue-1"]); expect((await prepareSonarImport(confirm, deps)).cloudError).toContain( "--to cloud --dry-run", ); diff --git a/sdk/typescript/tests-ts/stored-finding-projection.test.ts b/sdk/typescript/tests-ts/stored-finding-projection.test.ts new file mode 100644 index 0000000000..01cad9aa27 --- /dev/null +++ b/sdk/typescript/tests-ts/stored-finding-projection.test.ts @@ -0,0 +1,85 @@ +import { expect, test } from "bun:test"; +import { normalizeBlackDuckFindings } from "../src/blackduck-findings.js"; +import { normalizeContinuumFindings } from "../src/continuum-findings.js"; +import type { ExternalImportTarget } from "../src/external-import-contract.js"; +import type { ExternalFindings } from "../src/external-findings.js"; +import { normalizeInspectorFindings } from "../src/inspector-findings.js"; +import { projectStoredFinding } from "../src/stored-finding-projection.js"; +import { + blackDuckBomReport, + blackDuckBomRow, +} from "./support/blackduck-import-fixtures.js"; +import { + continuumDetail, + continuumRepository, +} from "./support/continuum-import-fixtures.js"; +import { + inspectorCodeFinding, + inspectorRepositoryUrl, +} from "./support/inspector-import-fixtures.js"; + +const blackDuck = { + ...blackDuckBomRow(), + packageUrl: `pkg:github/example/${"x".repeat(513)}@1.2.3`, +}; +const continuum = { + ...continuumDetail(), + affectedPackages: [ + { name: "界".repeat(200) }, + ...Array.from({ length: 101 }, (_, i) => ({ name: `package-${i}` })), + ], +}; +const inspector = inspectorCodeFinding(); +inspector.codeVulnerabilityDetails.detectorId = "x".repeat(513); + +const cases: [string, (target: ExternalImportTarget) => ExternalFindings][] = [ + [ + "blackduck", + (target) => + normalizeBlackDuckFindings( + blackDuckBomReport([blackDuck]), + continuumRepository, + target, + ), + ], + [ + "continuum", + (target) => + normalizeContinuumFindings(continuum, continuumRepository, target), + ], + [ + "inspector", + (target) => + normalizeInspectorFindings( + { findings: [inspector] }, + inspectorRepositoryUrl, + target, + ), + ], +]; + +test.each(cases)( + "stored %s enrichment uses the native Cloud projection only for matching reviewed evidence", + (provider, read) => { + const local = read("local").findings[0]!; + const cloud = read("cloud"); + expect(cloud.findings).toHaveLength(1); + expect(local.evidence).not.toEqual(cloud.findings[0]!.evidence); + const row = { + provider, + sourceFindingId: local.source_finding_id, + evidence: local.evidence, + }; + expect(projectStoredFinding(row, []).findings).toEqual(cloud.findings); + const edited = { + ...row, + evidence: { + ...row.evidence, + description: "Operator reviewed and edited this finding.", + }, + }; + const selection = projectStoredFinding(edited, []); + expect(selection.findings).toEqual([]); + expect(selection.excluded).toHaveLength(1); + }, +); diff --git a/sdk/typescript/tests-ts/veracode-repository-import.test.ts b/sdk/typescript/tests-ts/veracode-repository-import.test.ts index 31e32ae2e5..f67d4ac456 100644 --- a/sdk/typescript/tests-ts/veracode-repository-import.test.ts +++ b/sdk/typescript/tests-ts/veracode-repository-import.test.ts @@ -70,7 +70,11 @@ test("replaying a Veracode policy flaw tracks evidence without assessing another typed.evidence.details!.repository!.url = "git@github.com:example/mirror.git"; const prepareMixed = () => - prepareNormalizedLocalImport(mixed, options, dependencies); + prepareNormalizedLocalImport( + mixed, + { ...options, sourceRepositoryUrl: undefined }, + dependencies, + ); await expect(prepareMixed()).rejects.toThrow(/configured remote/); fixture.git("remote", "add", "mirror", "https://github.com/example/mirror"); expect((await prepareMixed()).preview.findings).toHaveLength(2); From dad2ca82061d4aa645d8d0166337333cf04bc0ad Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 13:36:51 +0000 Subject: [PATCH 2/7] fix(imports): preserve acknowledged batches when Cloud rejects a later batch --- sdk/typescript/scripts/package-dist-files.mjs | 1 - sdk/typescript/src/external-findings-local.ts | 2 +- .../src/external-findings-publish.ts | 41 ++++++- .../src/external-publication-review.ts | 34 ------ sdk/typescript/src/sonar-import.ts | 2 +- .../src/stored-findings-publication.ts | 9 +- .../external-findings-publish.test.ts | 108 +++++++++++------- 7 files changed, 111 insertions(+), 86 deletions(-) delete mode 100644 sdk/typescript/src/external-publication-review.ts diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index c975a49cb6..830dc8b3bd 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -60,7 +60,6 @@ const distModules = [ "trivy-findings", "trivy-import", "external-findings-publish", - "external-publication-review", "stored-finding-projection", "stored-findings-publication", "private-finding-import", diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts index be515d1a9e..50ca5e0f0d 100644 --- a/sdk/typescript/src/external-findings-local.ts +++ b/sdk/typescript/src/external-findings-local.ts @@ -152,7 +152,7 @@ export interface StoredExternalImport { count: number; } -interface LocalFindingReceipt { +export interface LocalFindingReceipt { id: string; version: number; receiptId: string; diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 8e23ad3611..4b79f7237e 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -223,6 +223,37 @@ export interface ExternalPublicationPreview extends ExternalFindings { }; } +/** Bind reviewed evidence to its account, repository, connector and destination version. */ +export function cloudReviewDigest( + preview: ExternalPublicationPreview, + checkout: string, + findings?: readonly { id: string; version: number }[], +) { + return hash( + "sha256", + canonicalJson({ + checkout, + api: preview.cloudApiUrl, + account: preview.accountId, + repository: preview.destination.id, + connector: preview.destination.repo_connector_id, + url: preview.destination.url, + reset: preview.destination.reset_marker, + source: preview.source, + privateImport: preview.privateImport, + ...(findings && { findings, excluded: preview.excluded }), + items: preview.requests.flatMap((request) => + request.items.map((item) => [ + item.source_finding_id, + item.expected_version, + request.repository.environment_id, + item.evidence, + ]), + ), + }), + ); +} + export interface PreparedExternalPublication { preview: ExternalPublicationPreview; /** Persist acknowledged repository receipts before retiring their resumable request. */ @@ -2063,8 +2094,14 @@ async function preparePublicationInput( return completed; } catch (error) { if (error instanceof CloudImportError && error.status === 409) { - await removePending(); - pending = undefined; + try { + await beforeRetire?.(result(acknowledged, verified)); + await removePending(); + pending = undefined; + } catch (recordError) { + // Keep acknowledged batches resumable when their receipts are not durable. + error = recordError; + } } const message = `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : pending ? `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}` : "No resumable request was saved. Correct the error and review the input before retrying."}`; throw new ExternalPublicationError( diff --git a/sdk/typescript/src/external-publication-review.ts b/sdk/typescript/src/external-publication-review.ts deleted file mode 100644 index 806cb63280..0000000000 --- a/sdk/typescript/src/external-publication-review.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { hash } from "node:crypto"; -import type { ExternalPublicationPreview } from "./external-findings-publish.js"; -import { canonicalJson } from "./record.js"; - -/** Bind reviewed evidence to its account, repository, connector and destination version. */ -export function cloudReviewDigest( - preview: ExternalPublicationPreview, - checkout: string, - findings?: readonly { id: string; version: number }[], -) { - return hash( - "sha256", - canonicalJson({ - checkout, - api: preview.cloudApiUrl, - account: preview.accountId, - repository: preview.destination.id, - connector: preview.destination.repo_connector_id, - url: preview.destination.url, - reset: preview.destination.reset_marker, - source: preview.source, - privateImport: preview.privateImport, - ...(findings && { findings, excluded: preview.excluded }), - items: preview.requests.flatMap((request) => - request.items.map((item) => [ - item.source_finding_id, - item.expected_version, - request.repository.environment_id, - item.evidence, - ]), - ), - }), - ); -} diff --git a/sdk/typescript/src/sonar-import.ts b/sdk/typescript/src/sonar-import.ts index cd462b562c..0a4d66ce9a 100644 --- a/sdk/typescript/src/sonar-import.ts +++ b/sdk/typescript/src/sonar-import.ts @@ -10,9 +10,9 @@ import { prepareLocalImport, } from "./external-findings-local.js"; import { prepareStoredFindingsPublication } from "./stored-findings-publication.js"; -import { cloudReviewDigest } from "./external-publication-review.js"; import { commandGitProtectionRoots } from "./local-git-metadata.js"; import { + cloudReviewDigest, ExternalPublicationError, externalPublicationStateDirectory, prepareNormalizedExternalPublication, diff --git a/sdk/typescript/src/stored-findings-publication.ts b/sdk/typescript/src/stored-findings-publication.ts index 34a533d033..69248ad5cf 100644 --- a/sdk/typescript/src/stored-findings-publication.ts +++ b/sdk/typescript/src/stored-findings-publication.ts @@ -7,10 +7,12 @@ import { recordExternalFindingPublications, selectLocalExternalFindings, type LocalExternalFindingSelection, + type LocalFindingReceipt, type LocalExternalImportDependencies, type StoredExternalFinding, } from "./external-findings-local.js"; import { + cloudReviewDigest, ExternalPublicationError, prepareNormalizedExternalPublication, type ExternalPublicationDependencies, @@ -24,7 +26,6 @@ import { type ExternalFindings, type ExternalSourceRepository, } from "./external-findings.js"; -import { cloudReviewDigest } from "./external-publication-review.js"; import { projectStoredFinding } from "./stored-finding-projection.js"; type Dependencies = ExternalPublicationDependencies & @@ -109,10 +110,6 @@ function cloudSelection( return selected; } -type ReceiptLink = Parameters< - typeof recordExternalFindingPublications ->[0]["entries"][number]; - function receiptRecorder( preview: ExternalPublicationPreview, rows: StoredExternalFinding[], @@ -142,7 +139,7 @@ function receiptRecorder( destinationId: preview.destination.id, }; return async (result: ExternalPublicationResult) => { - const entries: ReceiptLink[] = []; + const entries: LocalFindingReceipt[] = []; const job = result.privateJob; // The private inbox provides job totals, with no per-item outcomes. if ( diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 3b7e29987d..ca5ca0155d 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -483,6 +483,7 @@ async function fixture( catalogHeaders: {} as Record, sourceLookups: [] as string[], postBudget: Infinity, + conflictingFindingId: undefined as string | undefined, finalError: false, }; const destination = () => ({ @@ -575,6 +576,17 @@ async function fixture( return json(response, { error: { message: "Slow down" } }, 429); } state.postBudget--; + if ( + state.conflictingFindingId && + request.items.some( + (item) => item.source_finding_id === state.conflictingFindingId, + ) + ) + return json( + response, + { error: { message: "stale expected version" } }, + 409, + ); if (request.repository.reset_marker !== state.marker) return json( response, @@ -866,47 +878,61 @@ test.each([0, 1])( }, ); -test("stored publication retries a failed SQLite receipt write without another upload", async () => { - const cloud = await fixture(); - await using local = await storedPublicationFixture(cloud); - const imported = await local.store("wiz", { - read: 1, - findings: [localFinding("recover")], - excluded: [], - }); - const { repository } = local; - const [group] = ( - await local.prepare({ repository, importId: imported.importId! }) - ).groups; - using database = new Database( - join(local.stateDirectory, "workbench.sqlite3"), - ); - database.exec(`CREATE TRIGGER reject_receipts BEFORE INSERT ON external_finding_publications - BEGIN SELECT RAISE(ABORT, 'receipt write interrupted'); END`); - await expect(group!.publish(group!.reviewId)).rejects.toThrow( - /receipt write interrupted/, - ); - expect(cloud.posts).toHaveLength(1); - expect( - await listLocalFindingPublications({ repository }, local.deps), - ).toEqual([]); - database.exec("DROP TRIGGER reject_receipts"); - - const [retry] = (await local.prepare(group!.selection)).groups; - expect(retry!.reviewId).toBe(group!.reviewId); - const result = await retry!.publish(group!.reviewId); - expect(result.status).toBe("complete"); - expect(cloud.posts).toHaveLength(1); - expect( - await listLocalFindingPublications({ repository }, local.deps), - ).toMatchObject([ - { - id: imported.findings[0]!.id, - receiptId: result.receipts[0]!.id, - outcome: "created", - }, - ]); -}); +test.each(["complete", "conflict"])( + "stored publication recovers SQLite receipts before retiring a %s submission", + async (finish) => { + const cloud = await fixture(); + const rows = Array.from( + { length: finish === "conflict" ? 101 : 1 }, + (_, i) => localFinding("row-" + i), + ); + if (finish === "conflict") cloud.state.conflictingFindingId = "row-100"; + await using local = await storedPublicationFixture(cloud); + const imported = await local.store("wiz", { + read: rows.length, + findings: rows, + excluded: [], + }); + const { repository } = local; + const [group] = ( + await local.prepare({ repository, importId: imported.importId! }) + ).groups; + using database = new Database( + join(local.stateDirectory, "workbench.sqlite3"), + ); + database.exec(`CREATE TRIGGER reject_receipts BEFORE INSERT ON external_finding_publications + BEGIN SELECT RAISE(ABORT, 'receipt write interrupted'); END`); + await expect(group!.publish(group!.reviewId)).rejects.toThrow( + /receipt write interrupted/, + ); + expect(cloud.receipts.size).toBe(1); + const [originalReceiptId] = cloud.receipts.keys(); + expect( + await listLocalFindingPublications({ repository }, local.deps), + ).toEqual([]); + database.exec("DROP TRIGGER reject_receipts"); + cloud.state.conflictingFindingId = undefined; + + const [retry] = (await local.prepare(group!.selection)).groups; + expect(retry!.reviewId).toBe(group!.reviewId); + const result = await retry!.publish(group!.reviewId); + expect(result.status).toBe("complete"); + const firstBatch = cloud.posts[0]!; + expect(cloud.posts.filter((body) => body === firstBatch)).toHaveLength(1); + const links = await listLocalFindingPublications( + { repository }, + local.deps, + ); + expect(links).toHaveLength(rows.length); + expect(links).toContainEqual( + expect.objectContaining({ + id: imported.findings[0]!.id, + receiptId: originalReceiptId, + outcome: "created", + }), + ); + }, +); test("stored findings recover a private inbox upload and retain its job receipt", async () => { const { f, inbox, deps, uploads, jobs } = await privateUploadFixture( From 32f9143ad2ba2577ef45a21f77f5dfef0e25476e Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 13:36:38 +0000 Subject: [PATCH 3/7] test(workbench): restore the v53 schema before replaying import migrations --- plugins/codex-security/tests/test_workbench_deep_scan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/codex-security/tests/test_workbench_deep_scan.py b/plugins/codex-security/tests/test_workbench_deep_scan.py index 32748e3ff2..2563cf8aab 100644 --- a/plugins/codex-security/tests/test_workbench_deep_scan.py +++ b/plugins/codex-security/tests/test_workbench_deep_scan.py @@ -142,6 +142,9 @@ def restore_v53_schema(state_dir: Path) -> None: connection.execute("ALTER TABLE scans DROP COLUMN launch_package_version") connection.execute("ALTER TABLE scans DROP COLUMN launch_surface") connection.executescript(""" + DROP TABLE external_finding_publications; + DROP TABLE external_finding_import_members; + DROP TABLE external_finding_imports; DROP TRIGGER scans_assign_inserted_completion_sequence; DROP TRIGGER scans_assign_updated_completion_sequence; DROP INDEX scans_completion_sequence; From d9d28d3038cf4c8c5b44cab3a974b071faf6d231 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 13:39:07 +0000 Subject: [PATCH 4/7] fix(imports): keep versioned SQLite selections alive through readback --- .../mcp-app/src/workbench/external-findings.ts | 4 ++-- sdk/typescript/tests-ts/workbench-identity-cache.test.ts | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts index ab23710e53..5d90bd8f75 100644 --- a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts +++ b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts @@ -309,8 +309,8 @@ export function listExternalFindings( LEFT JOIN (${findingWithProvenance}) f ON f.id = p.id AND f.repository = ? ORDER BY p.position`, ) - .iterate(input, repository); - const findings = Array.from(rows, (row) => { + .all(input, repository); + const findings = rows.map((row) => { if (row.version !== row.pinned_version) throw new Error( `Finding ${row.pinned_id} version ${row.pinned_version} is missing or has changed. Review the current selection.`, diff --git a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts index f37e2b9aa9..2ac0de8028 100644 --- a/sdk/typescript/tests-ts/workbench-identity-cache.test.ts +++ b/sdk/typescript/tests-ts/workbench-identity-cache.test.ts @@ -1549,6 +1549,7 @@ with ExitStack() as stack: "foldedLegacyScopesEqual": originals["scope-upper"] == originals["scope-lower"], "currentScopesDistinct": first["scope-upper"] != first["scope-lower"], "targetCount": len(first), + "expectedMigrations": [version for version, _, _ in MIGRATIONS], "migrations": [row["version"] for row in connection.execute("SELECT version FROM schema_migrations ORDER BY version")], })) `; @@ -2010,9 +2011,7 @@ test.each(["migration", "migration-recorded31"])( expect(result["foldedLegacyScopesEqual"]).toBe(true); expect(result["currentScopesDistinct"]).toBe(true); expect(result["targetCount"]).toBe(12); - expect(result["migrations"]).toEqual( - Array.from({ length: 71 }, (_, index) => index + 1), - ); + expect(result["migrations"]).toEqual(result["expectedMigrations"]); }, ); From b9540f4860b4b3199d7f56fdcd916797f1aed975 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 13:47:23 +0000 Subject: [PATCH 5/7] fix(workbench): retain version-scoped Cloud receipts across repository resets --- docs/scanner-imports/schema-guide.md | 4 +- .../src/external-findings-publish.ts | 125 +++++++++++++++--- .../src/stored-findings-publication.ts | 107 ++++++--------- .../external-findings-publish.test.ts | 90 +++++++++++++ 4 files changed, 244 insertions(+), 82 deletions(-) diff --git a/docs/scanner-imports/schema-guide.md b/docs/scanner-imports/schema-guide.md index 67dd6015e0..1298708dd5 100644 --- a/docs/scanner-imports/schema-guide.md +++ b/docs/scanner-imports/schema-guide.md @@ -144,7 +144,9 @@ the same options. For `prepareExternalPublication()`, supply the original export file. For `prepareNormalizedExternalPublication()`, supply the original `ExternalFindings`. Check `preview.resumed` before calling `publish()`. For stored findings, reprepare the interrupted source group with its saved selection -and review ID. Completed groups already have their own receipts. +and review ID. Its checkpoint belongs to the selected checkout and finding versions. +After a repository reset, the workbench records any acknowledged receipts before +retiring the old request. Review the new destination before publishing again. **Upgrading from an earlier import client.** Finish pending Cloud submissions with the CLI or SDK build that created their checkpoints. Keep the original input and diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 4b79f7237e..7a785c6662 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -256,12 +256,28 @@ export function cloudReviewDigest( export interface PreparedExternalPublication { preview: ExternalPublicationPreview; - /** Persist acknowledged repository receipts before retiring their resumable request. */ - publish( - beforeRetire?: (result: ExternalPublicationResult) => Promise, - ): Promise; + publish(): Promise; } +/** @internal Persist only the acknowledgements made by the selected Cloud destination. */ +type RecordPublicationReceipts = ( + scope: Pick< + ExternalPublicationPreview, + "accountId" | "cloudApiUrl" | "destination" | "requests" | "findings" + >, + receipts: FindingImportReceipt[], + privateJobId?: string, +) => Promise; + +/** @internal Keep a stored submission scoped to the finding versions that were approved. */ +export interface WorkbenchPublication { + checkout: string; + findings: readonly { id: string; version: number }[]; + record: RecordPublicationReceipts; +} + +type BeforeRetire = (result: ExternalPublicationResult) => Promise; + // Limit read concurrency to avoid one network round trip per finding in series. // Wait for the entire batch on failure so callers never outlive their read tasks. async function readInBatches( @@ -497,6 +513,16 @@ function validatePublicationReceipt( return receipt; } +function acknowledgedReceipts({ receipts = [], requests }: SavedSubmission) { + if (receipts.length > requests.length) + throw new CodexSecurityError( + "Saved publication has more receipts than requests.", + ); + return receipts.map((receipt, index) => + validatePublicationReceipt(receipt, requests[index]!), + ); +} + async function withImportLock( state: string, key: string, @@ -666,6 +692,7 @@ export async function prepareExternalPublication( { ...settings, provider }, dependencies, readerSources, + undefined, settings.sourceRepositoryUrl, fileRepositories, ); @@ -676,6 +703,16 @@ export async function prepareNormalizedExternalPublication( input: ExternalFindings, options: NormalizedExternalPublicationOptions, dependencies: ExternalPublicationDependencies = {}, +): Promise { + return prepareNormalizedPublication(input, options, dependencies); +} + +/** @internal The workbench persists acknowledgements before a repository checkpoint is retired. */ +export async function prepareNormalizedPublication( + input: ExternalFindings, + options: NormalizedExternalPublicationOptions, + dependencies: ExternalPublicationDependencies, + workbench?: WorkbenchPublication, ): Promise { const settings = snapshotImportOptions(options); requireCloudReader(settings); @@ -684,6 +721,7 @@ export async function prepareNormalizedExternalPublication( settings, dependencies, snapshotSourceRepositories(dependencies.sourceRepositories), + workbench, ); } @@ -975,9 +1013,57 @@ async function preparePublicationInput( options: NormalizedExternalPublicationOptions, dependencies: ExternalPublicationDependencies, readerSources: readonly ExternalSourceRepository[], + workbench?: WorkbenchPublication, declaredUrl?: string, fileRepositories?: () => readonly ExternalSourceRepository[], ): Promise { + const record = workbench?.record; + function withReceipts( + scope: ExternalPublicationPreview, + prepared: { + preview: ExternalPublicationPreview; + publish(beforeRetire?: BeforeRetire): Promise; + }, + ): PreparedExternalPublication { + return { + preview: prepared.preview, + async publish() { + if (!record) return prepared.publish(); + let recorded = false; + const persist = async (result: ExternalPublicationResult) => { + if (recorded) return; + recorded = true; + const job = result.privateJob; + const privateJobId = + job?.status === "completed" && + !result.counts.error && + !result.unacknowledged + ? job.id + : undefined; + if (!privateJobId && !result.receipts.length) return; + try { + await record(scope, result.receipts, privateJobId); + } catch (error) { + throw new ExternalPublicationError( + `Cloud returned receipts, but the workbench could not record them: ${errorMessage(error)}`, + result, + error, + ); + } + }; + let result: ExternalPublicationResult; + try { + result = await prepared.publish(persist); + } catch (error) { + if (error instanceof ExternalPublicationError) + await persist(error.result); + throw error; + } + await persist(result); + return result; + }, + }; + } const { inspectorProjectMap } = options; const progress = (event: ExternalPublicationProgress): void => notify(() => dependencies.onProgress?.(event)); @@ -1157,6 +1243,7 @@ async function preparePublicationInput( destination.repo_connector_id, source, publicationIdentity(parsed, options.provider), + ...(workbench ? [workbench.checkout, workbench.findings] : []), ]), ); const pendingPath = join(state, key + ".pending.json"); @@ -1277,6 +1364,18 @@ async function preparePublicationInput( const current = await readPending(); if (current && sameSubmission(current, saved)) { await confirmReset(); + if (record && current.receipts?.length) { + await record( + { + cloudApiUrl, + accountId: credentials.account_id, + destination, + requests: current.requests, + findings: [], + }, + acknowledgedReceipts(current), + ); + } await removePending(); } }); @@ -1624,7 +1723,7 @@ async function preparePublicationInput( requests: [], privateImport, }; - return { + return withReceipts(preview, { preview: structuredClone(preview), publish: () => underPublishLock(async () => { @@ -1741,7 +1840,7 @@ async function preparePublicationInput( throw new ExternalPublicationError(result.error, result, error); } }), - }; + }); } // The original selection and request bytes govern a retry even if discovery changes. @@ -1881,7 +1980,7 @@ async function preparePublicationInput( resumed: saved !== undefined, requests, }; - return { + return withReceipts(preview, { preview: cloneImportPreview(preview), async publish(beforeRetire) { progress({ phase: "waiting", completed: 0 }); @@ -1934,18 +2033,10 @@ async function preparePublicationInput( ) { // The expected file is gone or another publisher owns the lock. Use only // the acknowledgements this caller saw when it prepared the request. - const known = submission.receipts ?? []; - if (known.length > submission.requests.length) - throw new CodexSecurityError( - "Saved publication has more receipts than requests.", - ); - const acknowledged = known.map((receipt, index) => - validatePublicationReceipt(receipt, submission.requests[index]!), - ); return new ExternalPublicationError( message, { - ...result(acknowledged, 0), + ...result(acknowledgedReceipts(submission), 0), status: "interrupted", error: message, ...(savedSubmission === undefined ? {} : { savedSubmission }), @@ -2127,5 +2218,5 @@ async function preparePublicationInput( ); }); }, - }; + }); } diff --git a/sdk/typescript/src/stored-findings-publication.ts b/sdk/typescript/src/stored-findings-publication.ts index 69248ad5cf..76d6617aed 100644 --- a/sdk/typescript/src/stored-findings-publication.ts +++ b/sdk/typescript/src/stored-findings-publication.ts @@ -1,6 +1,6 @@ import { isDeepStrictEqual } from "node:util"; import cloudSchema from "../schemas/external-findings.schema.json" with { type: "json" }; -import { CodexSecurityError, errorMessage } from "./errors.js"; +import { CodexSecurityError } from "./errors.js"; import type { Provider } from "./external-import-models.js"; import { distinctSourceRepositories, @@ -13,8 +13,8 @@ import { } from "./external-findings-local.js"; import { cloudReviewDigest, - ExternalPublicationError, - prepareNormalizedExternalPublication, + prepareNormalizedPublication, + type WorkbenchPublication, type ExternalPublicationDependencies, type ExternalPublicationPreview, type ExternalPublicationResult, @@ -111,69 +111,57 @@ function cloudSelection( } function receiptRecorder( - preview: ExternalPublicationPreview, rows: StoredExternalFinding[], dependencies: Dependencies, -) { - // Keep only the identifiers needed for receipts. The caller can edit its public preview. +): WorkbenchPublication["record"] { + const repository = rows[0]!.repository; const pins = new Map( rows.map(({ id, version, sourceFindingId }) => [ sourceFindingId, { id, version }, ]), ); - const ready = preview.findings.map((f) => pins.get(f.source_finding_id)!); - const batches = preview.requests.map( - ({ items }) => - new Map( - items.map((item) => [ + return async ( + { accountId, cloudApiUrl, destination, requests, findings }, + receipts, + privateJobId, + ) => { + const entries: LocalFindingReceipt[] = []; + if (privateJobId) { + for (const finding of findings) + entries.push({ + ...pins.get(finding.source_finding_id)!, + receiptId: privateJobId, + outcome: "private_job", + }); + } + for (const [index, receipt] of receipts.entries()) { + const batch = new Map( + requests[index]!.items.map((item) => [ item.client_id, pins.get(item.source_finding_id)!, ]), - ), - ); - const destination = { - repository: rows[0]!.repository, - cloudApiUrl: preview.cloudApiUrl, - accountId: preview.accountId, - destinationId: preview.destination.id, - }; - return async (result: ExternalPublicationResult) => { - const entries: LocalFindingReceipt[] = []; - const job = result.privateJob; - // The private inbox provides job totals, with no per-item outcomes. - if ( - job?.status === "completed" && - !result.counts.error && - !result.unacknowledged - ) { - for (const pin of ready) - entries.push({ ...pin, receiptId: job.id, outcome: "private_job" }); - } - for (const [index, receipt] of result.receipts.entries()) { - for (const item of receipt.results) { + ); + for (const item of receipt.results) if (item.outcome !== "error") entries.push({ - ...batches[index]!.get(item.client_id)!, + ...batch.get(item.client_id)!, receiptId: receipt.id, outcome: item.outcome, cloudFindingId: item.canonical_finding_id ?? undefined, }); - } } - if (!entries.length) return; - try { + if (entries.length) await recordExternalFindingPublications( - { ...destination, entries }, + { + repository, + accountId, + cloudApiUrl, + destinationId: destination.id, + entries, + }, dependencies, ); - } catch (error) { - throw new ExternalPublicationError( - `Cloud returned receipts, but the workbench could not record them: ${errorMessage(error)}`, - result, - error, - ); - } }; } @@ -181,6 +169,7 @@ function prepareSource( rows: StoredExternalFinding[], options: { repository: string; provider: Provider; sourceKey: string }, dependencies: Dependencies, + workbench: WorkbenchPublication, metadata?: Metadata, ) { const references = [...(dependencies.sourceRepositories ?? [])]; @@ -196,7 +185,7 @@ function prepareSource( } } const sourceRepositories = distinctSourceRepositories(references); - return prepareNormalizedExternalPublication( + return prepareNormalizedPublication( cloudSelection( rows, sourceRepositories, @@ -205,6 +194,7 @@ function prepareSource( ), { ...options, ...(inspectorProjectMap.size && { inspectorProjectMap }) }, { ...dependencies, sourceRepositories }, + workbench, ); } @@ -228,12 +218,18 @@ export async function prepareStoredFindingsPublication( for (const row of rows) excluded.push({ ...reference(row), reason }); continue; } + const findings = rows.map(({ id, version }) => ({ id, version })); let prepared: PreparedExternalPublication; try { prepared = await prepareSource( rows, { ...options, provider, sourceKey }, dependencies, + { + checkout: repository, + findings, + record: receiptRecorder(rows, dependencies), + }, importId === undefined ? undefined : stored, ); } catch (error) { @@ -256,9 +252,7 @@ export async function prepareStoredFindingsPublication( throw new CodexSecurityError( "The Cloud account or repository changed during publication preparation.", ); - const findings = rows.map(({ id, version }) => ({ id, version })); const reviewId = cloudReviewDigest(preview, repository, findings); - const record = receiptRecorder(preview, rows, dependencies); groups.push({ selection: importId === undefined @@ -271,22 +265,7 @@ export async function prepareStoredFindingsPublication( throw new CodexSecurityError( "The stored findings publication has changed. Review the new preview before publishing.", ); - let recordingStarted = false; - async function recordOnce(result: ExternalPublicationResult) { - if (recordingStarted) return; - recordingStarted = true; - await record(result); - } - let result: ExternalPublicationResult; - try { - result = await prepared.publish(recordOnce); - } catch (error) { - if (error instanceof ExternalPublicationError) - await recordOnce(error.result); - throw error; - } - await recordOnce(result); - return result; + return prepared.publish(); }, }); } diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index ca5ca0155d..8fa99ded1f 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -934,6 +934,96 @@ test.each(["complete", "conflict"])( }, ); +test("a reset retains the saved request until SQLite records its accepted receipt", async () => { + const cloud = await fixture(); + await using local = await storedPublicationFixture(cloud); + const imported = await local.store("wiz", { + read: 1, + findings: [localFinding("scope")], + excluded: [], + }); + const selection = { + repository: local.repository, + importId: imported.importId!, + }; + const [group] = (await local.prepare(selection)).groups; + using database = new Database( + join(local.stateDirectory, "workbench.sqlite3"), + ); + database.exec( + "CREATE TRIGGER reject_links BEFORE INSERT ON external_finding_publications BEGIN SELECT RAISE(ABORT,'cannot record'); END", + ); + await expect(group!.publish(group!.reviewId)).rejects.toThrow( + "cannot record", + ); + expect(cloud.posts).toHaveLength(1); + const [receiptId] = cloud.receipts.keys(); + cloud.state.marker = "generation-2"; + cloud.reports.clear(); + await expect(local.prepare(selection)).rejects.toThrow("cannot record"); + database.exec("DROP TRIGGER reject_links"); + await expect(local.prepare(selection)).rejects.toThrow("reset"); + const links = await listLocalFindingPublications( + { repository: local.repository }, + local.deps, + ); + expect(links).toHaveLength(1); + expect(links[0]?.receiptId).toBe(receiptId); + const [fresh] = (await local.prepare(selection)).groups; + await expect(fresh!.publish(group!.reviewId)).rejects.toThrow("changed"); + expect(cloud.posts).toHaveLength(1); +}); + +test("identical evidence at a later local version cannot inherit an earlier receipt", async () => { + const cloud = await fixture(); + await using local = await storedPublicationFixture(cloud); + const row = localFinding("pinned"); + const store = (finding: typeof row) => + local.store("wiz", { read: 1, findings: [finding], excluded: [] }); + const first = await store(row); + const selection = (importId: string) => ({ + repository: local.repository, + importId, + }); + const [original] = (await local.prepare(selection(first.importId!))).groups; + using database = new Database( + join(local.stateDirectory, "workbench.sqlite3"), + ); + database.exec( + "CREATE TRIGGER reject_links BEFORE INSERT ON external_finding_publications BEGIN SELECT RAISE(ABORT,'cannot record'); END", + ); + await expect(original!.publish(original!.reviewId)).rejects.toThrow( + "cannot record", + ); + database.exec("DROP TRIGGER reject_links"); + + const direct = await prepareNormalizedExternalPublication( + { read: 1, findings: [row], excluded: [] }, + { repository: options.repository, provider: "wiz", sourceKey: "selected" }, + local.deps, + ); + expect(direct.preview.resumed).toBe(false); + + await store({ + ...row, + evidence: { ...row.evidence, title: "Changed title" }, + }); + const restored = await store(row); + expect(restored.findings[0]!.version).toBe(3); + cloud.state.marker = "generation-2"; + cloud.reports.clear(); + const [current] = (await local.prepare(selection(restored.importId!))).groups; + expect(current!.preview.resumed).toBe(false); + await expect(current!.publish(original!.reviewId)).rejects.toThrow("changed"); + expect( + await listLocalFindingPublications( + { repository: local.repository }, + local.deps, + ), + ).toEqual([]); + expect(cloud.posts).toHaveLength(1); +}); + test("stored findings recover a private inbox upload and retain its job receipt", async () => { const { f, inbox, deps, uploads, jobs } = await privateUploadFixture( [], From 75f5f69586e1214f6875f81e0b4fa8bce9b4346d Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 18:20:27 +0000 Subject: [PATCH 6/7] Resume Sonar Cloud retries from the stored import checkpoint --- .../src/workbench/external-findings.ts | 16 +++-- sdk/typescript/src/external-findings-local.ts | 9 ++- .../src/external-findings-publish.ts | 12 ++-- sdk/typescript/src/sonar-import.ts | 63 +++++++++++++------ .../external-findings-publish.test.ts | 46 ++++++++++++++ 5 files changed, 117 insertions(+), 29 deletions(-) diff --git a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts index 5d90bd8f75..7909ad1df2 100644 --- a/plugins/codex-security/mcp-app/src/workbench/external-findings.ts +++ b/plugins/codex-security/mcp-app/src/workbench/external-findings.ts @@ -35,6 +35,7 @@ export interface ExternalFindingSelection { repository: string; id?: string; importId?: string; + ifCurrent?: boolean; findings?: Pick[]; } @@ -275,8 +276,10 @@ export function listExternalFindings( "SELECT selection_json FROM external_finding_imports WHERE repository = ? AND id = ?", ) .get(repository, importId); - if (!batch) + if (!batch) { + if (selection.ifCurrent) return { read: 0, excluded: [], findings: [] }; throw new Error("Import does not exist in the selected repository."); + } summary = parseJson(String(batch.selection_json)) as Record< string, unknown @@ -310,11 +313,14 @@ export function listExternalFindings( ORDER BY p.position`, ) .all(input, repository); + const stale = rows.find((row) => row.version !== row.pinned_version); + if (stale) { + if (selection.ifCurrent) return { read: 0, excluded: [], findings: [] }; + throw new Error( + `Finding ${stale.pinned_id} version ${stale.pinned_version} is missing or has changed. Review the current selection.`, + ); + } const findings = rows.map((row) => { - if (row.version !== row.pinned_version) - throw new Error( - `Finding ${row.pinned_id} version ${row.pinned_version} is missing or has changed. Review the current selection.`, - ); return { ...fromRow(row), ...(row.provenance_json != null && { diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts index 50ca5e0f0d..d2f37e9bf4 100644 --- a/sdk/typescript/src/external-findings-local.ts +++ b/sdk/typescript/src/external-findings-local.ts @@ -130,7 +130,12 @@ export interface ExternalFindingProvenance { } export type LocalExternalFindingSelection = { repository: string } & ( - | { importId: string; findings?: never } + | { + importId: string; + findings?: never; + /** Return an empty selection if the import is missing or any pinned version has changed. */ + ifCurrent?: boolean; + } | { findings: readonly Pick[]; importId?: never; @@ -1746,7 +1751,7 @@ export async function selectLocalExternalFindings( repository, "read-external-findings", importId !== undefined - ? { importId } + ? { importId, ifCurrent: selection.ifCurrent } : { findings: findings!.map(({ id, version }) => ({ id, version })) }, dependencies, ); diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 7a785c6662..3ca6666dde 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -712,7 +712,7 @@ export async function prepareNormalizedPublication( input: ExternalFindings, options: NormalizedExternalPublicationOptions, dependencies: ExternalPublicationDependencies, - workbench?: WorkbenchPublication, + workbench?: WorkbenchPublication | "preview", ): Promise { const settings = snapshotImportOptions(options); requireCloudReader(settings); @@ -1013,11 +1013,11 @@ async function preparePublicationInput( options: NormalizedExternalPublicationOptions, dependencies: ExternalPublicationDependencies, readerSources: readonly ExternalSourceRepository[], - workbench?: WorkbenchPublication, + workbench?: WorkbenchPublication | "preview", declaredUrl?: string, fileRepositories?: () => readonly ExternalSourceRepository[], ): Promise { - const record = workbench?.record; + const record = workbench !== "preview" && workbench?.record; function withReceipts( scope: ExternalPublicationPreview, prepared: { @@ -1243,7 +1243,11 @@ async function preparePublicationInput( destination.repo_connector_id, source, publicationIdentity(parsed, options.provider), - ...(workbench ? [workbench.checkout, workbench.findings] : []), + ...(workbench === "preview" + ? ["preview"] + : workbench + ? [workbench.checkout, workbench.findings] + : []), ]), ); const pendingPath = join(state, key + ".pending.json"); diff --git a/sdk/typescript/src/sonar-import.ts b/sdk/typescript/src/sonar-import.ts index 0a4d66ce9a..bfe53a6011 100644 --- a/sdk/typescript/src/sonar-import.ts +++ b/sdk/typescript/src/sonar-import.ts @@ -1,5 +1,12 @@ import { hash } from "node:crypto"; -import { lstat, open, readlink, realpath, stat } from "node:fs/promises"; +import { + lstat, + open, + readFile, + readlink, + realpath, + stat, +} from "node:fs/promises"; import { basename, dirname, join } from "node:path"; import { resolveConfigPath } from "./config-path.js"; import { collectSonarIssues } from "./sonar-collector.js"; @@ -15,7 +22,7 @@ import { cloudReviewDigest, ExternalPublicationError, externalPublicationStateDirectory, - prepareNormalizedExternalPublication, + prepareNormalizedPublication, writeAtomicJson, type ExternalPublicationDependencies, type ExternalPublicationPreview, @@ -31,7 +38,11 @@ import { requireOutputOutsideRepositories, workbenchEnvironment, } from "./runtime.js"; -import { gitMarkerRoot, normalizeRepository } from "./targets.js"; +import { + gitMarkerRoot, + normalizeRepository, + nullIfMissingFile, +} from "./targets.js"; import { errorMessage } from "./errors.js"; import { canonicalJson } from "./record.js"; import { @@ -286,6 +297,17 @@ export async function prepareSonarImport( [], fetchRemotes, ); + // Record the selection before submitting so a lost response can be replayed + // with the workbench checkpoint for these exact finding versions. + const selectionPath = `${localCapture}.import`; + const storedPublication = async (importId: string, ifCurrent = false) => + ( + await prepareStoredFindingsPublication( + { repository, importId, ifCurrent }, + { repository: repositoryUrl }, + dependencies, + ) + ).groups[0]; let cloud: ExternalPublicationPreview | undefined; let cloudError: string | undefined; const reviewedCapturePath = (preview: ExternalPublicationPreview) => @@ -314,15 +336,25 @@ export async function prepareSonarImport( "This checkout has no matching Git remote. Verify the Sonar project's repository and pass --source-repository-url to preview Cloud publication."; if (!confirmedRepository && !reviewedCheckout) throw new Error(bindingError); - const prepared = await prepareNormalizedExternalPublication( - normalizeSonarFindings( - report, - options.sourceRepositoryUrl ?? "", - "cloud", - ), - { repository: repositoryUrl, ...source }, - dependencies, + const savedId = await readFile(selectionPath, "utf8").catch( + nullIfMissingFile, ); + const previous = + savedId === null + ? undefined + : await storedPublication(JSON.parse(savedId), true); + const prepared = + previous ?? + (await prepareNormalizedPublication( + normalizeSonarFindings( + report, + options.sourceRepositoryUrl ?? "", + "cloud", + ), + { repository: repositoryUrl, ...source }, + dependencies, + "preview", + )); const reviewedCapture = reviewedCapturePath(prepared.preview); if (input !== reviewedCapture) { if (!confirmedRepository) throw new Error(bindingError); @@ -359,13 +391,7 @@ export async function prepareSonarImport( if (!cloud) return { ...saved, cloudError, cloud: undefined }; try { if (!importId) throw new Error("No stored Sonar import to publish."); - const { - groups: [publication], - } = await prepareStoredFindingsPublication( - { repository, importId }, - { repository: repositoryUrl }, - dependencies, - ); + const publication = await storedPublication(importId); if ( !publication || reviewedCapturePath(publication.preview) !== capture @@ -373,6 +399,7 @@ export async function prepareSonarImport( throw new Error( "The stored findings or Cloud destination have changed. Preview the Sonar capture again with --to cloud --dry-run.", ); + await writeAtomicJson(selectionPath, importId); return { ...saved, cloud: await publication.publish(publication.reviewId), diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 8fa99ded1f..81b3215245 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2920,11 +2920,34 @@ test("Sonar replays the selected evidence locally and to Cloud without signing i ).toBe(2); expect(JSON.parse(unavailable.stdout.text()).local.counts.created).toBe(1); + f.state.loseResponse = true; + const interrupted = createCliTest(main); + expect(await interrupted.runCli([...args, "--yes"], deps)).toBe(2); + expect(JSON.parse(interrupted.stdout.text()).cloud.status).toBe( + "interrupted", + ); + expect(f.posts).toHaveLength(1); + const retry = createCliTest(main); + expect(await retry.runCli([...previewArgs, "--dry-run"], deps)).toBe(0); + expect(JSON.parse(retry.stdout.text()).capture).toBe(reviewed.capture); + expect(f.posts).toHaveLength(1); + const replay = createCliTest(main); expect(await replay.runCli([...args, "--yes"], deps)).toBe(0); const result = JSON.parse(replay.stdout.text()); expect(result.local.counts.unchanged).toBe(1); expect(result.cloud.verified).toBe(1); + expect(f.posts).toEqual([f.posts[0], f.posts[0]]); + expect(f.reports.size).toBe(1); + const receipts = await listLocalFindingPublications( + { repository: checkout }, + { environment }, + ); + expect(receipts).toHaveLength(1); + expect(receipts[0]).toMatchObject({ + version: 1, + cloudFindingId: f.reports.get("issue-1")!.canonical_finding_id, + }); const posted = JSON.parse(f.posts[0]!); expect(posted.source).toEqual({ provider: "sonar", @@ -2944,6 +2967,29 @@ test("Sonar replays the selected evidence locally and to Cloud without signing i expect(posted.items[0].evidence.source_data.issue.flows).toEqual( source.pages[0]!.issues[0]!.flows, ); + + // An intervening local edit supersedes the saved pins. A fresh review of + // the original evidence must still allow it to be imported as a new version. + const changed = structuredClone(source); + changed.pages[0]!.issues[0]!.message += " (changed)"; + const changedPath = join(f.root, "changed.json"); + await writeFile(changedPath, JSON.stringify(changed)); + async function importSonar(path: string, ...flags: string[]) { + const cli = createCliTest(main); + const command = ["import", "sonar", path, ...flags, "--format", "json"]; + expect(await cli.runCli(command, deps), cli.stderr.text()).toBe(0); + return JSON.parse(cli.stdout.text()); + } + const changedPreview = await importSonar(changedPath, "--dry-run"); + await importSonar(changedPreview.capture, "--yes"); + const fresh = await importSonar(capture, "--to", "cloud", "--dry-run"); + expect(fresh.capture).not.toBe(reviewed.capture); + await importSonar(fresh.capture, "--to", "cloud", "--yes"); + const history = await listLocalFindingPublications( + { repository: checkout }, + { environment }, + ); + expect(history.map((row) => row.version).sort()).toEqual([1, 3]); }); test("normalized Sonar publication checks retained bindings on preview and retry", async () => { From 056da6b6cc3ea24b8f7cef0709ee26f1a0a501a5 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 18:24:30 +0000 Subject: [PATCH 7/7] Keep the Sonar replay assertion strictly typed --- sdk/typescript/tests-ts/external-findings-publish.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 81b3215245..dab1302826 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -2937,7 +2937,8 @@ test("Sonar replays the selected evidence locally and to Cloud without signing i const result = JSON.parse(replay.stdout.text()); expect(result.local.counts.unchanged).toBe(1); expect(result.cloud.verified).toBe(1); - expect(f.posts).toEqual([f.posts[0], f.posts[0]]); + expect(f.posts).toHaveLength(2); + expect(f.posts[1]).toBe(f.posts[0]); expect(f.reports.size).toBe(1); const receipts = await listLocalFindingPublications( { repository: checkout },