From a8b99204acbbdc3c667bf84dceadd9b79c6a4b32 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 08:11:43 +0000 Subject: [PATCH 1/3] refactor(imports)!: retire historical reader formats and Cloud checkpoints --- docs/scanner-imports/schema-guide.md | 28 +- docs/scanner-imports/semgrep.md | 26 +- docs/scanner-imports/snyk.md | 23 +- sdk/typescript/docs/cli.md | 13 +- sdk/typescript/docs/sdk.md | 2 +- sdk/typescript/scripts/package-dist-files.mjs | 4 - sdk/typescript/src/blackduck-findings.ts | 53 +- sdk/typescript/src/continuum-findings.ts | 91 +- sdk/typescript/src/endor-findings.ts | 36 +- sdk/typescript/src/external-findings-local.ts | 25 +- .../src/external-findings-publish.ts | 2315 +---- sdk/typescript/src/external-findings-read.ts | 115 +- sdk/typescript/src/external-findings.ts | 22 +- .../src/external-import-contract.ts | 220 +- .../src/github-dependabot-findings.ts | 7 +- sdk/typescript/src/github-findings.ts | 357 +- sdk/typescript/src/hosted-scanner-v1.ts | 68 - sdk/typescript/src/inspector-findings.ts | 102 +- sdk/typescript/src/opengrep-findings.ts | 18 +- sdk/typescript/src/private-finding-import.ts | 243 +- sdk/typescript/src/reader-selection.ts | 131 - sdk/typescript/src/record.ts | 67 +- sdk/typescript/src/scanner-import.ts | 138 +- sdk/typescript/src/semgrep-findings.ts | 78 +- .../src/semgrep-platform-findings.ts | 627 +- sdk/typescript/src/semgrep-platform-v1.ts | 132 - sdk/typescript/src/snyk-findings.ts | 22 +- sdk/typescript/src/snyk-rest-findings.ts | 312 +- sdk/typescript/src/snyk-rest-v1.ts | 149 - sdk/typescript/src/trivy-findings.ts | 25 - sdk/typescript/src/veracode-findings.ts | 64 +- sdk/typescript/src/wiz-findings.ts | 139 +- .../tests-ts/blackduck-findings.test.ts | 11 - .../tests-ts/continuum-findings.test.ts | 27 +- .../tests-ts/endor-findings.test.ts | 15 - .../external-findings-publish.test.ts | 7816 ++++------------- .../fixtures/boxed-number-checkpoints.json | 104 - .../fixtures/boxed-sonar-checkpoint.json | 81 - .../fixtures/snyk-released-checkpoints.json | 131 - .../fixtures/wiz-preview-checkpoint.json | 82 - .../tests-ts/github-findings.test.ts | 400 +- .../tests-ts/inspector-findings.test.ts | 79 +- .../tests-ts/opengrep-findings.test.ts | 75 - .../tests-ts/semgrep-findings-local.test.ts | 2 +- .../semgrep-platform-collector.test.ts | 335 +- .../semgrep-platform-normalization.test.ts | 303 + .../semgrep-platform-selection-file.test.ts | 30 +- .../semgrep-platform-selection.test.ts | 7 + .../semgrep-platform-v1-compat.test.ts | 1294 --- .../tests-ts/snyk-rest-boundaries.test.ts | 383 +- .../tests-ts/snyk-rest-findings.test.ts | 37 +- sdk/typescript/tests-ts/snyk-rest-v1.test.ts | 175 - .../stored-native-private-report.test.ts | 97 + .../tests-ts/trivy-findings.test.ts | 11 +- .../tests-ts/veracode-findings.test.ts | 17 - sdk/typescript/tests-ts/wiz-findings.test.ts | 666 +- 56 files changed, 3135 insertions(+), 14695 deletions(-) delete mode 100644 sdk/typescript/src/hosted-scanner-v1.ts delete mode 100644 sdk/typescript/src/reader-selection.ts delete mode 100644 sdk/typescript/src/semgrep-platform-v1.ts delete mode 100644 sdk/typescript/src/snyk-rest-v1.ts delete mode 100644 sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json delete mode 100644 sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json delete mode 100644 sdk/typescript/tests-ts/fixtures/snyk-released-checkpoints.json delete mode 100644 sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json create mode 100644 sdk/typescript/tests-ts/semgrep-platform-normalization.test.ts delete mode 100644 sdk/typescript/tests-ts/semgrep-platform-v1-compat.test.ts delete mode 100644 sdk/typescript/tests-ts/snyk-rest-v1.test.ts create mode 100644 sdk/typescript/tests-ts/stored-native-private-report.test.ts diff --git a/docs/scanner-imports/schema-guide.md b/docs/scanner-imports/schema-guide.md index ce35d7680b..7767d62ac1 100644 --- a/docs/scanner-imports/schema-guide.md +++ b/docs/scanner-imports/schema-guide.md @@ -143,26 +143,14 @@ With the SDK, repeat the preparation call that created the submission, using the same options. For `prepareExternalPublication()`, supply the original export file. For `prepareNormalizedExternalPublication()`, supply the original `ExternalFindings`. Check `preview.resumed` before calling `publish()`. -Use the same SDK version throughout a submission when possible. - -For hosted Semgrep or Snyk imports, finish any pending request before sending -updated evidence for the same finding IDs. Selections with distinct -scanner-issued IDs can proceed independently. If the checkpoint records -only the source, finish that request before starting another for the same -source and destination. - -**Resuming across SDK versions.** Keep the SDK input that produced the -checkpoint, including report context and caller-added fields. Serializing it -as JSON can change the values used to identify the approved request. Retry on -the original deployment. If the SDK can verify the saved selection, it reuses -the request IDs and skips batches with receipts. Otherwise it preserves the -checkpoint and stops before uploading. If it cannot identify the approved -findings, a new submission for the same source and destination can also be blocked. - -After a repository rename, a checkpoint that records only the repository ID -can still be retried with the original normalized findings and report metadata. -Omit supplemental repository declarations for that retry. A new declaration -requires a matching catalog entry or a URL recorded in the approval. + +**Upgrading from an earlier import client.** Finish pending Cloud submissions with +the CLI or SDK build that created their checkpoints. Keep the original input and +state directory. If the outcome is uncertain, check the private inbox job or +read back the repository’s source reports to identify the accepted findings +before preparing a fresh submission. The current client starts a new approval +for a submission made with the retired checkpoint format. +[Verify an import](README.md#verify-the-import) See [results and exit codes](../../sdk/typescript/docs/cli.md#results-and-exit-codes) for cumulative counts and interrupted readback. diff --git a/docs/scanner-imports/semgrep.md b/docs/scanner-imports/semgrep.md index 405b9cd915..939ff4bbea 100644 --- a/docs/scanner-imports/semgrep.md +++ b/docs/scanner-imports/semgrep.md @@ -446,7 +446,7 @@ when the project has no repository URL. The report records that mapping in For SSH projects, the report stores the original address in `nativeProjectUrl` and the resolved HTTP(S) address in `project.url`. It records any supplied mapping separately. Save the complete output from a successful command. The importer checks the recorded finding count and repository/ref -claims. It retains each original finding as JSON text so large IDs survive replay. The reader also accepts saved v1 project exports with native object rows and decimal type witnesses. +claims. It retains each original finding as JSON text so large IDs survive replay. A finding that identifies its repository only by name can use the project name, the Git URL path, or the final path component of either. For any other name, @@ -483,18 +483,14 @@ and advisory IDs when Semgrep supplies them. With the same source key, importing selection reports them as unchanged when switching between a selection and a full collection. See the [Semgrep Findings API](https://docs.semgrep.dev/api-reference/v1/findingsservice/list-code-supply-chain-or-ai-powered-detection-findings). -When replaying v1 exports whose findings were stored as objects, the first -replay converts the stored finding to JSON text and increments its version. -Its local and source IDs stay the same. +For an older `semgrep-platform-findings/v1` export, rerun `cs import semgrep` +with the selected deployment, project, product and ref to collect a current export. +Check its IDs and scope before saving or publishing it. ## Replay saved Platform findings Semgrep's native signed 64-bit finding IDs match across file and SDK imports. -For synthetic IDs above that range, repeat the original workflow and preserve -the ID's spelling and JSON type. A saved selection keeps its literal ref, -including older short refs. For a row without a URL but with a matching -project ID and ref, saved files also accept the complete percent-encoded -repository path. New selections require the decoded path. +Keep the original spelling and JSON type of each selected ID and the full Git ref in the saved capture. **Repository mapping.** Local preview compares the project URL with the checkout's named fetch URLs. For an IP address, use its canonical spelling, such as @@ -513,14 +509,8 @@ evidence unchanged. Use a separate source key for CLI scans. **Normalized SDK records.** Keep each generated `source_finding_id` with its native row and original deployment, project, ref and product. If you assign your own opaque ID, retain that scope in a marked Platform batch. The SDK checks -the serialized native row when present. In legacy v1 `source_data.finding`, -repository fields, `ref` (even `null`), recognized `product` or `issue_type`, -and non-null `found_dependency` or `vulnerability_identifier` can establish -scope. Put unrelated metadata under a different key, such as -`source_data.ticket`, or submit it in a batch without a Platform marker. +the serialized native row. Keep unrelated metadata in a separate field. **Cloud retries.** Cloud checks the original project's repository, including -for Supply Chain findings without a typed repository field. New approvals -retain the collection scope of caller-managed rows. Older approvals check -the repository and the scope retained in their evidence. Retry the original -input with the same account and destination. [Retry a Cloud submission](schema-guide.md#retry-a-cloud-submission). +for Supply Chain findings without a typed repository field. Retry the original +selection with the same account and destination. [Retry a Cloud submission](schema-guide.md#retry-a-cloud-submission). diff --git a/docs/scanner-imports/snyk.md b/docs/scanner-imports/snyk.md index 85df7f986d..70092ec1de 100644 --- a/docs/scanner-imports/snyk.md +++ b/docs/scanner-imports/snyk.md @@ -152,17 +152,13 @@ The SDK's `collectSnykIssues` defaults to Open Source and returns the same repor Passing the configured `SNYK_TOKEN` explicitly retains its configured regional API origin. Pass independently verified `sourceRepositories` in its dependencies when a private repository uses an SSH target. The offline reader -`readSnykFindings(path, "local")` accepts `snyk-rest-issues/v1` and -`snyk-rest-issues/v2` reports from Snyk REST API version `2026-03-25`. -The collector writes v2 and preserves the original project and issue evidence. -For open Snyk Code issues from an SCM integration, both versions import -repository-relative paths from open coordinates. V2 also imports each -coordinate's reported start line when available. V2 keeps URI and SSH addresses -in the original evidence and uses only repository file paths for code locations -and package manifests. For resolved issues, closed -coordinates, and projects monitored through the CLI or API, read the original -location from `evidence.source_data.issue` in v1 or from the JSON text in -`evidence.source_data.issueJson` in v2. +`readSnykFindings(path, "local")` accepts the collector's `snyk-rest-issues/v2` +report for Snyk REST API version `2026-03-25`. Run the collector again to replace +an archived v1 export. The report preserves the original project and issue +evidence. Open Snyk Code issues from an SCM integration import repository file +paths and reported start lines. For resolved issues, closed coordinates, and +CLI- or API-monitored projects, read the original location from +`evidence.source_data.issueJson`. Use the same source key on later imports. The issue key within the API origin, organization, project and product identifies the finding. Replaying the @@ -179,11 +175,6 @@ the monitored project for a row that retains only its original issue. The Cloud preview includes the verified project and any captured target in the evidence it will publish. -For a v1 row, keep `source_data.issue` and the original `source_data.numericFields` -when present. If you trim the metadata, keep its matching entries for every -`issue`, `project`, and `target` object you retain. V1 uses those entries to -distinguish exact numbers from strings with the same spelling. For a v2 row, -keep `source_data.issueJson`. The local workbench stores the verified project and target with that issue, so the stored row can be imported again. All selected issues must share the report’s API region, organization, project and product. Keep SCM-integrated diff --git a/sdk/typescript/docs/cli.md b/sdk/typescript/docs/cli.md index 4d87799276..04b3e7464c 100644 --- a/sdk/typescript/docs/cli.md +++ b/sdk/typescript/docs/cli.md @@ -1630,11 +1630,10 @@ Run the command with `--dry-run --format json`. Check `accountId`, - **Private findings inbox.** If the preview includes `privateImport`, the report goes to your inbox. The repository and connector identify the suggested assessment target. Check `format`, `bytes` and `sha256` in - `privateImport` to identify the upload. For an Endor Finding, an Endor - `list.objects` response or a Wiz vulnerability JSON export, the CLI can - send the original bytes when Cloud accepts the format and every record - matches your selection. Other inputs, including arrays of selected Endor - Findings, use `security.findings.v1`. + `privateImport` to identify the upload. The CLI and SDK encode the reviewed + occurrences as `security.findings.v1` when it fits. For Wiz and Endor, an + inbox that requires native records or has a smaller upload limit can receive + native records that reproduce the reviewed evidence. - **Repository publisher.** Check `repository.environment_id` in each entry of `requests`. The publisher checks the current source versions. Existing records keep their environment. New records use the repository's authorized @@ -1663,10 +1662,10 @@ instructions. API requests always use the API root. The older ### Resume interrupted imports If the result includes `savedSubmission`, keep that checkpoint. Rerun the -original command with `--dry-run --format json` using the same input, account, +original command with `--dry-run --format json` using the same build, input, account, destination, API endpoint and state directory. Check `cloudApiUrl`. If `resumed` is true, confirm to continue the saved submission. If it's false, -review the new submission before confirming. +review the new submission before confirming. Finish older-format pending submissions with the client that created them. If a response was lost, check the inbox job or repository records before starting a fresh upload. - **Private inbox.** The CLI checks the report digest and reuses its saved request IDs. It can read an accepted job even if the deployment has since diff --git a/sdk/typescript/docs/sdk.md b/sdk/typescript/docs/sdk.md index 157da213ff..c30e8b3d74 100644 --- a/sdk/typescript/docs/sdk.md +++ b/sdk/typescript/docs/sdk.md @@ -677,7 +677,7 @@ For normalized evidence, use `prepareNormalizedLocalImport(findings, options)` or `prepareNormalizedExternalPublication(findings, options)` and review the preview in the same way. Cloud file imports use `ExternalPublicationOptions`, and normalized Cloud imports use `NormalizedExternalPublicationOptions`. Keep -Cloud receipts and saved requests when retrying an interrupted submission. +Cloud receipts and saved requests when retrying an interrupted submission. Use the SDK build that created a pending submission to finish it before upgrading. Use `listLocalExternalFindings({ repository })` to read saved scanner evidence. It starts unassessed. Reimporting the same repository, provider, source key diff --git a/sdk/typescript/scripts/package-dist-files.mjs b/sdk/typescript/scripts/package-dist-files.mjs index 8b1a02e3ec..f537f98b56 100644 --- a/sdk/typescript/scripts/package-dist-files.mjs +++ b/sdk/typescript/scripts/package-dist-files.mjs @@ -31,7 +31,6 @@ const distModules = [ "external-findings-local", "external-findings-import", "external-findings-read", - "reader-selection", "scanner-import", "inspector-findings", "continuum-findings", @@ -42,7 +41,6 @@ const distModules = [ "checkmarx-xml", "snyk-findings", "snyk-rest-findings", - "snyk-rest-v1", "snyk-token-origin", "scanner-repository-url", "semgrep-findings", @@ -52,8 +50,6 @@ const distModules = [ "hosted-git-repository", "semgrep-platform-collector", "hosted-scanner-http", - "hosted-scanner-v1", - "semgrep-platform-v1", "sonar-findings", "sonar-collector", "sonar-import", diff --git a/sdk/typescript/src/blackduck-findings.ts b/sdk/typescript/src/blackduck-findings.ts index d4b8f5ef76..e5123cd472 100644 --- a/sdk/typescript/src/blackduck-findings.ts +++ b/sdk/typescript/src/blackduck-findings.ts @@ -9,7 +9,7 @@ import { canonicalJson, isRecord } from "./record.js"; import { array, normalizeRecords, - readScannerJson, + scannerJsonReader, record, reportableScannerSeverity, requiredText, @@ -357,7 +357,7 @@ function blackDuckPackages(row: Record) { ? decodeURIComponent(purl[3]) : text(row["componentVersionName"]), }; - // Earlier versions used the origin fallback for owner/repository PURLs. + // Retain the origin coordinate if the qualified owner/repository PURL exceeds a Cloud limit. return ecosystem === "github" && name.includes("/") ? { packages: fallback, enrichment: { packages: [pkg] } } : { packages: [pkg] }; @@ -370,11 +370,10 @@ function blackDuckPackages(row: Record) { return { packages: fallback }; } -function normalizeReport( +export function normalizeBlackDuckFindings( input: unknown, sourceRepositoryUrl: string, - target: ExternalImportTarget, - previous: boolean, + target: ExternalImportTarget = "cloud", ): ExternalFindings { const repository = declaredScannerRepositoryUrl( sourceRepositoryUrl, @@ -435,7 +434,7 @@ function normalizeReport( }, }, target, - previous ? undefined : enrichment, + enrichment, ); }, (entry) => occurrenceId(entry, projectVersion), @@ -459,42 +458,6 @@ function normalizeReport( return result; } -export function normalizeBlackDuckFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeReport(input, sourceRepositoryUrl, target, false); -} - -/** @internal Earlier readers used the native origin fallback for GitHub package URLs. */ -export function needsPreviousBlackDuckProjection({ - findings, -}: ExternalFindings) { - return findings.some(({ evidence }) => - evidence.packages?.some( - (pkg) => pkg.ecosystem === "github" && pkg.name.includes("/"), - ), - ); -} - -/** @internal Reconstruct the original evidence for an interrupted Cloud request. */ -export function normalizePreviousBlackDuckFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeReport(input, sourceRepositoryUrl, target, true); -} - -export async function readBlackDuckFindings( - path: string, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): Promise { - return normalizeBlackDuckFindings( - await readScannerJson(path), - sourceRepositoryUrl, - target, - ); -} +export const readBlackDuckFindings = scannerJsonReader( + normalizeBlackDuckFindings, +); diff --git a/sdk/typescript/src/continuum-findings.ts b/sdk/typescript/src/continuum-findings.ts index 0bc6163b24..1d5ec83306 100644 --- a/sdk/typescript/src/continuum-findings.ts +++ b/sdk/typescript/src/continuum-findings.ts @@ -12,7 +12,7 @@ import { awsTimestampSeconds, concreteVersion, normalizeRecords, - readScannerJson, + scannerJsonReader, record, repositoryFilePath, requiredText, @@ -149,8 +149,8 @@ function codeLocations( return { locations, code: { end_line: endLine, snippet } }; } -function reportRows(input: unknown, previous: boolean) { - if (Array.isArray(input) && !previous) return { rows: input, summary: false }; +function reportRows(input: unknown) { + if (Array.isArray(input)) return { rows: input, summary: false }; const document = record(input, "Continuum report"); if ( Object.hasOwn(document, "nextToken") || @@ -174,37 +174,6 @@ function subcategoryCwe(row: Record) { return /^(CWE-[1-9]\d*):/u.exec(text(row["subCategory"]) ?? "")?.[1]; } -/** @internal Match 0.9.x checkpoints for both raw imports and saved SDK output. */ -export function previousContinuumCweProjection( - current: ExternalFindings, -): ExternalFindings { - let changed = false; - const findings = current.findings.map((finding) => { - const { details, source_data } = finding.evidence; - const native = source_data?.["finding"]; - // Remove only the inferred field. Caller edits and all other evidence must - // still match the immutable checkpoint before it can be resumed. - if ( - source_data?.["format"] !== format || - !isRecord(native) || - details?.kind !== "sast" || - details.weakness_ids?.length !== 1 || - strings(native["cweIds"]).length || - details.weakness_ids[0] !== subcategoryCwe(native) - ) - return finding; - changed = true; - return { - ...finding, - evidence: { - ...finding.evidence, - details: { ...details, weakness_ids: [] }, - }, - }; - }); - return changed ? { ...current, findings } : current; -} - function statusUpdatedAt(value: unknown) { const seconds = Math.floor(awsTimestampSeconds(value)); return Number.isSafeInteger(seconds) && seconds >= 0 ? seconds : null; @@ -253,18 +222,17 @@ function affectedPackages( return [...unique.values()]; } -function normalize( +export function normalizeContinuumFindings( input: unknown, sourceRepositoryUrl: string, - target: ExternalImportTarget, - previous: boolean, + target: ExternalImportTarget = "cloud", ): ExternalFindings { const repository = declaredScannerRepositoryUrl( sourceRepositoryUrl, "Continuum", target, ); - const { rows, summary } = reportRows(input, previous); + const { rows, summary } = reportRows(input); const scopes = new Set(); const output = normalizeRecords( rows, @@ -339,18 +307,14 @@ function normalize( finding: structuredClone(row), }, }; - const updatedAt = previous - ? null - : statusUpdatedAt(row["statusUpdatedAt"]); - const enrichment = previous - ? undefined - : { - packages: affectedPackages(packages, target), - source_updated_at: updatedAt, - }; + const updatedAt = statusUpdatedAt(row["statusUpdatedAt"]); + const enrichment = { + packages: affectedPackages(packages, target), + source_updated_at: updatedAt, + }; // Native code findings can identify a CWE only in subCategory. const namedCwe = - !previous && evidence.details?.weakness_ids.length === 0 + evidence.details?.weakness_ids.length === 0 ? subcategoryCwe(row) : undefined; const inferred = namedCwe @@ -380,31 +344,6 @@ function normalize( return output; } -export function normalizeContinuumFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalize(input, sourceRepositoryUrl, target, false); -} - -/** @internal Reconstruct the original evidence for an interrupted Cloud request. */ -export function normalizePreviousContinuumFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalize(input, sourceRepositoryUrl, target, true); -} - -export async function readContinuumFindings( - path: string, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): Promise { - return normalizeContinuumFindings( - await readScannerJson(path), - sourceRepositoryUrl, - target, - ); -} +export const readContinuumFindings = scannerJsonReader( + normalizeContinuumFindings, +); diff --git a/sdk/typescript/src/endor-findings.ts b/sdk/typescript/src/endor-findings.ts index 6769fb1830..38ae3b5458 100644 --- a/sdk/typescript/src/endor-findings.ts +++ b/sdk/typescript/src/endor-findings.ts @@ -1,4 +1,3 @@ -import { markPreviousCloudProjection } from "./reader-selection.js"; import { type ExternalImportTarget, validateEnrichedExternalEvidence, @@ -101,14 +100,6 @@ function endorListObjects(root: Record): unknown[] { return values; } -/** Saved Endor Finding objects, an array, or an unmasked ListFindingsResponse. */ -export function normalizeEndorFindings( - input: unknown, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeReport(input, target, false); -} - const reducedReports = new WeakSet(); /** A native upload would ask the worker to reconstruct fields that Cloud could not accept. */ @@ -116,18 +107,10 @@ export function requiresStructuredEndorReport(findings: ExternalFindings) { return reducedReports.has(findings); } -/** @internal Reproduce the selection from before malware and fully qualified package mapping. */ -export function normalizePreviousEndorFindings( +/** Saved Endor Finding objects, an array, or an unmasked ListFindingsResponse. */ +export function normalizeEndorFindings( input: unknown, target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeReport(input, target, true); -} - -function normalizeReport( - input: unknown, - target: ExternalImportTarget, - previous: boolean, ): ExternalFindings { const root = Array.isArray(input) ? null : record(input, "Endor report"); if (root?.["version"] === "2.1.0" && root["runs"] !== undefined) @@ -138,7 +121,6 @@ function normalizeReport( : Array.isArray(input) ? input : [input]; - let changed = false; let reduced = false; const result = normalizeRecords( values, @@ -148,9 +130,7 @@ function normalizeReport( const spec = record(finding["spec"], "Endor spec"); requiredText(spec["project_uuid"], "Endor project_uuid"); const bareName = text(spec["target_dependency_name"]); - const name = - bareName ?? - (previous ? null : text(spec["target_dependency_package_name"])); + const name = bareName ?? text(spec["target_dependency_package_name"]); const paths = [ ...new Set( strings(spec["dependency_file_paths"]) @@ -204,7 +184,7 @@ function normalizeReport( const vulnerabilityIds = [ ...new Set([...(advisory ? [advisory] : []), ...aliases]), ]; - const malware = previous ? undefined : asRecord(metadata["malware"]); + const malware = asRecord(metadata["malware"]); const advisories = new Set([ ...vulnerabilityIds, ...strings([asRecord(malware?.["meta"])?.["name"]]), @@ -266,9 +246,6 @@ function normalizeReport( ); const acceptedIds = evidence.advisory_ids!.length; const acceptedPackages = evidence.packages!.length; - changed ||= - acceptedIds > vulnerabilityIds.length || - (!bareName && acceptedPackages > 0); reduced ||= acceptedIds < advisories.size || acceptedPackages < packages.length; return evidence; @@ -280,10 +257,7 @@ function normalizeReport( ), target, ); - if (target === "cloud") { - if (changed) markPreviousCloudProjection(result); - if (reduced) reducedReports.add(result); - } + if (target === "cloud" && reduced) reducedReports.add(result); return result; } diff --git a/sdk/typescript/src/external-findings-local.ts b/sdk/typescript/src/external-findings-local.ts index f7f47846a1..e99f977ed3 100644 --- a/sdk/typescript/src/external-findings-local.ts +++ b/sdk/typescript/src/external-findings-local.ts @@ -1,6 +1,5 @@ import { resolve } from "node:path"; import { - semgrepLegacyRepositoryWitnesses, semgrepPlatformFindingBinding, semgrepPlatformReportScope, semgrepPlatformRepositoryReference, @@ -773,7 +772,12 @@ export async function prepareLocalImport( ): Promise { requireIdentity(options.sourceKey, "Source key"); const parsedSources = distinctSourceRepositories(parsedInput); - const current = snapshotNormalizedFindings(input, checkStoredNumber); + const current = snapshotNormalizedFindings( + input, + checkStoredNumber, + "local", + options.provider, + ); if ( options.provider !== "tenable" && (current.report?.["format"] === tenableReportFormat || @@ -785,17 +789,6 @@ export async function prepareLocalImport( "Retained Tenable evidence requires the tenable provider.", ); checkTrivyReportProvider(options.provider, current.report); - // The JSON snapshot omits inherited or non-enumerable SDK reader properties. - if (!("snykRestContext" in current) && "snykRestContext" in input) - current.snykRestContext = structuredClone(input.snykRestContext); - if (options.provider === "snyk" || current.snykRestContext !== undefined) { - for (const key of [ - "sourceRepositoryUrl", - "sourceRepositoryPlatform", - ] as const) - if (key in input && !(key in current)) - Object.assign(current, { [key]: structuredClone(input[key]) }); - } let dependabotSource: string | undefined; const identities = new Set(); for (const finding of current.findings) { @@ -975,11 +968,6 @@ export async function prepareLocalImport( if (semgrepPlatformRepositoryUrl) addSemgrepSource(semgrepPlatformRepositoryUrl); const known = new Set(parsedSources.map(({ url }) => url)); - const legacyWitnesses = semgrepLegacyRepositoryWitnesses( - current.findings, - options.provider, - readerSources, - ); const hasPlatformEvidence = options.provider === "semgrep" && (semgrepReportScope !== undefined || @@ -997,7 +985,6 @@ export async function prepareLocalImport( finding, options.provider, readerSources, - legacyWitnesses, semgrepReportScope, [retained, ...native, ...sarif], ); diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 11293a6914..2252ef7e75 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -1,23 +1,11 @@ +import { readScannerJson } from "./scanner-import.js"; import { hash, randomUUID } from "node:crypto"; import { chmodSync } from "node:fs"; -import { - lstat, - mkdir, - open, - readFile, - readdir, - rename, - rm, -} from "node:fs/promises"; +import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises"; import { join } from "node:path"; import { nullIfMissingFile } from "./targets.js"; import { setTimeout as delay } from "node:timers/promises"; -import { - isBoxedPrimitive, - isNativeError, - isProxy, - isStringObject, -} from "node:util/types"; +import { isProxy } from "node:util/types"; import { privateImportClient, privateJobVersions, @@ -38,18 +26,13 @@ import { guardianFindingRepository, guardianSelectionFormat, } from "./semgrep-guardian-findings.js"; -import { - cloudBaseUrl, - cloudFindingsUrl, - DEFAULT_CLOUD_BASE_URL, -} from "./cloud-endpoint.js"; +import { cloudBaseUrl, cloudFindingsUrl } from "./cloud-endpoint.js"; import { CodexSecurityError, errorMessage } from "./errors.js"; import { codexSecurityStateDirectory, createWindowsOutputDirectory, validatePreparedOutputDir, sqliteDatabaseOpener, - type RuntimeDatabase, } from "./runtime.js"; import { EXTERNAL_FINDINGS_PROVIDERS, @@ -59,36 +42,22 @@ import { snykFileRepositories, type CloudFindingsReader, } from "./external-findings-read.js"; -import { - canSnapshotWithoutGetters, - canonicalJson, - isRecord, -} from "./record.js"; +import { canonicalJson, isRecord } from "./record.js"; import { notify } from "./value.js"; import { wizCsvRepositoryUrl } from "./wiz-csv.js"; import { isWizPackageFinding, wizFindingRepository } from "./wiz-findings.js"; import { getOpenGrepSourcePositions } from "./opengrep-findings.js"; -import { - previousReaderSelection, - hasPreviousCloudProjection, -} from "./reader-selection.js"; import { dependabotRepository } from "./github-dependabot-findings.js"; import { retainedGitHubRepository } from "./github-findings.js"; import { - normalizePreviousSemgrepFindings, normalizeSemgrepFindings, readSemgrepReport, } from "./semgrep-findings.js"; -import { - normalizeSnykFile, - readSnykReport, - snykRepositoryUrls, -} from "./snyk-findings.js"; +import { normalizeSnykFile, snykRepositoryUrls } from "./snyk-findings.js"; import { type SnykIssuesFindings, isSnykRestFinding, isSnykRestFormat, - snykRestCheckpointFindingId, snykRestImportRepository, snykRestRepositoryMatches, } from "./snyk-rest-findings.js"; @@ -97,33 +66,23 @@ import { isSemgrepPlatformEvidence, isSemgrepPlatformFormat, semgrepPlatformFormat, - parseSemgrepResponse, - semgrepLegacyRepositoryWitnesses, - semgrepNativeProjectUrl, semgrepPlatformFindingBinding, semgrepPlatformRepositoryReference, semgrepPlatformReportScope, } from "./semgrep-platform-findings.js"; -import { semgrepV1Id } from "./semgrep-platform-v1.js"; import { hostedRepositoryUrlKey, normalizeHostedGitRepositoryUrl, } from "./hosted-git-repository.js"; import { trivyRemoteKey, trivyRepositoryUrl } from "./trivy-findings.js"; import { hostedRepositoryUrl } from "./hosted-scanner-http.js"; -import { - endorRepositoryUrls, - requiresStructuredEndorReport, -} from "./endor-findings.js"; +import { endorRepositoryUrls } from "./endor-findings.js"; import { blackDuckReportRepositories, projectedBlackDuckRepositoryUrl, } from "./blackduck-findings.js"; import { checkmarxSelection } from "./checkmarx-repository.js"; -import { - continuumReportRepositories, - previousContinuumCweProjection, -} from "./continuum-findings.js"; +import { continuumReportRepositories } from "./continuum-findings.js"; import { veracodeReportRepositories } from "./veracode-findings.js"; import { inspectorReportRepository } from "./inspector-findings.js"; import { sonarRepositoryUrls } from "./sonar-findings.js"; @@ -142,19 +101,14 @@ import { type ExternalSourceRepository, snapshotImportOptions, cloneImportPreview, - nativeErrorStackGetter, snapshotNormalizedFindings, - type ExternalFinding, type ExternalFindings, - type ExternalPublicationInput, } from "./external-findings.js"; import { canonicalNumber, cloudRepositoryUrlKey, - CloudEvidenceDepthError, validateImportRequest, validateExternalEvidence, - validatePreviousCloudEvidence, validateExternalReport, validateImportReceipt, validateRepositories, @@ -187,7 +141,7 @@ export interface ExternalPublicationOptions extends LocalExternalImportOptions { /** Optional transport, account, cancellation, and progress hooks. */ export interface ExternalPublicationDependencies { - /** Independently verified source repositories for hosted Semgrep or Snyk evidence. */ + /** Independently verified source repositories supplied by the reader. */ sourceRepositories?: readonly ExternalSourceRepository[]; environment?: NodeJS.ProcessEnv; fetch?: (url: string, options: RequestInit) => Promise; @@ -244,20 +198,8 @@ interface SavedSubmission { trivyRepositoryUrl?: string; requests: FindingImportRequest[]; receipts?: FindingImportReceipt[]; - inputProjection?: { - repositorySeverities?: string[]; - retainedHostedSelection?: true; - semgrepPartialScopeDigest?: string; - identityFormat?: 2; - inputDigest: string; - savedEvidenceDigest: string; - }; - // A verified older approval stays at its original deployment-bound filename. - wireKey?: string; - // Only a verified released approval may preserve a native value different from fresh JSON. - legacyApproval?: true; - // The older canonical file was checked under its lock for a second approval. - canonicalChecked?: true; + repositorySeverities?: string[]; + semgrepPartialScopeDigest?: string; /** Catalog URL when this source selection was approved, before any repository rename. */ approvedRepositoryUrl?: string; approvedRepositoryPlatform?: "other"; @@ -305,22 +247,12 @@ async function readInBatches( return output; } -// JSON.stringify serializes boxed primitives after the replacer. Let it -// produce their actual wire values, including false and strings. -function wireJson(value: unknown): string { - return canonicalJson(value, undefined, isBoxedPrimitive); -} - -function jsonDigest(value: unknown, serialize = wireJson): string { - return hash("sha256", serialize(value)); -} - function matchesRepositoryEvidence( expected: ExternalFindings["findings"][number]["evidence"], stored: ExternalFindings["findings"][number]["evidence"], ): boolean { - const storedWire = wireJson(stored); - if (wireJson(expected) === storedWire) return true; + const storedWire = canonicalJson(stored); + if (canonicalJson(expected) === storedWire) return true; // A repository can store a literal first-component colon with an explicit // relative prefix. Match only that spelling in typed locations and their manifests. const qualified = new Map(); @@ -337,7 +269,7 @@ function matchesRepositoryEvidence( const path = item.manifest_path && qualified.get(item.manifest_path); return path ? { ...item, manifest_path: path } : item; }); - return wireJson({ ...expected, locations, packages }) === storedWire; + return canonicalJson({ ...expected, locations, packages }) === storedWire; } function isCompleteUnscoredOpenGrepReport(report: ExternalFindings) { @@ -420,7 +352,7 @@ function sameSubmission( left.accountId === right.accountId && left.trivyRepositoryUrl === right.trivyRepositoryUrl && left.inspectorProjectMapRequired === right.inspectorProjectMapRequired && - wireJson(left.requests) === wireJson(right.requests) + canonicalJson(left.requests) === canonicalJson(right.requests) ); } @@ -437,62 +369,6 @@ function submissionFindings( return requests.flatMap((batch) => batch.items.map(wireFinding)); } -function sameLegacyJson( - saved: unknown, - current: unknown, - equivalent: (saved: unknown, current: unknown) => boolean, -): boolean { - if (saved === current || equivalent(saved, current)) return true; - if (Array.isArray(saved) && Array.isArray(current)) - return ( - saved.length === current.length && - saved.every((child, i) => sameLegacyJson(child, current[i], equivalent)) - ); - if (!isRecord(saved) || !isRecord(current)) return false; - return ( - Object.keys(current).every((key) => Object.hasOwn(saved, key)) && - Object.keys(saved).every((key) => - sameLegacyJson(saved[key], current[key], equivalent), - ) - ); -} - -// A copied reader result can retain the JSON from Buffer.toJSON but lose -// the native Buffer that the previous reader cloned into a Uint8Array. -function previousBufferJson(saved: unknown, current: unknown): boolean { - if ( - !isRecord(current) || - current["type"] !== "Buffer" || - !Array.isArray(current["data"]) - ) - return false; - return ( - isRecord(saved) && - Object.keys(current).length === 2 && - Object.keys(saved).length === current["data"].length && - current["data"].every( - (byte, index) => - Number.isInteger(byte) && - byte >= 0 && - byte <= 255 && - saved[index] === byte, - ) - ); -} - -function sameFindingSelection( - left: ExternalFindings["findings"], - right: ExternalFindings["findings"], -): boolean { - return ( - left.length === right.length && - left.every( - (finding, index) => - finding.source_finding_id === right[index]!.source_finding_id, - ) - ); -} - async function readOptionalFile(path: string): Promise { return (await readFile(path, "utf8").catch(nullIfMissingFile)) ?? undefined; } @@ -546,8 +422,8 @@ function validatePublicationReceipt( const remaining = new Set(batch.items.map((item) => item.client_id)); if ( receipt.id !== batch.request_id || - wireJson(receipt.repository) !== wireJson(batch.repository) || - wireJson(receipt.source) !== wireJson(batch.source) || + canonicalJson(receipt.repository) !== canonicalJson(batch.repository) || + canonicalJson(receipt.source) !== canonicalJson(batch.source) || receipt.item_count !== batch.items.length || receipt.results.length !== batch.items.length || receipt.results.some((item) => !remaining.delete(item.client_id)) @@ -580,16 +456,16 @@ function validatePublicationReceipt( "Cloud returned an incomplete saved finding.", ); } - if (wireJson(receipt.counts) !== wireJson(actual)) + if (canonicalJson(receipt.counts) !== canonicalJson(actual)) throw new CodexSecurityError( "Cloud returned inconsistent publication counts.", ); return receipt; } -async function withImportLocks( +async function withImportLock( state: string, - keys: string[], + key: string, signal: AbortSignal | undefined, operation: () => Promise, ): Promise { @@ -597,43 +473,29 @@ async function withImportLocks( await createWindowsOutputDirectory(state, true); else await mkdir(state, { recursive: true, mode: 0o700 }); await validatePreparedOutputDir(state, undefined, true); - const openDatabase = sqliteDatabaseOpener(); - const guards: RuntimeDatabase[] = []; + const path = join(state, key + ".lock.sqlite"); + const metadata = await lstat(path).catch(nullIfMissingFile); + if (metadata && !metadata.isFile()) + throw new CodexSecurityError("The publication lock is not a regular file."); + const guard = sqliteDatabaseOpener()(path); try { - for (const key of [...new Set(keys)].sort()) { - const path = join(state, `${key}.lock.sqlite`); - const metadata = await lstat(path).catch(nullIfMissingFile); - if (metadata && !metadata.isFile()) - throw new CodexSecurityError( - "The publication lock is not a regular file.", - ); - const guard = openDatabase(path); - guards.push(guard); - if (process.platform !== "win32") chmodSync(path, 0o600); - guard.exec("PRAGMA busy_timeout = 0"); - } + if (process.platform !== "win32") chmodSync(path, 0o600); + guard.exec("PRAGMA busy_timeout = 0"); for (;;) { signal?.throwIfAborted(); - let acquired = 0; try { - // SQLite releases these transactions on process exit. - for (const guard of guards) { - guard.exec("BEGIN EXCLUSIVE"); - acquired++; - } + guard.exec("BEGIN EXCLUSIVE"); break; } catch (error) { const sqliteError = error as { code?: string; errcode?: number }; if (sqliteError.errcode !== 5 && sqliteError.code !== "SQLITE_BUSY") throw error; - // Older SDKs can acquire historical keys in a different order. - for (const guard of guards.slice(0, acquired)) guard.exec("ROLLBACK"); await delay(50, undefined, { signal }); } } return await operation(); } finally { - for (const guard of guards) guard.close(); + guard.close(); } } @@ -663,28 +525,6 @@ function hostedSourceReferences( : sources; } -function storedFindingId(finding: ExternalFinding): string { - const seen = new Set(); - for ( - let owner: object | null = finding; - owner && !seen.has(owner); - owner = Object.getPrototypeOf(owner) as object | null - ) { - const property = Object.getOwnPropertyDescriptor( - owner, - "source_finding_id", - ); - if (property) { - const id: unknown = property.value; - return typeof id === "string" || isStringObject(id) - ? String.prototype.valueOf.call(id) - : ""; - } - seen.add(owner); - } - return ""; -} - function checkFindingProvider( input: ExternalFindings, provider: Provider, @@ -697,13 +537,11 @@ function checkFindingProvider( "Use the semgrep provider for Guardian findings.", ); } - checkFormat(ownData(ownData(input, "report"), "format")); - if (!Array.isArray(input?.findings)) return; - for (const row of input.findings) { - if (!row) continue; - if (guardianFindingId.test(storedFindingId(row))) + checkFormat(input.report?.["format"]); + for (const { source_finding_id, evidence } of input.findings) { + if (guardianFindingId.test(source_finding_id)) checkFormat(guardianSelectionFormat); - checkFormat(ownData(ownData(row.evidence, "source_data"), "format")); + checkFormat(evidence.source_data?.["format"]); } } @@ -724,7 +562,7 @@ export async function prepareExternalPublication( const semgrep = settings.provider === "semgrep"; const report = await (semgrep ? readSemgrepReport(path) - : readSnykReport(path)); + : readScannerJson(path)); const normalize = async (url?: string, platform?: "other") => { const references = hostedSourceReferences( readerSources, @@ -738,12 +576,7 @@ export async function prepareExternalPublication( const findings = semgrep ? normalizeSemgrepFindings(report, semgrepOptions, "cloud") : await normalizeSnykFile(report, "cloud", references); - const selection = normalizedPublicationInput(findings, provider); - if (semgrep && hasPreviousCloudProjection(findings)) - selection.previousReaders = [ - () => normalizePreviousSemgrepFindings(report, semgrepOptions), - ]; - return selection; + return normalizedPublicationInput(findings, provider); }; const hosted = isRecord(report) && @@ -790,18 +623,9 @@ export async function prepareExternalPublication( : await normalize(); } else { const parsed = await readExternalFindings(path, settings); - checkFindingProvider(parsed.findings, provider); - input = - parsed.publication ?? - normalizedPublicationInput(parsed.findings, provider); - if (parsed.previousFindings) - (input.previousReaders ??= []).push(parsed.previousFindings); - if (provider === "endor") { + input = normalizedPublicationInput(parsed.findings, provider); + if (provider === "endor") fileRepositories = () => parsed.sourceRepositories ?? []; - input.requiresStructuredReport = requiresStructuredEndorReport( - parsed.findings, - ); - } } return preparePublicationInput( input, @@ -809,7 +633,6 @@ export async function prepareExternalPublication( dependencies, readerSources, settings.sourceRepositoryUrl, - path, fileRepositories, ); } @@ -849,7 +672,6 @@ function hostedRepositoryMatcher( declaredUrl: string | undefined, destination: Pick, readerSources: readonly ExternalSourceRepository[], - allowOpaqueLegacyRows = false, ): { github: boolean; matches: (url: string) => boolean } | undefined { const references = hostedSourceReferences( readerSources, @@ -871,7 +693,6 @@ function hostedRepositoryMatcher( ], references, undefined, - allowOpaqueLegacyRows, declaredSource ? destination.url : undefined, ); if (binding) @@ -910,93 +731,19 @@ function hostedRepositoryMatcher( claims.add(guardian); continue; } - const data = finding.evidence.source_data; const sarif = semgrepSarifRepositories(finding, hasNativeSource); for (const { url } of sarif) claims.add(hostedRepositoryUrl(url, "Semgrep source repository")); - const project = data?.["project"]; - if ( - !allowOpaqueLegacyRows || - !isSemgrepPlatformEvidence(finding, provider) || - (isRecord(project) && - project["id"] != null && - project["name"] != null && - data?.["deployment"] != null && - data["ref"] != null && - data["product"] != null && - ((isRecord(data["finding"]) && data["finding"]["id"] != null) || - typeof data["semgrepPlatformFindingJson"] === "string")) - ) { - nativeRows.push({ finding, sarif }); - continue; - } - // Released SDKs could save partial native records. Retain any source still - // identifiable on that row while validating complete siblings under the new mapping. - if (!data) continue; - if (reportScope?.projectId !== undefined) { - const id = isRecord(project) ? project["id"] : undefined; - if ( - (id != null && - id !== reportScope.projectId && - semgrepV1Id(id) !== reportScope.projectId) || - (["deployment", "ref", "product"] as const).some( - (field) => data[field] != null && data[field] !== reportScope[field], - ) - ) - throw new CodexSecurityError( - "The retained Semgrep finding does not match the supplied report scope.", - ); - } - const rows = [data["finding"]]; - const raw = data["semgrepPlatformFindingJson"]; - if (typeof raw === "string") { - try { - rows.push(parseSemgrepResponse(raw)); - } catch { - // An unreadable historical row contributes no URL. Check the remaining claims. - } - } - const nativeUrls = [isRecord(project) ? project["url"] : undefined]; - if (data["format"] === semgrepPlatformFormat) - nativeUrls.push(data["nativeProjectUrl"]); - for (const row of rows) { - if (!isRecord(row)) continue; - if ( - reportScope?.ref !== undefined && - row["ref"] !== undefined && - row["ref"] !== reportScope.ref - ) - throw new CodexSecurityError( - "The retained Semgrep finding does not match the supplied report scope.", - ); - const repository = row["repository"]; - if (isRecord(repository)) nativeUrls.push(repository["url"]); - } - const urls = nativeUrls.map((url) => - semgrepNativeProjectUrl(url, references), - ); - for (const url of [...urls, data["sourceRepositoryUrl"]]) { - if (typeof url !== "string") continue; - original ??= url; - claims.add(url); - } + nativeRows.push({ finding, sarif }); } - const witnesses = semgrepLegacyRepositoryWitnesses( - nativeRows.map(({ finding }) => finding), - provider, - references, - "cloud", - ); for (const { finding, sarif } of nativeRows) { const native = semgrepPlatformFindingBinding( finding, provider, references, - witnesses, reportScope, [{ url: projectedBlackDuckRepositoryUrl(finding, "cloud") }, ...sarif], "cloud", - allowOpaqueLegacyRows, )?.url; if (native) { original ??= native; @@ -1032,23 +779,9 @@ function hostedRepositoryMatcher( return { github: platform === "github", matches }; } -interface PublicationSelection extends ExternalPublicationInput { - requiresStructuredReport?: boolean; +interface PublicationSelection { + current: ExternalFindings; sourcePositions?: ReadonlyMap; - previousReaders?: (() => ExternalFindings)[]; - original?: { - parsed: ExternalFindings; - identity: unknown; - wire: string; - hasBoxedScalars: boolean; - }; - - potentialIds?: (string | undefined)[] | null; - uncertainFindingIds?: string[]; - unprovenOccurrenceCount?: number; - knownFindings?: ExternalFindings["findings"]; - unprovenIdPrefixes?: string[]; - freshError?: Error; } function ownData(value: unknown, key: string) { @@ -1057,107 +790,6 @@ function ownData(value: unknown, key: string) { : undefined; } -// When native capture would invoke caller code, only the original data -// properties can identify the earlier selection. Unknown IDs cannot rule it out. -function previousFindingIds( - input: ExternalFindings, -): (string | undefined)[] | null { - const rows = ownData(input, "findings"); - if (!Array.isArray(rows) || isProxy(rows)) return null; - return Array.from({ length: rows.length }, (_, index) => { - const row: unknown = Object.getOwnPropertyDescriptor(rows, index)?.value; - const id = ownData(row, "source_finding_id"); - return typeof id === "string" ? id : undefined; - }); -} - -function withDataProperty( - record: T, - key: string, - value: unknown, -): T { - const fields = Object.getOwnPropertyDescriptors(record); - return Object.create(Object.getPrototypeOf(record) as object | null, { - ...fields, - [key]: { ...fields[key], value }, - }) as T; -} - -// Native cloning discarded enumerable diagnostic fields. Recover its message -// without reading the caller's stack getter or changing their report. -const nativeDiagnosticPrototypes = new Set([ - Error.prototype, - EvalError.prototype, - RangeError.prototype, - ReferenceError.prototype, - SyntaxError.prototype, - TypeError.prototype, - URIError.prototype, -]); - -function previousDiagnosticMessage(diagnostic: Error): string | undefined { - const stack = Object.getOwnPropertyDescriptor(diagnostic, "stack"); - const message = ownData(diagnostic, "message"); - const cause = Object.getOwnPropertyDescriptor(diagnostic, "cause"); - const name = Object.getOwnPropertyDescriptor(diagnostic, "name"); - if ( - !nativeDiagnosticPrototypes.has(Object.getPrototypeOf(diagnostic)) || - typeof message !== "string" || - (stack?.get && stack.get !== nativeErrorStackGetter) || - (name && typeof name.value !== "string") || - (cause && - (!Object.hasOwn(cause, "value") || - !canSnapshotWithoutGetters(cause.value))) - ) - return; - return message; -} - -function previousDiagnosticInput( - input: ExternalFindings, - nativeDiagnostics: Map, -): ExternalFindings { - const report = ownData(input, "report"); - const errors = ownData(report, "errors"); - if (!isRecord(report) || !Array.isArray(errors) || isProxy(errors)) - return input; - let entries: PropertyDescriptorMap | undefined; - for (let index = 0; index < errors.length; index++) { - const descriptor = Object.getOwnPropertyDescriptor(errors, index); - const diagnostic: unknown = descriptor?.value; - if (!descriptor || !isNativeError(diagnostic)) continue; - const message = previousDiagnosticMessage(diagnostic); - if (message === undefined) return input; - nativeDiagnostics.set(index, diagnostic); - (entries ??= Object.getOwnPropertyDescriptors(errors as object))[index] = { - ...descriptor, - value: { message }, - }; - } - if (!entries) return input; - const capturedErrors = Object.defineProperties([], entries) as unknown[]; - const capturedReport = withDataProperty(report, "errors", capturedErrors); - return withDataProperty(input, "report", capturedReport); -} - -// Capture before JSON hooks can change their source. If native capture would -// invoke caller code too, the original bytes cannot be observed independently. -function previousNormalizedFindings( - input: ExternalFindings, - nativeDiagnostics: Map, - observe: (value: object) => void, -): ExternalFindings | undefined { - input = previousDiagnosticInput(input, nativeDiagnostics); - if (!canSnapshotWithoutGetters(input, observe)) return; - // Object spread skips a hidden report, but explicitly reading it could run a getter. - const reportField = Object.getOwnPropertyDescriptor(input, "report"); - if (reportField && !Object.hasOwn(reportField, "value")) return; - const { report, ...parsed } = input; - const snapshot: ExternalFindings = structuredClone(parsed); - if (report !== undefined) snapshot.report = structuredClone(report); - return snapshot; -} - function publicationIdentity( parsed: ExternalFindings, provider?: NormalizedExternalPublicationOptions["provider"], @@ -1172,10 +804,7 @@ function publicationIdentity( return { findings: parsed.findings, report: reportForJson(parsed.report) }; } -function validatePublicationFindings( - parsed: ExternalFindings, - validateEvidence: typeof validateExternalEvidence, -) { +function validatePublicationFindings(parsed: ExternalFindings) { validateExternalReport(parsed.report); const ids = new Set(); for (const finding of parsed.findings) { @@ -1189,34 +818,7 @@ function validatePublicationFindings( `Duplicate source finding id ${JSON.stringify(id)}.`, ); ids.add(id); - finding.evidence = validateEvidence(finding.evidence); - } -} - -function historicalPublication( - native: ExternalFindings | undefined, -): PublicationSelection["original"] { - if (!native) return; - try { - // Older SDKs detached all raw records before Cloud validation added defaults. - for (const finding of native.findings) - if (finding.evidence.source_data !== undefined) - finding.evidence.source_data = structuredClone( - finding.evidence.source_data, - ); - validatePublicationFindings(native, validatePreviousCloudEvidence); - const identity = publicationIdentity(native); - let hasBoxedScalars = false; - const wire = canonicalJson(identity, undefined, (value) => { - if (!isBoxedPrimitive(value)) return false; - hasBoxedScalars = true; - return true; - }); - return { parsed: native, identity, wire, hasBoxedScalars }; - } catch { - // The caller may export valid JSON even if the native candidate cannot be - // validated or hashed, for example when an unused row property holds a BigInt. - return; + finding.evidence = validateExternalEvidence(finding.evidence); } } @@ -1224,313 +826,114 @@ function normalizedPublicationInput( input: ExternalFindings & Partial, provider: NormalizedExternalPublicationOptions["provider"], ): PublicationSelection { - const nativeDiagnostics = new Map(); - let native: ExternalFindings | undefined; - let plainJson = true; - try { - native = previousNormalizedFindings(input, nativeDiagnostics, (value) => { - if (!plainJson) return; - const prototype = Object.getPrototypeOf(value); - plainJson = - (prototype === null || - prototype === Object.prototype || - prototype === Array.prototype) && - !("toJSON" in value); - }); - } catch { - /* A valid JSON export can contain an uncloneable value. */ - } - let priorIds = native ? undefined : previousFindingIds(input); - let current: ExternalFindings; - let freshError: Error | undefined; - try { - current = snapshotNormalizedFindings(input, undefined, "cloud"); - } catch (error) { - if ( - (!isNativeError(error) && !(error instanceof Error)) || - !native?.findings.length - ) - throw error; - freshError = error; - current = { read: native.read, findings: [], excluded: [] }; - } - // Repository and format declarations can be non-enumerable SDK getters. - for (const key of [ - "sourceFormat", - "semgrepPlatformScope", - "snykRestContext", - ...(provider === "semgrep" || provider === "snyk" - ? ["sourceRepositoryUrl", "sourceRepositoryPlatform"] - : []), - ]) { - if ( - !(key in current) && - !Object.prototype.propertyIsEnumerable.call(input, key) && - key in input - ) - Object.assign(current, { - [key]: structuredClone(Reflect.get(input, key)), - }); - } - const parsed: ExternalFindings & Partial = current; - if (parsed.snykRestContext !== undefined) { + const current = snapshotNormalizedFindings( + input, + undefined, + "cloud", + provider, + ); + if (current.snykRestContext !== undefined) { if (provider !== "snyk") throw new CodexSecurityError( "Snyk REST evidence requires the snyk provider.", ); - if (!isSnykRestFormat(parsed.snykRestContext?.["format"])) + if (!isSnykRestFormat(current.snykRestContext?.["format"])) throw new CodexSecurityError( "Expected a versioned Snyk REST report context.", ); } const markedSemgrep = - "sourceFormat" in parsed && isSemgrepPlatformFormat(parsed.sourceFormat); + "sourceFormat" in current && isSemgrepPlatformFormat(current.sourceFormat); if (markedSemgrep && provider !== "semgrep") throw new CodexSecurityError( "Semgrep Platform evidence requires the semgrep provider.", ); - if (markedSemgrep || "semgrepPlatformScope" in parsed) - semgrepPlatformReportScope(parsed, provider); - if (native && nativeDiagnostics.size) { - const historical = native.report!["errors"] as Error[]; - const exported = current.report?.["errors"]; - for (const [index, source] of nativeDiagnostics) { - const message = historical[index]!.message; - let diagnostic: unknown = Array.isArray(exported) && exported[index]; - // A failed export may never reach the report. Its hooks have already run, - // so check the original descriptors again before native cloning reads a stack. - if (freshError && previousDiagnosticMessage(source) === message) - diagnostic = source; - if ( - isNativeError(diagnostic) && - Object.getPrototypeOf(diagnostic) === Object.getPrototypeOf(source) - ) { - try { - const copy = structuredClone(diagnostic); - if (copy.message === message) { - historical[index] = copy; - continue; - } - } catch { - // A failed native clone cannot establish a historical report. - } - } - if (freshError) throw freshError; - priorIds = previousFindingIds(native); - native = undefined; - break; - } - } - // A Cloud depth error can interrupt a local reader's JSON export. Its plain, - // independently captured output can still verify the original reader selection. - const readerOutput = freshError - ? freshError instanceof CloudEvidenceDepthError && - plainJson && - !nativeDiagnostics.size && - native - : current; - const reader = readerOutput - ? previousReaderSelection(input, readerOutput, native ?? null) - : undefined; - if (reader) native = reader.snapshot; - const original = historicalPublication(native); - if (original) checkFindingProvider(original.parsed, provider); - try { - validatePublicationFindings(current, validateExternalEvidence); - } catch (error) { - if (!(error instanceof Error) || !original?.parsed.findings.length) - throw error; - freshError = error; - } - if (freshError) { - if (!original) throw freshError; - // Only the old selection can be reviewed when fresh validation failed. - current = { read: original.parsed.read, findings: [], excluded: [] }; - } + if (markedSemgrep || "semgrepPlatformScope" in current) + semgrepPlatformReportScope(current, provider); + validatePublicationFindings(current); checkFindingProvider(current, provider); - const identity = publicationIdentity(current, provider); return { current, - legacy: current, - currentIdentity: identity, - legacyIdentity: identity, - previousReaders: - provider === "continuum" - ? [() => previousContinuumCweProjection(current)] - : undefined, sourcePositions: getOpenGrepSourcePositions(ownData(input, "findings")), - original, - freshError, - unprovenIdPrefixes: reader?.unprovenIdPrefixes, - uncertainFindingIds: reader?.uncertainFindingIds, - unprovenOccurrenceCount: reader?.unprovenOccurrenceCount, - knownFindings: reader?.knownFindings, - potentialIds: - reader?.sourceFindingIds ?? - (original - ? undefined - : (native?.findings.map((row) => row.source_finding_id) ?? priorIds)), }; } -// The old hash represented Number objects as {}. Unpublished row properties -// and report metadata cannot be recovered from the immutable saved request. -function verifiableLegacyMetadata( - { report, findings }: ExternalFindings, - wireIdentity: boolean, -): boolean { - let verifiable = true; - // Only values in these records can be old scalar projections. Their record - // containers are required by the normalized selection, including an empty report. - const { errors, ...metadata } = report ?? {}; - JSON.stringify( - [ - Object.values(metadata), - // The original hash used the serialized form of report.errors arrays. - Array.isArray(errors) ? undefined : errors, - findings.map(({ evidence: _, ...fields }) => Object.values(fields)), - ], - (_key, value: unknown) => { - if (isRecord(value) && !(wireIdentity && isBoxedPrimitive(value))) { - // {} and string-index records can both be the old form of a scalar. - let index = 0; - if ( - Object.keys(value).every((key) => { - const item = value[key]; - return ( - item === undefined || - (key === String(index++) && - typeof item === "string" && - item.length === 1) - ); - }) - ) - verifiable = false; - } - return value; - }, - ); - return verifiable; -} - -// Prefer the current native evidence for new publications. The original SDK input -// remains a candidate so an approved request can resume under its exact digest. -function* hostedPublicationProjections( - input: ResolvedPublicationInput, - provider: "semgrep" | "snyk", +// A normalized hosted input can omit native fields that the authenticated +// repository and the retained source response establish for each occurrence. +function hostedPublicationFindings( + current: ExternalFindings & Partial, + provider: Provider, destination: ImportRepository, readerSources: readonly ExternalSourceRepository[], -): Generator { - const current: ExternalFindings & Partial = input.current; +): ExternalFindings { if (provider === "snyk") { const first = current.findings.find(isSnykRestFinding); - if (!first && current.snykRestContext === undefined) return; - // A previous publisher may have approved the exact raw selection. - yield input; - const originalUrl = + if (!first && current.snykRestContext === undefined) return current; + const url = current.snykRestContext?.["sourceRepositoryUrl"] ?? current.sourceRepositoryUrl ?? first?.evidence.source_data?.["sourceRepositoryUrl"]; - // The original spelling reconstructs a lookup key after a catalog rename. - // Admission validates the input against the saved approval or live destination. - const retained = structuredClone(current); - try { - snykRestImportRepository( - retained, - provider, - [], - hostedSourceReferences( - readerSources, - typeof originalUrl === "string" ? originalUrl : destination.url, - current.sourceRepositoryPlatform, - ), - (finding, source) => { - finding.evidence.source_data = source; - }, - ); - } catch (error) { - // A headerless input can still identify an already-approved opaque request. - if (current.snykRestContext !== undefined) throw error; - return; - } - yield { - ...input, - current: retained, - // The raw input above owns its historical native checkpoint. - original: undefined, - currentIdentity: publicationIdentity(retained), - }; - return; + snykRestImportRepository( + current, + provider, + [], + hostedSourceReferences( + readerSources, + typeof url === "string" ? url : destination.url, + current.sourceRepositoryPlatform, + ), + (finding, source) => { + finding.evidence.source_data = source; + }, + ); + return current; } - const originalRow = (finding: ExternalFindings["findings"][number]) => - isRecord(finding.evidence.source_data?.["finding"]) && - isSemgrepPlatformEvidence(finding, "semgrep"); - const currentBulkRow = (finding: ExternalFindings["findings"][number]) => - finding.evidence.source_data?.["format"] === semgrepPlatformFormat; - const first = - current.findings.find(originalRow) ?? current.findings.find(currentBulkRow); - if (!first) return; - const reportScope = semgrepPlatformReportScope(current, "semgrep"); + if (provider !== "semgrep") return current; + const first = current.findings.find((finding) => + isSemgrepPlatformEvidence(finding, provider), + ); + if (!first) return current; const retained = first.evidence.source_data; const project = retained?.["project"]; - const retainedUrl = + const url = current.sourceRepositoryUrl ?? retained?.["sourceRepositoryUrl"] ?? (isRecord(project) ? project["url"] : undefined); - // A renamed repository may need the original spelling to identify its checkpoint. - // Preparation validates the unmodified report against the authenticated approval. - for (const url of new Set([destination.url, retainedUrl])) { - if (typeof url !== "string") continue; - const references = hostedSourceReferences( - readerSources, - url, - current.sourceRepositoryPlatform, + const references = hostedSourceReferences( + readerSources, + typeof url === "string" + ? normalizeHostedGitRepositoryUrl(url, destination.url, true) + : destination.url, + current.sourceRepositoryPlatform, + ); + const scope = semgrepPlatformReportScope(current, provider, references); + const findings = current.findings.map((finding) => { + const data = finding.evidence.source_data; + const original = + isRecord(data?.["finding"]) && + isSemgrepPlatformEvidence(finding, provider); + if (!original && data?.["format"] !== semgrepPlatformFormat) return finding; + const binding = semgrepPlatformFindingBinding( + finding, + provider, + references, + scope, + undefined, + "cloud", ); - try { - const witnesses = semgrepLegacyRepositoryWitnesses( - current.findings, - "semgrep", - references, - "cloud", - ); - const findings = current.findings.map((finding) => { - const original = originalRow(finding); - if (!original && !currentBulkRow(finding)) return finding; - const binding = semgrepPlatformFindingBinding( - finding, - "semgrep", - references, - witnesses, - reportScope, - undefined, - "cloud", - ); - // The authenticated repository can make an offline backlog row selectable. - return binding?.sourceData && - (original || binding.sourceData["format"] !== semgrepPlatformFormat) - ? { - ...finding, - evidence: { - ...finding.evidence, - ...binding.migration, - source_data: binding.sourceData, - }, - } - : finding; - }); - const upgraded = { ...current, findings }; - yield { - ...input, - current: upgraded, - original: undefined, - currentIdentity: publicationIdentity(upgraded), - }; - } catch { - // The released SDK also accepted partial original records; keep the raw - // identity as a fallback so its exact saved request remains recoverable. - } - } - yield input; + return binding?.sourceData && + (original || binding.sourceData["format"] !== semgrepPlatformFormat) + ? { + ...finding, + evidence: { + ...finding.evidence, + ...binding.migration, + source_data: binding.sourceData, + }, + } + : finding; + }); + return { ...current, findings }; } async function preparePublicationInput( @@ -1539,7 +942,6 @@ async function preparePublicationInput( dependencies: ExternalPublicationDependencies, readerSources: readonly ExternalSourceRepository[], declaredUrl?: string, - nativePath?: string, fileRepositories?: () => readonly ExternalSourceRepository[], ): Promise { const { inspectorProjectMap } = options; @@ -1569,48 +971,14 @@ async function preparePublicationInput( ? (url: string) => cloudSourceKey(url, "github") : cloudRepositoryUrlKey; const state = externalPublicationStateDirectory(environment); - function pendingFile(key: string) { - return join(state, `${key}.pending.json`); - } - let previousFindings: ExternalFindings[] | undefined; - async function previousFindingsIfSaved( - input: PublicationSelection, - currentKey?: string, - ) { - if (!input.previousReaders || previousFindings) return previousFindings; - // An empty state directory (or just this current receipt) needs only the - // current reader for its preview. - const entries = (await readdir(state).catch(nullIfMissingFile)) ?? []; - if ( - entries.some( - (name) => - /^[a-f0-9]{64}\.(?:pending|private-import)\.json$/u.test(name) && - name.split(".", 1)[0] !== currentKey, - ) - ) - previousFindings = input.previousReaders.map((read) => read()); - return previousFindings; - } - async function checkSelection(input: PublicationSelection) { + function checkSelection(input: PublicationSelection) { const parsed = input.current; checkTrivyReportProvider(options.provider, parsed.report); for (const finding of parsed.findings) checkRetainedTrivyProvider(options.provider, finding); - if ( - !parsed.findings.length && - !input.original?.parsed.findings.length && - input.potentialIds !== null && - !input.potentialIds?.length && - !input.uncertainFindingIds?.length && - !input.unprovenOccurrenceCount && - !input.unprovenIdPrefixes?.length && - !(await previousFindingsIfSaved(input))?.some( - (report) => report.findings.length, - ) - ) - throw new NoPublishableFindingsError(parsed); + if (!parsed.findings.length) throw new NoPublishableFindingsError(parsed); } - if (typeof supplied !== "function") await checkSelection(supplied); + if (typeof supplied !== "function") checkSelection(supplied); progress({ phase: "discovering", completed: 0 }); const apiBaseUrl = cloudBaseUrl(environment); const cloudApiUrl = `${apiBaseUrl}/external`; @@ -1732,196 +1100,45 @@ async function preparePublicationInput( const destination = await discoverDestination(); const deferred = typeof supplied === "function"; const resolved = deferred ? await supplied(destination) : supplied; - if (deferred) await checkSelection(resolved); + if (deferred) checkSelection(resolved); const report: ExternalFindings & Partial = resolved.current; const source = { provider: options.provider, source_key: options.sourceKey }; const importPrefix = source.provider === "sonar" ? "/v2" : ""; const sourceReportsPath = importPrefix + sourcePath(destination); - const checkpointIdentity = [ - // Keep existing production checkpoints readable; other deployments must - // never share a saved request or receipt with the production default. - ...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]), - credentials.account_id, - destination.id, - destination.repo_connector_id, - source, - ]; - function scopeKey(identity: unknown, serialize = wireJson) { - return jsonDigest([...checkpointIdentity, identity], serialize); - } - function localeKey(rows: ExternalFindings["findings"]) { - return hash( - "sha256", - canonicalJson([...checkpointIdentity, rows], (left, right) => - left.localeCompare(right), - ), - ); - } const hosted = options.provider === "snyk" || options.provider === "semgrep"; - // Different selections can share native occurrences. Serialize their checkpoint - // creation under the authenticated source and release before calling Cloud. - const sourceLock = - "source-" + hash("sha256", canonicalJson(checkpointIdentity)); - const withSourceAdmission = (operation: () => Promise) => - hosted ? underLocks([sourceLock], operation) : operation(); - let projection = resolved; - let capturedPending: string | undefined; - const projectionKeys = new Set(); - if (options.provider === "semgrep" || options.provider === "snyk") { - for (const candidate of hostedPublicationProjections( - resolved, - options.provider, - destination, - readerSources, - )) { - const candidateKey = scopeKey(candidate.currentIdentity); - if (projectionKeys.has(candidateKey)) continue; - // New imports use the first reconstructed projection; an exact pending key - // takes precedence, including an earlier SDK's unmodified original input. - capturedPending = await readOptionalFile(pendingFile(candidateKey)); - if (projection === resolved || capturedPending !== undefined) - projection = candidate; - projectionKeys.add(candidateKey); - if (capturedPending !== undefined) break; - } - } - // Changed reader projections get new keys, but both transports must replay the - // exact bytes already approved by the earlier reader when a checkpoint exists. + const parsed = hostedPublicationFindings( + report, + options.provider, + destination, + readerSources, + ); + const key = hash( + "sha256", + canonicalJson([ + "external-findings/2", + apiBaseUrl, + credentials.account_id, + destination.id, + destination.repo_connector_id, + source, + publicationIdentity(parsed, options.provider), + ]), + ); + const pendingPath = join(state, key + ".pending.json"); + const underPublishLock = (operation: () => Promise) => + withImportLock(state, key, dependencies.signal, operation); + const removePending = () => rm(pendingPath, { force: true }); const privateClient = privateImportClient( apiBaseUrl, credentials, dependencies.fetch ?? globalThis.fetch, dependencies.signal, ); - let previewContext: PrivateImportContext | undefined; - const currentReaderKey = resolved.previousReaders - ? scopeKey(resolved.currentIdentity) - : undefined; - const alternateReaders = new Map(); - let resumingPreviousReader = false; - function addPreviousReaders(readers: ExternalFindings[] | undefined) { - // A source can produce identical evidence under multiple reader versions. - for (const reader of readers ?? []) { - if (reader === resolved.current || !reader.findings.length) continue; - const key = scopeKey(publicationIdentity(reader, options.provider)); - if (key !== currentReaderKey) alternateReaders.set(key, reader); - } - } - async function alternateCheckpoint( - key: string, - reader: ExternalFindings, - context: () => Promise, - ): Promise<{ pending: string | undefined } | undefined> { - const pending = await readOptionalFile(pendingFile(key)); - if (pending !== undefined) return { pending }; - const saved = await readOptionalFile( - join(state, key + ".private-import.json"), - ); - if (saved === undefined) return; - const checkpoint = parsePrivateCheckpoint(saved); - // A completed import cannot acknowledge findings newly admitted by the - // current reader. Pending uploads and jobs always retain their original bytes. - if ( - key !== currentReaderKey && - checkpoint.receipt?.status === "completed" && - !sameFindingSelection(reader.findings, resolved.current.findings) - ) - return; - if (checkpoint.contextVersion === (await context()).context_version) - return { pending }; - } - async function assertNoAlternateCheckpoint( - context = () => privateClient.context(), - ) { - for (const [key, reader] of alternateReaders) { - if (await alternateCheckpoint(key, reader, context)) - throw new CodexSecurityError( - resumingPreviousReader - ? "Multiple reader versions have saved publications for this report. Resume one using its original normalized findings." - : "Another publication prepared this input. Run the command again to review and resume the saved request.", - ); - } - } - if (currentReaderKey) { - addPreviousReaders( - await previousFindingsIfSaved(resolved, currentReaderKey), - ); - const context = async () => - (previewContext ??= await privateClient.context()); - for (const [key, reader] of alternateReaders) { - const checkpoint = await alternateCheckpoint(key, reader, context); - if (!checkpoint) continue; - capturedPending = checkpoint.pending; - projection = normalizedPublicationInput(reader, options.provider); - resumingPreviousReader = true; - alternateReaders.delete(key); - alternateReaders.set(currentReaderKey, resolved.current); - await assertNoAlternateCheckpoint(context); - break; - } - } - const input = projection; - let { - current: parsed, - legacy: legacyParsed, - currentIdentity, - legacyIdentity, - recheckFileRepository, - } = input; const scopedSemgrepReport = options.provider === "semgrep" && "sourceFormat" in parsed && isSemgrepPlatformFormat(parsed.sourceFormat); - const original = input.original; - const originalParsed = original?.parsed ?? parsed; - const originalIdentity = original?.identity ?? currentIdentity; - const hasBoxedScalars = original?.hasBoxedScalars === true; - const changedCapture = - original && original.wire !== wireJson(currentIdentity); - const originalKey = - changedCapture || hasBoxedScalars - ? scopeKey(originalIdentity, canonicalJson) - : undefined; - let historicalOnly = false; - const originalWireKey = changedCapture - ? scopeKey(originalIdentity) - : undefined; - if (changedCapture && originalKey && originalWireKey) { - // A released SDK could stage either spelling. A modern approval for the - // fresh JSON can occupy the very same historical filename. - for (const pathKey of new Set([ - originalKey, - originalWireKey, - localeKey(original.parsed.findings), - ])) { - const bytes = await readOptionalFile(pendingFile(pathKey)); - if (bytes === undefined) continue; - const record = JSON.parse(bytes) as SavedSubmission | null; - if ( - (record?.legacyApproval && - (record.wireKey ?? pathKey) === originalWireKey) || - (record?.wireKey === undefined && - record?.inputProjection?.identityFormat !== 2 && - (pathKey === originalKey || - pathKey === originalWireKey || - original.parsed.report === undefined || - record?.inputProjection?.inputDigest === - jsonDigest(originalIdentity, canonicalJson))) - ) { - parsed = legacyParsed = original.parsed; - currentIdentity = legacyIdentity = original.identity; - historicalOnly = true; - break; - } - } - } - if (input.freshError && !historicalOnly) throw input.freshError; - const inputDigest = jsonDigest(currentIdentity); - const legacyInputDigest = - currentIdentity === legacyIdentity - ? inputDigest - : jsonDigest(legacyIdentity); function matchesSavedDestination(requests: FindingImportRequest[]): boolean { return ( requests.length > 0 && @@ -1930,194 +1147,9 @@ async function preparePublicationInput( batch.repository.id === destination.id && batch.repository.repo_connector_id === destination.repo_connector_id && - wireJson(batch.source) === wireJson(source), - ) - ); - } - const originalHostedRetry = - "This source has a pending hosted Cloud request. Resume it with the SDK build that created it and the original input before publishing another selection."; - const unknownHostedRetry = - "A pending hosted request matches this provider and repository, but its deployment could not be established. Resume it with the SDK build that created it, on the original deployment, using the full original input and metadata."; - function originalHostedPending( - content: SavedSubmission, - filenameKey: string, - ) { - const projection = content.inputProjection; - if ( - (options.provider !== "semgrep" && options.provider !== "snyk") || - content.accountId !== credentials.account_id || - !projection || - !content.requests?.length || - !content.requests.every( - ({ repository, source: savedSource }) => - repository.id === destination.id && - repository.repo_connector_id === destination.repo_connector_id && - canonicalJson(savedSource) === canonicalJson(source), + canonicalJson(batch.source) === canonicalJson(source), ) - ) - return false; - const findings = submissionFindings(content.requests); - if ( - projection.savedEvidenceDigest !== hash("sha256", canonicalJson(findings)) - ) - return false; - const projections = [findings]; - // Historical filenames include caller fields discarded by request validation. - // Reuse them by stable ID with the evidence and order of the approved request. - const metadata = new Map( - legacyParsed.findings - .filter((row) => - Object.keys(row).some( - (key) => key !== "evidence" && key !== "source_finding_id", - ), - ) - .map((row) => [row.source_finding_id, row]), - ); - if (findings.some((row) => metadata.has(row.source_finding_id))) - projections.push( - findings.map((row) => ({ - ...metadata.get(row.source_finding_id), - ...row, - })), - ); - const identities: unknown[] = [...projections]; - if (parsed.report !== undefined) { - const report = reportForJson(parsed.report); - identities.push( - ...projections.map((rows) => ({ findings: rows, report })), - ); - } - // Caller metadata on a normalized finding is part of the filename but not the - // request body. An exact retry still supplies that original identity. - if (filenameKey === scopeKey(legacyIdentity)) - identities.unshift(legacyIdentity); - const identity = identities.find( - (value) => filenameKey === scopeKey(value), - ); - const digest = (value: unknown) => hash("sha256", canonicalJson(value)); - const semgrep = options.provider === "semgrep"; - const isHosted = (row: (typeof findings)[number]) => - semgrep - ? isSemgrepPlatformEvidence(row, "semgrep") - : isSnykRestFinding(row); - const scannerId = (row: (typeof findings)[number]) => - isHosted({ - ...row, - evidence: { ...row.evidence, source_data: undefined }, - }); - const hasNativeEvidence = (row: (typeof findings)[number]) => { - const data = row.evidence.source_data; - // A discriminator alone cannot identify a native occurrence. - return ( - data && - isHosted({ - source_finding_id: "", - evidence: { - ...row.evidence, - source_data: { ...data, format: undefined }, - }, - }) - ); - }; - const references = hostedSourceReferences( - readerSources, - content.approvedRepositoryUrl ?? destination.url, - content.approvedRepositoryPlatform, ); - const occurrenceIds = (rows: typeof findings) => { - const ids = new Set(rows.map((row) => row.source_finding_id)); - let native = false; - let unresolved = false; - for (const row of rows) { - if (!hasNativeEvidence(row)) continue; - native = true; - try { - const id = semgrep - ? semgrepPlatformFindingBinding( - { ...row, source_finding_id: "" }, - "semgrep", - references, - undefined, - undefined, - undefined, - "cloud", - )?.nativeId - : snykRestCheckpointFindingId(row, references); - if (id) { - ids.add(id); - continue; - } - } catch { - // An earlier SDK may have accepted a partial native projection. - } - if (!scannerId(row)) unresolved = true; - } - return { ids, native, unresolved }; - }; - const previous = occurrenceIds(findings); - const requested = occurrenceIds(parsed.findings); - const hostedSelection = - projection.retainedHostedSelection || - scopedSemgrepReport || - previous.native || - requested.native; - // Submitted and native IDs share one namespace even when one side drops its raw evidence. - const couldOverlap = - (hostedSelection && - Array.from(requested.ids).some((id) => previous.ids.has(id))) || - (previous.native && - requested.native && - (previous.unresolved || requested.unresolved)); - if (content.approvedRepositoryUrl && !projection.retainedHostedSelection) { - if (!couldOverlap) return false; - if (identity) - return projection.inputDigest === digest(identity) && "verified"; - // The filename includes metadata omitted from the saved request. A reproducible - // original digest under another filename proves this is another deployment. - return ( - !identities.some((value) => digest(value) === projection.inputDigest) && - "unidentified" - ); - } - if (identity && projection.inputDigest !== digest(identity)) - return ( - (couldOverlap || !previous.native || !requested.native) && "verified" - ); - - if (!couldOverlap) return false; - if (!identity) { - // If the saved selections reconstruct an old input digest but not its scoped - // filename, the pending request belongs to another deployment. The older - // publisher included its repository comparison flags in the digest. - const repositories = new Set([destination.url]); - for (const { evidence } of findings) { - const data = evidence.source_data; - const project = data?.["project"]; - const retained = - (semgrep && isRecord(project) ? project["url"] : undefined) ?? - data?.["sourceRepositoryUrl"]; - if (typeof retained === "string") repositories.add(retained); - } - const paths = semgrep ? [null, true] : [false, true]; - const suffixes = semgrep ? [false] : [false, true]; - for (const candidate of identities) { - if (digest(candidate) === projection.inputDigest) return false; - for (const strict of paths) { - for (const suffix of suffixes) { - for (const repository of repositories) { - const url = hostedRepositoryUrlKey(repository, !strict, suffix); - if ( - digest([candidate, [url, strict, suffix]]) === - projection.inputDigest - ) - return false; - } - } - } - } - return "unidentified"; - } - return "verified"; } function wasReset(content: SavedSubmission) { return content.requests.some( @@ -2153,33 +1185,6 @@ async function preparePublicationInput( ), ); } - async function retireIfReset( - content: SavedSubmission, - retire: () => Promise, - ) { - if (wasReset(content)) { - if (hosted) await confirmReset(); - await retire(); - throw new CloudImportError( - 409, - "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", - ); - } - } - function retirePendingIfReset( - content: SavedSubmission, - keys: string[], - path: string, - remove = () => rm(path), - ) { - return retireIfReset(content, () => - underLocks(keys, async () => { - const bytes = await readOptionalFile(path); - if (bytes !== undefined && sameSubmission(JSON.parse(bytes), content)) - await remove(); - }), - ); - } function readSaved( content: SavedSubmission | null | undefined, ): SavedSubmission { @@ -2195,699 +1200,61 @@ async function preparePublicationInput( checkRetainedTrivyProvider(options.provider, finding); if (!matchesSavedDestination(content.requests)) throw new Error("Saved import destination does not match."); - if (source.provider === "trivy" && !content.trivyRepositoryUrl) + const actual = submissionFindings(content.requests); + checkFindingProvider({ ...parsed, findings: actual }, options.provider); + const expected = repositoryFindings(parsed, content.repositorySeverities); + if (canonicalJson(actual) !== canonicalJson(expected.map(wireFinding))) throw new CodexSecurityError( - "This build cannot verify the original repository for this pending Trivy import. Check its destination before resuming with the build that created it.", - ); - return content; - } - - function validateSaved( - record: SavedSubmission | null | undefined, - legacy = false, - filenameKey = key, - ): SavedSubmission { - const content = readSaved(record); - const saved = submissionFindings(content.requests); - const wire = wireJson(saved); - const projection = content.inputProjection; - const expected = repositoryFindings( - legacy ? legacyParsed : parsed, - projection?.repositorySeverities, - ); - if ( - projection && - (!sameFindingSelection(saved, expected) || - projection.inputDigest !== (legacy ? legacyInputDigest : inputDigest) || - projection.savedEvidenceDigest !== hash("sha256", wire)) - ) { - const status = originalHostedPending(content, filenameKey); - if (status === "unidentified") - throw new CodexSecurityError(unknownHostedRetry); - if (status) { - if (wasReset(content)) return content; - throw new CodexSecurityError(originalHostedRetry); - } - throw new Error( - "Saved import evidence does not match the selected input.", - ); - } - // Older file readers can have a different, explicitly bound projection. - // A corrected numeric checkpoint must carry the selected evidence verbatim. - const exactWire = - historicalOnly || hasBoxedScalars || content.wireKey !== undefined; - if ( - (!projection || exactWire) && - wire !== wireJson(exactWire ? expected.map(wireFinding) : expected) - ) - throw new Error( "Saved import evidence does not match the selected input.", ); + const scope = content.semgrepPartialScopeDigest; if ( - !legacy && - hasBoxedScalars && - projection?.identityFormat !== 2 && - !verifiableLegacyMetadata(originalParsed, true) - ) - throw unmatchedPreviousNumbers(); - checkFindingProvider({ ...parsed, findings: saved }, options.provider); - const approvedScope = content.inputProjection?.semgrepPartialScopeDigest; - if ( - approvedScope !== undefined && + scope !== undefined && !wasReset(content) && - approvedScope !== suppliedSemgrepScopeDigest() + scope !== suppliedSemgrepScopeDigest() ) throw new CodexSecurityError( "The supplied Semgrep report scope does not match the approved request.", ); return content; } - - // The original filename and digests identify the previous selection, but - // contained {} for every boxed number. Only its actual saved request can - // prove which numbers were approved. - function readPreviousNumbers( - record: SavedSubmission | null, - ): SavedSubmission | "different-wire" | undefined { - const content = readSaved(record); - const projection = content.inputProjection; - const saved = submissionFindings(content.requests); - const savedWire = wireJson(saved); - const wireDigest = hash("sha256", savedWire); - if (projection?.inputDigest !== jsonDigest(originalIdentity, canonicalJson)) - throw unmatchedPreviousNumbers(); - if (projection.identityFormat === 2) { - // A current writer may own the old filename for an ordinary object. - if (projection.savedEvidenceDigest === wireDigest) return undefined; - throw unmatchedPreviousNumbers(); - } - const selected = originalParsed.findings.map(wireFinding); - if (projection.savedEvidenceDigest !== jsonDigest(selected, canonicalJson)) - throw unmatchedPreviousNumbers(); - if (savedWire !== wireJson(selected)) { - // A genuinely empty object at this key is a separate ordinary selection. - if (wireDigest === projection.savedEvidenceDigest) return undefined; - return "different-wire"; - } - if ( - (changedCapture && !historicalOnly) || - !verifiableLegacyMetadata(originalParsed, false) - ) - throw unmatchedPreviousNumbers(); - checkFindingProvider({ ...parsed, findings: saved }, options.provider); - content.legacyApproval = true; - content.inputProjection = { - ...content.inputProjection, - identityFormat: 2, - inputDigest, - savedEvidenceDigest: wireDigest, - }; - return content; - } - function unmatchedPreviousNumbers() { - return new CodexSecurityError( - "The older saved import cannot be matched to the selected input. Its checkpoint was preserved without uploading.", - ); - } - const key = scopeKey(currentIdentity); - const stagingKey = originalKey ?? key; - const canonicalPath = pendingFile(key); - const lookupPath = join(state, `${key}.lookup.json`); - // A lookup only suggests a regular file to validate. A missing or stale hint - // leaves the same on-disk discovery available as before. - async function readOptionalRegularFile(path: string) { - try { - if ((await lstat(path)).isFile()) return await readFile(path, "utf8"); - } catch { - // Optional discovery must not veto an independently verified checkpoint. - } - } - async function lookupFileKey() { - const serialized = await readOptionalRegularFile(lookupPath); - return serialized?.trim().match(/^"([a-f0-9]{64})"$/u)?.[1]; - } - - const historicalKey = localeKey(legacyParsed.findings); - const legacyIdentityKey = scopeKey(legacyIdentity); - const legacyKeys = new Set([legacyIdentityKey, historicalKey]); - function underLocks(keys: string[], operation: () => Promise) { - return withImportLocks(state, keys, dependencies.signal, operation); - } - let recovered: SavedSubmission | undefined; - for (const legacyKey of legacyKeys) { - if (legacyKey === key || legacyKey === originalKey) continue; - const legacyPath = pendingFile(legacyKey); - const captured = await readOptionalFile(legacyPath); - if (captured !== undefined) { - // Preserve requests staged by the earlier locale-based filename format. - // Hold both publication locks while moving the original checkpoint. - await underLocks([key, legacyKey], async () => { - const current = await readOptionalFile(canonicalPath); - const serialized = await readOptionalFile(legacyPath); - let prior: SavedSubmission | undefined | null; - if (serialized !== undefined) { - try { - prior = JSON.parse(serialized) as SavedSubmission | null; - } catch (error) { - if (current === undefined || !(error instanceof SyntaxError)) - throw error; - } - } - if (prior?.wireKey !== undefined) return; - if (current !== undefined) { - recovered = validateSaved(JSON.parse(current)); - if (prior && sameSubmission(prior, recovered)) await rm(legacyPath); - return; - } - // A report-free approval at the locale-era filename can be a separate input. - if ( - legacyKey !== legacyIdentityKey && - prior?.inputProjection && - prior.inputProjection.inputDigest !== legacyInputDigest - ) { - const projection = prior.inputProjection; - const rows = legacyParsed.findings; - if ( - projection.identityFormat === 2 || - (projection.inputDigest === jsonDigest(rows) && - projection.savedEvidenceDigest === - jsonDigest(submissionFindings(prior.requests))) - ) - return; - // The released SDK hashed report-free rows as an array and boxed scores as {}. - const originalRows = original?.parsed.findings ?? rows; - if ( - hasBoxedScalars && - legacyParsed.report !== undefined && - legacyKey === scopeKey(originalRows, canonicalJson) && - projection.inputDigest === - jsonDigest(originalRows, canonicalJson) && - projection.savedEvidenceDigest === - jsonDigest(originalRows.map(wireFinding), canonicalJson) && - sameFindingSelection( - submissionFindings(readSaved(prior).requests), - rows, - ) - ) - return; - } - const content = validateSaved( - serialized === undefined ? JSON.parse(captured) : prior, - true, - legacyKey, - ); - if (!sameFindingSelection(legacyParsed.findings, parsed.findings)) - throw new CodexSecurityError( - "The saved import selects different findings from this input. Resume the original selection before publishing this selection.", - ); - await retireIfReset(content, async () => { - if (serialized !== undefined) await rm(legacyPath); - }); - const savedEvidenceDigest = - content.inputProjection?.savedEvidenceDigest ?? - jsonDigest(legacyParsed.findings); - const rebindProjection = - content.inputProjection !== undefined || - savedEvidenceDigest !== inputDigest; - if (rebindProjection) { - if (content.inputProjection?.identityFormat !== 2) - content.legacyApproval = true; - content.inputProjection = { - ...content.inputProjection, - identityFormat: 2, - inputDigest, - savedEvidenceDigest, - }; - } - recovered = content; - if (serialized === undefined) return; - await withSourceAdmission(async () => { - if (!rebindProjection) { - await rename(legacyPath, canonicalPath); - } else { - // Bind the new input projection to the original immutable evidence, - // so later inventory reordering still resumes these exact requests. - await writeAtomicJson(canonicalPath, content); - await rm(legacyPath); - } - }); - }); - } - } - - interface Checkpoint { - fileKey: string; - content: SavedSubmission; - historical: boolean; - checkedCanonical?: true; - } - // Some getter-backed batches cannot reproduce a released filename. Their IDs - // can stop a conflicting upload, but never authorize replay of a saved request. - const potentialIds = - input.potentialIds === undefined && changedCapture - ? original.parsed.findings.map((row) => row.source_finding_id) - : input.potentialIds; - const unprovenPrefixes = new Set(input.unprovenIdPrefixes); - const uncertainIds = new Set(input.uncertainFindingIds); - const uncertainMembership = - input.unprovenOccurrenceCount || - uncertainIds.size > 0 || - unprovenPrefixes.size > 0; - function uncertainId(id: string) { - return ( - uncertainIds.has(id) || - unprovenPrefixes.has(id.replace(/\/[a-f0-9]{64}$/u, "/")) - ); - } - const stableIds = uncertainMembership - ? ( - potentialIds ?? parsed.findings.map((row) => row.source_finding_id) - ).filter((id) => id === undefined || !uncertainId(id)) - : undefined; - const knownMembership = input.unprovenOccurrenceCount - ? new Set(stableIds) - : undefined; - const knownRows = new Map( - input.knownFindings?.map((row) => [ - row.source_finding_id, - wireFinding(row), - ]), - ); - let originalInputDigest: string | undefined; - const cannotCaptureOriginal = - !input.original && - (potentialIds === null || !!(potentialIds?.length || uncertainMembership)); - const possibleBufferExport = - options.provider === "github" && original && !changedCapture; - function potentialOriginal(record: SavedSubmission, fileKey: string) { - const projection = record.inputProjection; - if (record.accountId !== credentials.account_id) return false; - if (!projection) { - // Locale-era requests had no projection. Their scoped occurrence IDs - // can stop an opaque retry but cannot authorize replay. - if (!cannotCaptureOriginal || record.wireKey !== undefined) return false; - } else if ( - !/^[a-f0-9]{64}$/u.test(projection.inputDigest) || - !/^[a-f0-9]{64}$/u.test(projection.savedEvidenceDigest) || - (projection.identityFormat === 2 && !record.legacyApproval) - ) - return false; - try { - // Older evidence may fail today's limits. Scope and selection can still - // identify an outstanding approval and prevent a duplicate submission. - if (!matchesSavedDestination(record.requests)) return false; - const rows = submissionFindings(record.requests); - const savedWire = wireJson(rows); - const evidenceMatches = - projection && - projection.savedEvidenceDigest === hash("sha256", savedWire); - if (evidenceMatches) { - // Matching old digests reconstruct its scoped filename. A report from - // this caller is useful only when the saved digest independently binds it. - const reportFree = - projection.inputDigest === projection.savedEvidenceDigest; - const identity = reportFree - ? rows - : parsed.report !== undefined - ? { findings: rows, report: reportForJson(parsed.report) } - : undefined; - if ( - identity && - (reportFree || projection.inputDigest === jsonDigest(identity)) - ) { - if ( - record.wireKey - ? record.wireKey !== scopeKey(identity) - : fileKey !== scopeKey(identity, canonicalJson) && - fileKey !== localeKey(rows) - ) - return false; - if ( - possibleBufferExport && - projection.inputDigest !== inputDigest && - sameLegacyJson(identity, currentIdentity, previousBufferJson) - ) { - return true; - } - } - if ( - !record.wireKey && - reportFree && - original && - original.wire !== savedWire - ) - return false; - } - if (record.wireKey && !cannotCaptureOriginal) return false; - if ( - projection && - original && - fileKey !== stagingKey && - fileKey !== originalWireKey && - projection.inputDigest === - (originalInputDigest ??= jsonDigest( - original.identity, - canonicalJson, - )) && - fileKey !== localeKey(original.parsed.findings) && - projection.savedEvidenceDigest === - jsonDigest( - original.parsed.findings.map(wireFinding), - canonicalJson, - ) && - savedWire === wireJson(original.parsed.findings.map(wireFinding)) - ) - return false; - if (stableIds) { - // An opaque occurrence may have had a different old ID. Compare all - // definite members and count the remaining IDs without claiming them. - const certain = rows.filter( - (row) => !uncertainId(row.source_finding_id), - ); - const stableSaved = knownMembership - ? certain.filter((row) => knownMembership.has(row.source_finding_id)) - : certain; - if ( - certain.length - stableSaved.length > - (input.unprovenOccurrenceCount ?? 0) - ) - return false; - return ( - rows.length > 0 && - stableSaved.length === stableIds.length && - stableSaved.every((row, i) => { - const known = knownRows.get(row.source_finding_id); - return ( - row.source_finding_id === stableIds[i] && - (!known || wireJson(row) === wireJson(known)) - ); - }) - ); - } - const candidates = originalParsed.findings; - const ids = - potentialIds === undefined - ? parsed.findings.map((row) => row.source_finding_id) - : potentialIds; - if ( - ids && - (!ids.length || - rows.length !== ids.length || - !rows.every( - (row, i) => - ids[i] === undefined || row.source_finding_id === ids[i], - )) - ) - return false; - // An opaque serializer can add a report or replace every visible field. - // Only scoped, independently known occurrences can rule out its old request. - if (cannotCaptureOriginal || !projection) return true; - if ( - projection.inputDigest === projection.savedEvidenceDigest && - parsed.report !== undefined - ) - return false; - if (potentialIds === undefined && (hasBoxedScalars || evidenceMatches)) - return false; - return ( - !sameFindingSelection(rows, candidates) || - rows.every((row, i) => { - const { source_data: savedData, ...savedFields } = row.evidence; - const { source_data: selectedData, ...selectedFields } = - candidates[i]!.evidence; - if (wireJson(savedFields) !== wireJson(selectedFields)) return false; - // Once the original SDK discarded boxed positions, a plain JSON export - // alone cannot attribute a new numeric spelling to that saved request. - return sameLegacyJson( - savedData, - selectedData, - (value) => typeof value === "number", - ); - }) - ); - } catch { - return false; - } - } - async function selectedCheckpoint(knownFileKey?: string, preparing = false) { - const directory = await validatePreparedOutputDir( - state, - undefined, - true, - ).catch(nullIfMissingFile); - if (!directory) return undefined; - const known = new Set( - [key, originalKey, knownFileKey].filter((value) => value !== undefined), - ); - let unidentified = false; - let unprovenOriginal = false; - let incomplete = false; - let stagingPathOccupied = false; - async function readCandidate( - fileKey: string, - selected?: Checkpoint, - contents?: string, - ): Promise { - const path = pendingFile(fileKey); - const serialized = - contents ?? - (selected?.fileKey === key && !known.has(fileKey) - ? await readOptionalRegularFile(path) - : await readOptionalFile(path)) ?? - (preparing && fileKey === key ? capturedPending : undefined); - if (serialized === undefined) { - if (!known.has(fileKey)) incomplete = true; - return selected; - } - if (fileKey === stagingKey) stagingPathOccupied = true; - let record: SavedSubmission | null; - try { - record = JSON.parse(serialized) as SavedSubmission | null; - } catch (error) { - if (known.has(fileKey)) throw error; - incomplete = true; - return selected; - } - // A migrated checkpoint records the wire identity it belongs to even - // when a separate input hashes to its historical filename. - const tag = record?.wireKey; - const tagged = tag !== undefined; - const ownerKey = tagged ? tag : fileKey; - const historical = !tagged && fileKey === originalKey && fileKey !== key; - if (ownerKey !== key && !historical) { - if (hosted && record?.inputProjection) { - let status; - try { - status = originalHostedPending(record, ownerKey); - } catch { - // An unrelated or unreadable checkpoint cannot identify this source. - } - if (status === "unidentified") - throw new CodexSecurityError(unknownHostedRetry); - if (status) { - if (wasReset(record)) { - if (!preparing) - throw new CodexSecurityError( - "The pending request and this preview have different repository reset markers. Review the destination again.", - ); - await retirePendingIfReset(record, [fileKey], path); - } - throw new CodexSecurityError(originalHostedRetry); - } - } - if ( - record && - (!tagged || cannotCaptureOriginal || possibleBufferExport) && - potentialOriginal(record, fileKey) - ) - unprovenOriginal = true; - // Some early locale filenames carry no deployment identity. Never guess their target. - if ( - !tagged && - record && - !record.inputProjection && - !legacyKeys.has(fileKey) - ) { - try { - // A different locale cannot authorize replay, even when the - // original native capture proves these rows were selected. - if ( - !original || - wireJson(submissionFindings(readSaved(record).requests)) !== - wireJson(original.parsed.findings.map(wireFinding)) - ) - validateSaved(record, true); - unidentified = true; - } catch { - /* Unrelated input. */ - } - } - return selected; - } - const content = historical - ? readPreviousNumbers(record) - : validateSaved(record); - if (content === "different-wire") { - // Another spelling may already have a verified checkpoint for this wire value. - unprovenOriginal = true; - return selected; - } - if (!content) return selected; - if (selected) { - const conflict = new CodexSecurityError( - "Two saved imports match this input. Their original requests and receipts were preserved without uploading.", - ); - if (!preparing) throw conflict; - // The two unlocked reads may have straddled completion of the first request. - return await underLocks([key, selected.fileKey, fileKey], async () => { - const first = await readCandidate(selected.fileKey); - if (first) { - if (await readCandidate(fileKey)) throw conflict; - if (sameSubmission(first.content, selected.content)) return first; - } - return completedCheckpoint(selected); - }); - } - return { fileKey, content, historical }; - } - let selected: Checkpoint | undefined; - for (const fileKey of known) - selected = await readCandidate(fileKey, selected); - // A known file that disappeared can be checked against its completed receipt. - if (!selected && knownFileKey !== undefined && !hosted) return undefined; - const hint = await lookupFileKey(); - if (hint && !known.has(hint)) { - const contents = await readOptionalRegularFile(pendingFile(hint)); - if (contents !== undefined) { - selected = await readCandidate(hint, selected, contents); - known.add(hint); - } - } - // A released publisher could stage the canonical request while another SDK - // tagged its old filename but had not yet written the optional lookup. - if ( - selected && - !cannotCaptureOriginal && - !selected.historical && - (selected.fileKey !== key || - selected.content.canonicalChecked || - selected.content.inputProjection?.identityFormat === 2) - ) - return selected; - const entries = - (await readdir(state, { withFileTypes: true }).catch( - nullIfMissingFile, - )) ?? []; - for (const entry of entries) { - if (!entry.isFile() || !/^[a-f0-9]{64}\.pending\.json$/u.test(entry.name)) - continue; - const fileKey = entry.name.slice(0, -".pending.json".length); - if (!known.has(fileKey)) - selected = await readCandidate(fileKey, selected); - } - if (unprovenOriginal && (!selected || cannotCaptureOriginal)) - throw unmatchedPreviousNumbers(); - if (!selected && stagingPathOccupied) throw occupiedCheckpoint(); - if (!selected && unidentified) - throw new CodexSecurityError( - "An older pending import matches this input, but its checkpoint identity could not be established. Resume it using the original deployment and locale before publishing this selection. The saved request was preserved without uploading.", - ); - if (selected?.fileKey === key && !preparing && !incomplete) - selected.checkedCanonical = true; - return selected; - } - async function readCompletedPublication( - requests: FindingImportRequest[], - observedKey: string, - ) { - for (const resultKey of new Set([key, observedKey])) { - const serialized = await readOptionalFile( - join(state, `${resultKey}.result.json`), - ); - if (serialized === undefined) continue; - const completed = JSON.parse( - serialized, - ) as ExternalPublicationResult | null; - if ( - (completed?.status === "complete" || completed?.status === "partial") && - Array.isArray(completed.receipts) && - completed.receipts.length === requests.length && - completed.receipts.every( - (receipt, i) => receipt?.id === requests[i]!.request_id, - ) - ) - return { - receipts: completed.receipts.map((receipt, i) => - validatePublicationReceipt(receipt, requests[i]!), - ), - verified: completed.verified, - }; - } + async function readPending() { + const bytes = await readOptionalFile(pendingPath); + return bytes === undefined ? undefined : readSaved(JSON.parse(bytes)); } - async function completedCheckpoint(observed: Checkpoint) { + async function readCompletedPublication(requests: FindingImportRequest[]) { + const bytes = await readOptionalFile(join(state, key + ".result.json")); + if (bytes === undefined) return; + const result = JSON.parse(bytes) as ExternalPublicationResult; if ( - await readCompletedPublication( - observed.content.requests, - observed.fileKey, - ) + (result.status === "complete" || result.status === "partial") && + result.receipts.length === requests.length && + result.receipts.every((r, i) => r.id === requests[i]!.request_id) ) - return observed; - throw new CodexSecurityError( - "The saved publication changed while preparing this input. Review it again.", - ); - } - function occupiedCheckpoint() { - return new CodexSecurityError( - "Finish the other pending import at this checkpoint first, then review this input again.", - ); - } - let selected = await selectedCheckpoint(recovered ? key : undefined, true); - const fileKey = selected?.fileKey ?? (recovered ? key : stagingKey); - const pendingPath = pendingFile(fileKey); - async function rememberFile() { - if (fileKey !== key && (await lookupFileKey()) !== fileKey) - await writeAtomicJson(lookupPath, fileKey).catch(() => undefined); - } - async function removePending() { - await rm(pendingPath, { force: true }); - if (fileKey !== key && (await lookupFileKey()) === fileKey) - await rm(lookupPath, { force: true }).catch(() => undefined); - } - - const lockKeys = [key, fileKey, stagingKey, ...alternateReaders.keys()]; - async function underPublishLocks(operation: () => Promise) { - // An older publisher may already hold its lock before its checkpoint exists. - // Derive that key for actual writes, even after a checkpoint-free preview. - if (resolved.previousReaders && !previousFindings) { - previousFindings = resolved.previousReaders.map((read) => read()); - addPreviousReaders(previousFindings); - lockKeys.push(...alternateReaders.keys()); - } - return underLocks(lockKeys, operation); + return { + receipts: result.receipts.map((r, i) => + validatePublicationReceipt(r, requests[i]!), + ), + verified: result.verified, + }; } - if (selected?.historical) { - const observed = selected; - selected = await underLocks(lockKeys, async () => { - const current = await selectedCheckpoint(fileKey); - if ( - !current || - current.fileKey !== fileKey || - !sameSubmission(current.content, observed.content) - ) - return completedCheckpoint(observed); - if (current.historical) { - current.content.wireKey = key; - await writeAtomicJson(pendingPath, current.content); - await rememberFile(); + await validatePreparedOutputDir(state, undefined, true).catch( + nullIfMissingFile, + ); + const saved = await readPending(); + if (saved && wasReset(saved)) { + await underPublishLock(async () => { + const current = await readPending(); + if (current && sameSubmission(current, saved)) { + await confirmReset(); + await removePending(); } - return current; }); + throw new CloudImportError( + 409, + "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.", + ); } - const saved = selected?.content ?? recovered; - if (historicalOnly && !saved) throw unmatchedPreviousNumbers(); - if (!parsed.findings.length) throw new NoPublishableFindingsError(parsed); - if (saved) - await retirePendingIfReset(saved, lockKeys, pendingPath, removePending); if (saved?.inspectorProjectMapRequired && !inspectorProjectMap) throw new CodexSecurityError( "Supply the verified Inspector project binding again to resume.", @@ -2895,26 +1262,24 @@ async function preparePublicationInput( // Existing repository submissions keep their original request bodies and receipts. const privatePath = join(state, `${key}.private-import.json`); - const privateSaved = await readOptionalFile(privatePath); - if (resumingPreviousReader && !saved && privateSaved === undefined) - throw new CodexSecurityError( - "The saved publication changed. Run the command again to preview this report.", - ); + const readPrivate = async () => + parsePrivateCheckpoint(await readOptionalFile(privatePath)); + const privateSaved = await readPrivate(); const usePrivateImport = !saved && (canStartJob || privateSaved !== undefined); async function preparePrivateAttempt() { - const context = previewContext ?? (await privateClient.context()); + const context = await privateClient.context(); if (source.source_key.trim() !== source.source_key) throw new CodexSecurityError( "A private source key cannot have leading or trailing whitespace.", ); - const report = await selectPrivateReport( - parsed, + const report = selectPrivateReport( + parsed.findings, source.provider, - nativePath, context, - privateSaved, - resolved.requiresStructuredReport, + privateSaved?.contextVersion === context.context_version + ? privateSaved.format + : undefined, ); const binding = { format: report.format, @@ -2938,18 +1303,17 @@ async function preparePublicationInput( } // Use an earlier catalog URL only after the account, connector, repository, // inbox context and exact upload digest all match the immutable checkpoint. - const existingPrivate = - usePrivateImport && privateSaved !== undefined - ? await preparePrivateAttempt() - : undefined; + const privateAttempt = usePrivateImport + ? await preparePrivateAttempt() + : undefined; const previous = - existingPrivate && - readPrivateCheckpoint(privateSaved, existingPrivate.binding); + privateAttempt && + readPrivateCheckpoint(privateSaved, privateAttempt.binding); const retryOfJobId = unsuccessfulJob(previous?.receipt) && canStartJob && - existingPrivate?.context.can_upload && - acceptsJob(existingPrivate.context, existingPrivate.report.format) + privateAttempt?.context.can_upload && + acceptsJob(privateAttempt.context, privateAttempt.report.format) ? previous?.jobId : undefined; const resumed = previous !== undefined && retryOfJobId === undefined; @@ -2961,74 +1325,20 @@ async function preparePublicationInput( // The findings-only checkpoint key lets projected and native callers resume one immutable // request. Supplemental report claims are outside that digest and must still match its source. - const recheckOriginal = - options.provider === "semgrep" && - projection !== resolved && - saved?.approvedRepositoryPlatform === "other"; - // Native Semgrep and OpenGrep files include their required source URL in every - // saved finding. An exact replay by repository ID survives a catalog rename. - const recordedFileSource = - saved && - !saved.approvedRepositoryUrl && - options.provider === "semgrep" && - !deferred && - declaredUrl !== undefined && - saved.requests.every(({ items }) => - items.every( - ({ evidence }) => evidence.details?.repository?.url === declaredUrl, - ), - ); - const declaredSource = - recheckOriginal || - readerSources.length > 0 || - (declaredUrl !== undefined && !recordedFileSource) || - report.sourceRepositoryUrl !== undefined || - report.sourceRepositoryPlatform !== undefined || - report.snykRestContext !== undefined; - let legacyTypedUrl: string | undefined; - let suppliedReport = report; - const sourceFor = (url: string, platform?: "other") => { - try { - return hostedRepositoryMatcher( - platform - ? { ...suppliedReport, sourceRepositoryPlatform: platform } - : suppliedReport, - options.provider, - declaredUrl, - { url }, - readerSources, - saved !== undefined, - ); - } catch (error) { - if (!recheckOriginal || readerSources.length) throw error; - // An older strict approval did not distinguish GitLab from another case-sensitive SCM. - throw new CodexSecurityError( - `${error instanceof Error ? error.message : String(error)} Resume with the original Semgrep input and SCM repository mapping, or the SDK build that created this request.`, - { cause: error }, - ); - } - }; - if (hosted && saved && declaredSource) { - // The released publisher checked typed repository URLs against its authenticated catalog. - // Opaque original records could name a different project and cannot establish a rename. - const findings = submissionFindings(saved.requests); - if (!saved.approvedRepositoryUrl) - legacyTypedUrl = findings.find( - ({ evidence }) => evidence.details?.repository?.url, - )?.evidence.details?.repository?.url; - // Supplemental claims must agree with each verifiable row from the immutable request. - suppliedReport = { - ...report, - // The v1 upgrade may drop a redundant mapping using the catalog's GitHub casing. - findings: recheckOriginal ? [...report.findings, ...findings] : findings, - }; - } - // The released request retained its typed finding URL, but only the authenticated - // catalog or a previously recorded approval can bind a new source declaration. const approvedUrl = - saved?.approvedRepositoryUrl ?? privateUrl ?? destination.url; + saved?.approvedRepositoryUrl ?? + saved?.trivyRepositoryUrl ?? + privateUrl ?? + destination.url; + const sourceFor = (url: string, platform?: "other") => + hostedRepositoryMatcher( + platform ? { ...report, sourceRepositoryPlatform: platform } : report, + options.provider, + declaredUrl, + { url }, + readerSources, + ); if ( - hosted && readerSources.some(({ platform, url }) => { const source = normalizeHostedGitRepositoryUrl( url, @@ -3044,11 +1354,10 @@ async function preparePublicationInput( throw new CodexSecurityError( "The reader's source repository does not match the selected Cloud repository.", ); - await recheckFileRepository?.(approvedUrl, approvedPlatform); - const hostedSource = - hosted && (!saved || declaredSource) - ? sourceFor(approvedUrl, approvedPlatform) - : undefined; + await resolved.recheckFileRepository?.(approvedUrl, approvedPlatform); + const hostedSource = hosted + ? sourceFor(approvedUrl, approvedPlatform) + : undefined; const matchesHostedRepository = hostedSource?.matches; function requireApprovedSource(url: string, platform?: "other") { const binding = @@ -3061,7 +1370,6 @@ async function preparePublicationInput( ); } requireApprovedSource(approvedUrl); - if (legacyTypedUrl) requireApprovedSource(legacyTypedUrl); const trivyTarget = saved?.trivyRepositoryUrl ?? privateUrl ?? destination.url; @@ -3074,12 +1382,7 @@ async function preparePublicationInput( ); } } - if (trivyReportUrls) { - for (const url of trivyReportUrls) requireTrivyRemote(url); - if (parsed !== resolved.current) - for (const url of trivyBatchSourceUrls(parsed, true)) - requireTrivyRemote(url); - } + for (const url of trivyReportUrls ?? []) requireTrivyRemote(url); function checkRepositories( { findings, report }: Pick, @@ -3256,54 +1559,10 @@ async function preparePublicationInput( ); } } - if (!usePrivateImport) checkRepositories(parsed); - // A retry sends its saved projection, which may differ from today's reader. - if (saved) { - // Only use the report associated with the input digest in the saved checkpoint. - const report = - saved.inputProjection?.inputDigest === inputDigest - ? parsed.report - : undefined; - checkRepositories({ findings: submissionFindings(saved.requests), report }); - } - - // An older, genuinely unbound checkpoint first acquires a URL on a matching preview. - // Retain it at the original path so its bare caller can still recover the same request. - if (saved && !saved.approvedRepositoryUrl && hostedSource) { - await underLocks(lockKeys, async () => { - const serialized = await readOptionalFile(pendingPath); - if (serialized === undefined) return; - const current = validateSaved(JSON.parse(serialized)); - if (!sameSubmission(current, saved)) - throw new CodexSecurityError( - "Another publication prepared this input. Review the saved request again.", - ); - if (current.approvedRepositoryUrl) { - requireApprovedSource( - current.approvedRepositoryUrl, - current.approvedRepositoryPlatform, - ); - } else { - // Adding the catalog URL cannot erase an older SDK's same-selection hold. - if ( - current.inputProjection && - originalHostedPending(current, current.wireKey ?? fileKey) === - "verified" - ) - current.inputProjection.retainedHostedSelection = true; - current.approvedRepositoryUrl = approvedUrl; - await writeAtomicJson(pendingPath, current); - } - saved.inputProjection = current.inputProjection; - saved.approvedRepositoryUrl = current.approvedRepositoryUrl; - saved.approvedRepositoryPlatform = current.approvedRepositoryPlatform; - }); - } - - if (usePrivateImport) { + if (!privateAttempt) checkRepositories(parsed); + if (privateAttempt) { const client = privateClient; - const { context, report, binding } = - existingPrivate ?? (await preparePrivateAttempt()); + const { context, report, binding } = privateAttempt; function requireAccess(value: typeof context, saved: typeof previous) { if ( !value.can_read || @@ -3338,22 +1597,14 @@ async function preparePublicationInput( return { preview: structuredClone(preview), publish: () => - underPublishLocks(async () => { + underPublishLock(async () => { const current = await client.context(); if (current.context_version !== binding.contextVersion) throw new CodexSecurityError( "The private inbox changed. Run the command again to review the new destination.", ); - await assertNoAlternateCheckpoint(async () => current); - const stored = readPrivateCheckpoint( - await readOptionalFile(privatePath), - binding, - ); - if ( - !stored && - (resumingPreviousReader || - (privateUrl && privateUrl !== destination.url)) - ) + const stored = readPrivateCheckpoint(await readPrivate(), binding); + if (previous && !stored) throw new CodexSecurityError( "The saved private import changed. Run the command again to preview this report.", ); @@ -3361,28 +1612,23 @@ async function preparePublicationInput( const retrying = retryOfJobId !== undefined && stored?.receipt?.id === retryOfJobId; const active = retrying ? undefined : stored; - if (resumed && !active) checkRepositories(parsed); requireAccess(current, active); - if (await selectedCheckpoint()) + if (await readPending()) throw new CodexSecurityError( "A repository submission is pending. Review its saved request before continuing.", ); - // Both routes share request IDs. A server upgrade can promote an - // existing job, and its saved route also survives gate rollback. - const checkpoint = { - ...(active ?? { - ...binding, - approvedRepositoryUrl: destination.url, - approvedRepositoryPlatform: - hostedSource?.github === false ? ("other" as const) : undefined, - uploadRequestId: randomUUID(), - jobRequestId: randomUUID(), - fileId: undefined, - jobId: undefined, - receipt: undefined, - previousReceipt: retrying ? stored?.receipt : undefined, - }), - jobApiVersion: stored?.jobApiVersion ?? jobApiVersion, + const checkpoint = active ?? { + ...binding, + approvedRepositoryUrl: destination.url, + approvedRepositoryPlatform: + hostedSource?.github === false ? ("other" as const) : undefined, + uploadRequestId: randomUUID(), + jobRequestId: randomUUID(), + jobApiVersion: retrying ? stored?.jobApiVersion : jobApiVersion, + fileId: undefined, + jobId: undefined, + receipt: undefined, + previousReceipt: retrying ? stored?.receipt : undefined, }; const result: ExternalPublicationResult = { cloudApiUrl: preview.cloudApiUrl, @@ -3424,8 +1670,7 @@ async function preparePublicationInput( const savedReceipt = checkpoint.receipt ?? checkpoint.previousReceipt; if (savedReceipt) acknowledge(savedReceipt); - if (!active || active.jobApiVersion !== checkpoint.jobApiVersion) - await writeAtomicJson(privatePath, checkpoint); + if (!active) await writeAtomicJson(privatePath, checkpoint); if (!checkpoint.fileId) { progress({ phase: "uploading", completed: 0 }); checkpoint.fileId = await client.upload({ @@ -3470,10 +1715,12 @@ async function preparePublicationInput( } // The original selection and request bytes govern a retry even if discovery changes. - const severities = saved - ? saved.inputProjection?.repositorySeverities - : repositorySeverities; - const direct = repositorySelection(parsed, severities, input.sourcePositions); + const severities = saved ? saved.repositorySeverities : repositorySeverities; + const direct = repositorySelection( + parsed, + severities, + resolved.sourcePositions, + ); if (!direct.findings.length) throw new NoPublishableFindingsError(direct); let requests = saved?.requests; if (!requests) { @@ -3581,25 +1828,12 @@ async function preparePublicationInput( ...(trivyReportUrls && { trivyRepositoryUrl: trivyTarget }), requests, receipts: saved?.receipts, - legacyApproval: saved?.legacyApproval, - wireKey: saved?.wireKey ?? (fileKey === key ? undefined : key), - inputProjection: saved?.inputProjection ?? { - ...(direct !== parsed && { repositorySeverities: severities }), - identityFormat: saved ? undefined : 2, - ...(scopedSemgrepReport && { retainedHostedSelection: true }), - ...(bindSemgrepScope && { - semgrepPartialScopeDigest: suppliedSemgrepScopeDigest(), - }), - inputDigest, - savedEvidenceDigest: jsonDigest(submissionFindings(requests)), - }, + repositorySeverities: severities, + semgrepPartialScopeDigest: bindSemgrepScope + ? suppliedSemgrepScopeDigest() + : saved?.semgrepPartialScopeDigest, + approvedRepositoryUrl: hosted ? approvedUrl : undefined, }; - if ( - hosted && - (!saved || saved.approvedRepositoryUrl || matchesHostedRepository) - ) - submission.approvedRepositoryUrl = - saved?.approvedRepositoryUrl ?? approvedUrl; if ( saved?.approvedRepositoryPlatform || (hostedSource && !hostedSource.github) @@ -3690,17 +1924,13 @@ async function preparePublicationInput( ); } let acquired = false; - return await underPublishLocks(async () => { + return await underPublishLock(async () => { acquired = true; - await assertNoAlternateCheckpoint(); // Re-read under the lock to keep checkpoints from another publisher. - const selected = await selectedCheckpoint( - saved ? fileKey : undefined, - ).catch((error: unknown) => { + let pending = await readPending().catch((error: unknown) => { if (!submission.receipts?.length) throw error; throw interruptedPreview(errorMessage(error), error); }); - let pending = selected?.content; const current = pending ?? submission; const adoptProjectMap = submission.inspectorProjectMapRequired && @@ -3711,28 +1941,21 @@ async function preparePublicationInput( current.approvedRepositoryUrl, current.approvedRepositoryPlatform, ); - if ( - !selected || - selected.fileKey !== fileKey || - selected.historical || - !sameSubmission(current, submission) - ) { - const message = selected + if (!pending || !sameSubmission(current, submission)) { + const message = pending ? "Another publication prepared this input. Run the command again to review and resume that saved request." : "The saved publication changed after preview. Review its result before publishing again."; if (saved) { - const completed = await readCompletedPublication(requests, fileKey); + const completed = await readCompletedPublication(requests); if (completed) return result(completed.receipts, completed.verified); throw interruptedPreview(message); } - if (selected) throw new CodexSecurityError(message); + if (pending) throw new CodexSecurityError(message); if ((await readOptionalFile(privatePath)) !== undefined) throw new CodexSecurityError( "A private import already owns this selection. Review its saved job before continuing.", ); - if ((await readOptionalFile(pendingPath)) !== undefined) - throw occupiedCheckpoint(); } // A preview can pin the catalog URL. Tighten its SCM comparison only // once the caller confirms publication, including when resuming a request. @@ -3748,39 +1971,10 @@ async function preparePublicationInput( const acknowledged: FindingImportReceipt[] = []; let verified = 0; try { - if (selected?.checkedCanonical) current.canonicalChecked = true; - if (pending === undefined) { - await withSourceAdmission(async () => { - if (hosted) { - for (const otherKey of new Set([ - ...projectionKeys, - ...legacyKeys, - ])) { - if ( - otherKey !== key && - otherKey !== fileKey && - (await readOptionalFile(pendingFile(otherKey))) !== - undefined - ) - throw new CodexSecurityError( - "Another publication prepared this input. Run the command again to review and resume that saved request.", - ); - } - if (await selectedCheckpoint()) - throw new CodexSecurityError( - "Another publication prepared this input. Run the command again to review and resume that saved request.", - ); - } - await writeAtomicJson(pendingPath, current); - }); - pending = current; - } else if (adoptProjectMap || adoptStrictRepository) { + if (!pending || adoptProjectMap || adoptStrictRepository) { await writeAtomicJson(pendingPath, current); - } else if (selected?.checkedCanonical) { - // The existing approval and receipts are usable if caching the scan fails. - await writeAtomicJson(pendingPath, current).catch(() => undefined); + pending = current; } - await rememberFile(); progress({ phase: "uploading", completed: 0, @@ -3804,8 +1998,7 @@ async function preparePublicationInput( // Acknowledged batches must not consume another POST quota on retry. // Readback can still resume after this checkpoint without replaying writes. await writeAtomicJson(pendingPath, { - ...submission, - canonicalChecked: current.canonicalChecked, + ...current, receipts: acknowledged, }); } @@ -3850,7 +2043,7 @@ async function preparePublicationInput( report.repo_id !== destination.id || report.repo_connector_id !== destination.repo_connector_id || report.environment_id !== batch.repository.environment_id || - wireJson(report.source) !== wireJson(source) + canonicalJson(report.source) !== canonicalJson(source) ) throw new CodexSecurityError( "Cloud readback did not match the saved finding identity.", diff --git a/sdk/typescript/src/external-findings-read.ts b/sdk/typescript/src/external-findings-read.ts index 1bc0fb8a8b..5fcb5b316d 100644 --- a/sdk/typescript/src/external-findings-read.ts +++ b/sdk/typescript/src/external-findings-read.ts @@ -1,7 +1,6 @@ import type { ExternalFinding, ExternalFindings, - ExternalPublicationInput, ExternalSourceRepository, } from "./external-findings.js"; import { readScannerJson, sarifRunRepositories } from "./scanner-import.js"; @@ -12,7 +11,6 @@ import type { ExternalImportTarget } from "./external-import-contract.js"; import { dependabotRepository } from "./github-dependabot-findings.js"; import { normalizeSnykFile, - readSnykReport, emptySnykReportRepositories, snykRepositoryUrls, } from "./snyk-findings.js"; @@ -22,44 +20,21 @@ import { readSemgrepReport, } from "./semgrep-findings.js"; import { semgrepPlatformReportRepository } from "./semgrep-platform-findings.js"; -import { - normalizeTrivyFindings, - needsPreviousTrivyProjection, - normalizePreviousTrivyFindings, -} from "./trivy-findings.js"; +import { normalizeTrivyFindings } from "./trivy-findings.js"; import { readOpenGrepFindings } from "./opengrep-findings.js"; import { normalizeEndorFindings, - normalizePreviousEndorFindings, endorRepositoryUrls, } from "./endor-findings.js"; import { readGitHubFindings } from "./github-findings.js"; import { readSonarFindings } from "./sonar-findings.js"; -import { - normalizeVeracodeFindings, - normalizePreviousVeracodeFindings, -} from "./veracode-findings.js"; -import { - needsPreviousBlackDuckProjection, - normalizeBlackDuckFindings, - normalizePreviousBlackDuckFindings, -} from "./blackduck-findings.js"; -import { - normalizeContinuumFindings, - normalizePreviousContinuumFindings, -} from "./continuum-findings.js"; +import { normalizeVeracodeFindings } from "./veracode-findings.js"; +import { normalizeBlackDuckFindings } from "./blackduck-findings.js"; +import { normalizeContinuumFindings } from "./continuum-findings.js"; import { readCheckmarxFindings } from "./checkmarx-findings.js"; -import { - normalizeInspectorFindings, - normalizePreviousInspectorFindings, -} from "./inspector-findings.js"; -import { hasPreviousCloudProjection } from "./reader-selection.js"; +import { normalizeInspectorFindings } from "./inspector-findings.js"; import { isRecord } from "./record.js"; -import { - readLocalWizFindings, - readVendorFindingsForPublication, - wizFindingRepository, -} from "./wiz-findings.js"; +import { readWizSelection, wizFindingRepository } from "./wiz-findings.js"; export type LocalFindingProvider = Provider | "tenable"; @@ -108,12 +83,9 @@ export interface ExternalFindingsReaderOptions { trivyScanRoot?: string; } -/** @internal Shared parser output, including immutable Cloud retry projections. */ +/** @internal Normalized file contents and any repository claims supplied by its reader. */ export interface ReadExternalFindings { findings: ExternalFindings; - previousFindings?: () => ExternalFindings; - /** Preserve previous Wiz projections when resuming immutable Cloud requests. */ - publication?: ExternalPublicationInput; sourceRepositories?: ExternalSourceRepository[]; } @@ -209,18 +181,6 @@ export async function readExternalFindings( ): Promise { const { provider } = options; const url = options.sourceRepositoryUrl ?? ""; - // Both projections describe the same captured rows, including exclusions. - function withReplay( - findings: ExternalFindings, - couldDiffer: (findings: ExternalFindings) => boolean, - previous: () => ExternalFindings, - ): ReadExternalFindings { - return { - findings, - previousFindings: - target === "cloud" && couldDiffer(findings) ? previous : undefined, - }; - } if (provider === "tenable") { if (target === "cloud") requireCloudReader(options); return { findings: await readTenableFindings(path, url) }; @@ -241,7 +201,7 @@ export async function readExternalFindings( return { findings, sourceRepositories: [{ url }] }; } if (provider === "snyk") { - const input = await readSnykReport(path); + const input = await readScannerJson(path); const findings = await normalizeSnykFile(input, target, references); return { findings, sourceRepositories: snykFileRepositories(input) }; } @@ -264,34 +224,24 @@ export async function readExternalFindings( if (provider === "github") return { findings: await readGitHubFindings(path, target) }; if (provider === "wiz") { - const wiz = - target === "local" - ? await readLocalWizFindings(path) - : await readVendorFindingsForPublication(path, target); - const sourceRepositories = wiz.current.findings.flatMap((finding) => + const wiz = await readWizSelection(path, target); + const sourceRepositories = wiz.findings.findings.flatMap((finding) => localFindingRepositories("wiz", finding), ); return { - findings: wiz.current, - ...("legacy" in wiz && { publication: wiz }), + findings: wiz.findings, sourceRepositories: [...wiz.sourceRepositories, ...sourceRepositories], }; } const input = await readScannerJson(path); - if (provider === "trivy") { - const args = [input, target, options.trivyScanRoot] as const; - return withReplay( - normalizeTrivyFindings(...args), - needsPreviousTrivyProjection, - () => normalizePreviousTrivyFindings(...args), - ); - } + if (provider === "trivy") + return { + findings: normalizeTrivyFindings(input, target, options.trivyScanRoot), + }; if (provider === "endor") { const findings = normalizeEndorFindings(input, target); return { - ...withReplay(findings, hasPreviousCloudProjection, () => - normalizePreviousEndorFindings(input, target), - ), + findings, sourceRepositories: sarifFileRepositories(input, "Endor") ?? findings.findings.flatMap((row) => @@ -299,33 +249,14 @@ export async function readExternalFindings( ), }; } - const [normalize, previous, couldDiffer] = ( - { - inspector: [ - normalizeInspectorFindings, - normalizePreviousInspectorFindings, - () => true, - ], - continuum: [ - normalizeContinuumFindings, - normalizePreviousContinuumFindings, - () => !Array.isArray(input), - ], - blackduck: [ - normalizeBlackDuckFindings, - normalizePreviousBlackDuckFindings, - needsPreviousBlackDuckProjection, - ], - veracode: [ - normalizeVeracodeFindings, - normalizePreviousVeracodeFindings, - hasPreviousCloudProjection, - ], - } as const - )[provider]; - const args = [input, url, target, options.inspectorProjectMap] as const; + const normalize = { + inspector: normalizeInspectorFindings, + continuum: normalizeContinuumFindings, + blackduck: normalizeBlackDuckFindings, + veracode: normalizeVeracodeFindings, + }[provider]; return { - ...withReplay(normalize(...args), couldDiffer, () => previous(...args)), + findings: normalize(input, url, target, options.inspectorProjectMap), sourceRepositories: [{ url }], }; } diff --git a/sdk/typescript/src/external-findings.ts b/sdk/typescript/src/external-findings.ts index eb84108653..688d7b0b2c 100644 --- a/sdk/typescript/src/external-findings.ts +++ b/sdk/typescript/src/external-findings.ts @@ -122,6 +122,7 @@ export function snapshotNormalizedFindings>( input: T, checkEvidenceNumber = (value: number) => requireJsonNumber(value, "Evidence"), target: ExternalImportTarget = "local", + provider?: string, ): T { const fields = new Map(); const errors: { path: string[]; error: Error }[] = []; @@ -195,6 +196,19 @@ export function snapshotNormalizedFindings>( ); parent[key] = error; } + // Reader scope can live on a getter. Do not evaluate enumerable getters twice. + if (provider !== undefined) { + const keys = ["sourceFormat", "semgrepPlatformScope", "snykRestContext"]; + if (provider === "semgrep" || provider === "snyk") + keys.push("sourceRepositoryUrl", "sourceRepositoryPlatform"); + for (const key of keys) + if ( + !(key in result) && + key in input && + !Object.prototype.propertyIsEnumerable.call(input, key) + ) + result[key] = structuredClone(Reflect.get(input, key)); + } return result; } @@ -207,14 +221,6 @@ export function cloneImportPreview(input: T): T { }; } -/** @internal Keep earlier provider projections only for immutable request replay. */ -export interface ExternalPublicationInput { - current: ExternalFindings; - legacy: ExternalFindings; - currentIdentity: unknown; - legacyIdentity: unknown; -} - /** Capture the caller's import settings before reading an export or resolving its destination. */ export function snapshotImportOptions< P extends LocalExternalImportOptions["provider"], diff --git a/sdk/typescript/src/external-import-contract.ts b/sdk/typescript/src/external-import-contract.ts index f20cf8d93a..902c37f088 100644 --- a/sdk/typescript/src/external-import-contract.ts +++ b/sdk/typescript/src/external-import-contract.ts @@ -1,5 +1,5 @@ import { isIP } from "node:net"; -import { isNumberObject, isProxy } from "node:util/types"; +import { isNumberObject } from "node:util/types"; import Ajv2020, { type ValidateFunction } from "ajv/dist/2020.js"; import schema from "../schemas/external-findings.schema.json" with { type: "json" }; import type { @@ -155,8 +155,7 @@ function normalizeEvidenceDetails( ): void { if (value !== null && typeof value === "object") { const fields = value as Record; - // Cloud omits these new optional fields when null. Preserve every legacy - // default so saved Wiz requests still replay the same evidence bytes. + // Cloud omits these optional fields when null. if (Array.isArray(fields["packages"])) for (const value of fields["packages"]) { if ( @@ -365,16 +364,6 @@ export function validateExternalEvidenceFields( return result; } -// Bind changed admission to the validated row or the error returned to its reader. -const previousAdmissions = new WeakMap< - object, - ExternalFindingEvidence | null | "uncertain" ->(); - -export function previousAdmission(value: object) { - return previousAdmissions.get(value); -} - /** Keep a valid native record when its optional structured fields exceed the target's bounds. */ export function validateEnrichedExternalEvidence( evidence: Record, @@ -397,18 +386,7 @@ export function validateExternalEvidence( target: ExternalImportTarget = "cloud", ): ExternalFindingEvidence { const result = validateExternalEvidenceFields(input, target); - let changed = false; - try { - validateExternalEvidenceJson(result, target, 0, (previous, current) => { - changed = - previous > CLOUD_EVIDENCE_BYTES !== current > CLOUD_EVIDENCE_BYTES; - }); - } catch (error) { - if (changed && error instanceof Error) - previousAdmissions.set(error, result); - throw error; - } - if (changed) previousAdmissions.set(result, null); + validateExternalEvidenceJson(result, target); return result; } @@ -497,87 +475,32 @@ function serializeImportJson( }); } -function* dataValues(value: object, skipped?: () => void): Generator { - if (isProxy(value)) { - skipped?.(); - return; - } - const keys = Object.keys(value); - // Array indices precede named keys. Dense arrays retain every index in a native clone. - if ( - Array.isArray(value) && - value.length && - keys[value.length - 1] !== String(value.length - 1) && - Object.getOwnPropertyNames(value).some( - (key) => - /^(?:0|[1-9]\d*)$/u.test(key) && - Number(key) < value.length && - !Object.prototype.propertyIsEnumerable.call(value, key), - ) - ) { - // The released mapper could read non-enumerable indices before cloning them away. - skipped?.(); - } - for (const key of keys) { - const field = Object.getOwnPropertyDescriptor(value, key); - if (field && Object.hasOwn(field, "value")) yield field.value; - else skipped?.(); - } -} - -// Validate original numeric fields even if a serializer omits them. Object.values -// reads siblings before descending, as the released provider reader did. -function rawJsonNumbers( +/** Enforce the numeric and depth limits on raw evidence as well as its serialized form. */ +function validateRawEvidence( input: unknown, target: ExternalImportTarget, - oldSize = false, - dataOnly = false, - skipped?: () => void, -): number { - let adjustment = 0; - function countNumber(value: number) { - evidenceNumber(value); - if (target === "cloud") adjustment += cloudNumberByteAdjustment(value); - } - function exceedDepth() { - if (!oldSize) - throw new Error("Evidence exceeds the Cloud JSON depth limit of 20."); - adjustment = Infinity; - } - - if (input === null || typeof input !== "object") { - if (typeof input === "number") countNumber(input); - return adjustment; - } - const pending: object[] = [input]; - const depths = [0]; +): void { + if (typeof input === "number") evidenceNumber(input); + if (input === null || typeof input !== "object") return; + const checked = target === "local" ? new WeakSet() : undefined; const ancestors: object[] = []; - let checked = target === "local" ? new WeakSet() : undefined; + const pending = [{ value: input, depth: 0 }]; while (pending.length) { - const value = pending.pop()!; - const depth = depths.pop()!; - if (target === "cloud" && depth > 20) exceedDepth(); - // After a historical rejection, aliases only need one raw numeric check. - if (adjustment === Infinity) checked ??= new WeakSet(); + const { value, depth } = pending.pop()!; if (checked?.has(value)) continue; checked?.add(value); ancestors.length = depth; - if (ancestors.includes(value)) { - if (oldSize) adjustment = Infinity; - continue; - } + if (ancestors.includes(value)) continue; ancestors[depth] = value; - for (const child of dataOnly - ? dataValues(value, skipped) - : Object.values(value)) { - if (typeof child === "number") countNumber(child); + for (const child of Object.values(value)) { + if (target === "cloud" && depth >= 20) + throw new CloudEvidenceDepthError(); + if (typeof child === "number") evidenceNumber(child); if (child !== null && typeof child === "object") { - pending.push(child); - depths.push(depth + 1); - } else if (target === "cloud" && depth >= 20) exceedDepth(); + pending.push({ value: child, depth: depth + 1 }); + } } } - return adjustment; } const CLOUD_EVIDENCE_BYTES = 256 * 1024; @@ -592,61 +515,23 @@ export function validateExternalEvidenceJson( input: unknown, target: ExternalImportTarget, additionalCloudBytes = 0, - observe?: (previousBytes: number, currentBytes: number) => void, ): number { let adjustment = 0; const validateNumber = (value: unknown) => { - try { - const number = checkEmittedJsonNumber(value, evidenceNumber); - if (target === "cloud" && number !== undefined) - adjustment += cloudNumberByteAdjustment(number); - } catch (error) { - // The released walk checked primitive numeric fields, not JSON-emitted wrappers. - if (error instanceof Error) previousAdmissions.set(error, "uncertain"); - throw error; - } + const number = checkEmittedJsonNumber(value, evidenceNumber); + if (target === "cloud" && number !== undefined) + adjustment += cloudNumberByteAdjustment(number); }; - // Keep local serialization before the raw walk. If an emitted number is invalid, - // ordinary invalid primitive fields still give a definitive historical exclusion. if (target === "local") { - let invalid: Error | undefined; - try { - serializeImportJson(input, validateNumber); - } catch (error) { - if (!(error instanceof Error) || previousAdmission(error) !== "uncertain") - throw error; - invalid = error; - } - rawJsonNumbers(input, target); - if (invalid) throw invalid; + serializeImportJson(input, validateNumber); + validateRawEvidence(input, target); return 0; } - const rawAdjustment = rawJsonNumbers(input, target); - let text: string; - try { - text = serializeImportJson(input, validateNumber, true); - } catch (error) { - // A serializer can introduce levels that were absent from the raw evidence. - if (error instanceof CloudEvidenceDepthError) - previousAdmissions.set(error, "uncertain"); - throw error; - } - const bytes = Buffer.byteLength(text) + additionalCloudBytes; - observe?.(bytes + rawAdjustment, bytes + adjustment); - return checkEvidenceSize(bytes + adjustment); -} - -/** Reproduce the released SDK's checks on a detached native finding. */ -export function validatePreviousCloudEvidence( - input: unknown, -): ExternalFindingEvidence { - const evidence = validateExternalEvidenceFields(input, "cloud"); - // Older validation counted enumerable primitive numbers, while JSON encoded - // the native wrappers. Check depth before serializing a possibly shared graph. - const adjustment = rawJsonNumbers(evidence, "cloud", true, true); - if (!Number.isFinite(adjustment)) throw new CloudEvidenceDepthError(); - checkEvidenceSize(Buffer.byteLength(JSON.stringify(evidence)) + adjustment); - return evidence; + validateRawEvidence(input, target); + const text = serializeImportJson(input, validateNumber, true); + return checkEvidenceSize( + Buffer.byteLength(text) + additionalCloudBytes + adjustment, + ); } /** Use the array index as toJSON's key, as when saving the supplied GitHub records. */ @@ -654,27 +539,7 @@ export function exportedGitHubRecord( value: unknown, index: number, target: ExternalImportTarget, - checkPreviousLocal = false, ) { - // The released readers validated numeric fields hidden by serializers. - let adjustment = 0; - let previousInvalid: unknown; - let skippedRawValue = false; - if (target === "cloud" || checkPreviousLocal) { - try { - adjustment = rawJsonNumbers( - { source_data: value }, - target, - target === "cloud", - true, - () => { - skippedRawValue = true; - }, - ); - } catch (error) { - previousInvalid = error; - } - } let invalid: unknown; const text = serializeImportJson( { [index]: value }, @@ -689,34 +554,7 @@ export function exportedGitHubRecord( target === "cloud", ); const record: unknown = JSON.parse(text)[index]; - const bytes = - Buffer.byteLength(text) - - Buffer.byteLength(JSON.stringify({ [index]: null })) + - 4; - return { - record, - bytes, - adjustment, - previousInvalid, - skippedRawValue, - invalid, - }; -} - -/** Combine the observed vendor bytes with the old native mapping's typed fields. */ -export function previousGitHubExclusion( - evidence: ExternalFindingEvidence, - sourceBytes: number, - adjustment: number, -): string | undefined { - if (adjustment === Infinity) - return "Evidence exceeds the Cloud JSON depth limit of 20."; - const envelope = { ...evidence, source_data: null }; - const bytes = Buffer.byteLength(JSON.stringify(envelope)) - 4 + sourceBytes; - return bytes + adjustment + rawJsonNumbers(envelope, "cloud", true) > - CLOUD_EVIDENCE_BYTES - ? "Evidence exceeds the Cloud limit of 256 KiB." - : undefined; + return { record, invalid }; } export function validateImportRequest(input: unknown): FindingImportRequest { diff --git a/sdk/typescript/src/github-dependabot-findings.ts b/sdk/typescript/src/github-dependabot-findings.ts index 16689fb9fd..58b6e8171f 100644 --- a/sdk/typescript/src/github-dependabot-findings.ts +++ b/sdk/typescript/src/github-dependabot-findings.ts @@ -10,7 +10,7 @@ import { isGitHubCloudHost } from "./git-remote.js"; import { isRecord } from "./record.js"; import { REPORTABLE_SEVERITIES } from "./scan-settings.js"; import { - trackReaderSelection, + appendFinding, record, relativePath, requiredText, @@ -334,7 +334,6 @@ export function normalizeGitHubDependabotAlerts( findings: [], excluded: [], }; - const selection = trackReaderSelection(result); const ids = new Set(); let sourceRepository: string | undefined; for (const [index, alert] of alerts.entries()) { @@ -350,7 +349,7 @@ export function normalizeGitHubDependabotAlerts( ); ids.add(sourceId); const inputEvidence = evidence(alert, url); - selection.append(index + 1, sourceId, () => { + appendFinding(result, index + 1, sourceId, () => { if (target === "cloud" && Buffer.byteLength(sourceId) > 512) throw new Error( "Source finding ID exceeds the Cloud limit of 512 UTF-8 bytes.", @@ -358,5 +357,5 @@ export function normalizeGitHubDependabotAlerts( return validateExternalEvidence(inputEvidence, target); }); } - return selection.finish(); + return result; } diff --git a/sdk/typescript/src/github-findings.ts b/sdk/typescript/src/github-findings.ts index 14b300a2b9..0cbefe8a26 100644 --- a/sdk/typescript/src/github-findings.ts +++ b/sdk/typescript/src/github-findings.ts @@ -1,16 +1,11 @@ import type { ExternalImportTarget } from "./external-import-contract.js"; import { hash } from "node:crypto"; -import { isDeepStrictEqual } from "node:util"; -import { isBoxedPrimitive, isDate, isProxy } from "node:util/types"; +import { isProxy } from "node:util/types"; import { GHSA_ID, githubRepositoryParts } from "./github-identifiers.js"; import { CodexSecurityError, errorMessage } from "./errors.js"; import { cloudRepositoryUrlKey, - CloudEvidenceDepthError, exportedGitHubRecord, - previousGitHubExclusion, - validateExternalEvidenceFields, - validateExternalEvidenceJson, validateExternalEvidence, } from "./external-import-contract.js"; import type { ExternalFinding, ExternalFindings } from "./external-findings.js"; @@ -20,11 +15,7 @@ import { isGitHubDependabotAlert, normalizeGitHubDependabotAlerts, } from "./github-dependabot-findings.js"; -import { - asRecord, - canSnapshotWithoutGetters, - cloneWithoutGetters, -} from "./record.js"; +import { asRecord } from "./record.js"; import { parseJsonOrJsonl, readScannerText, @@ -32,11 +23,6 @@ import { text, } from "./scanner-import.js"; -import { - rememberPreviousSelection, - type LegacyReaderSelection, -} from "./reader-selection.js"; - function repositoryName(value: unknown): string { const repository = text(value); if (!repository || !githubRepositoryParts(repository)) @@ -342,120 +328,9 @@ function ownData(value: unknown, key: string): unknown { return Object.getOwnPropertyDescriptor(value, key)?.value; } -function priorAdvisoryId(value: unknown): string | undefined { - if (ownData(value, "source") !== "github-security-advisory") return; - const id = text(ownData(value, "ghsaId")); - const host = text(ownData(value, "githubHost"))?.toLowerCase(); - const selection = ownData(value, "selection"); - const repository = text(ownData(value, "repository")); - if ( - id && - GHSA_ID.test(id) && - host && - repository && - githubRepositoryParts(repository) && - (selection === "repository" || selection === "global") - ) - return advisorySourceId(host, selection, repository, id); -} - -// Determine whether JSON can call a serializer without evaluating a getter. -function maySerialize(value: unknown): boolean { - while (value && typeof value === "object") { - if (isProxy(value)) return true; - const field = Object.getOwnPropertyDescriptor(value, "toJSON"); - if (field) - return ( - !Object.hasOwn(field, "value") || typeof field.value === "function" - ); - value = Object.getPrototypeOf(value) as unknown; - } - return false; -} - -function codeOccurrencePrefix(value: unknown): string | undefined { - const repository = ownData(value, "repository"), - number = ownData(value, "number"), - url = ownData(value, "url"); - if ( - ownData(value, "source") !== "github-code-scanning" || - typeof repository !== "string" || - !githubRepositoryParts(repository) || - typeof number !== "number" || - !Number.isSafeInteger(number) || - number <= 0 || - typeof url !== "string" - ) - return; - try { - return codeScanningPrefix(sourceUrl(url), repository, number); - } catch { - /* No validated occurrence anchor. */ - } -} - -// The released reader measured the caller's source_data JSON before native capture. -function captureGitHubRecord(input: unknown, target: ExternalImportTarget) { - // Only plain records, arrays and standard Buffers have a reconstructable wire size. - let plainBuffers: Buffer[] | undefined = []; - let hiddenFields = false; - let largeLocalRow = false; - const record = cloneWithoutGetters(input, (value) => { - const prototype = Object.getPrototypeOf(value); - if ( - target === "local" && - ((Array.isArray(value) && value.length > 256 * 1024) || - (prototype === Buffer.prototype && - Buffer.isBuffer(value) && - !Object.hasOwn(value, "byteLength") && - value.byteLength > 256 * 1024)) - ) { - largeLocalRow = true; - return false; - } - // Native capture drops non-enumerable fields the released vendor mapper could read. - if ( - !Array.isArray(value) && - !ArrayBuffer.isView(value) && - !isBoxedPrimitive(value) && - !isDate(value) && - Object.getOwnPropertyNames(value).some( - (key) => - key !== "toJSON" && - !Object.prototype.propertyIsEnumerable.call(value, key), - ) - ) - hiddenFields = true; - - if (!plainBuffers) return; - if (Buffer.isBuffer(value)) { - if ( - prototype === Buffer.prototype && - !Object.hasOwn(value, "toJSON") && - !Object.hasOwn(value, "length") && - !("_arr" in value) - ) - plainBuffers.push(value); - else plainBuffers = undefined; - } else if ( - (prototype !== Object.prototype && - prototype !== null && - prototype !== Array.prototype) || - maySerialize(value) - ) - plainBuffers = undefined; - }); - return { - record: asRecord(record), - hiddenFields, - plainBuffers, - largeLocalRow, - }; -} - function normalizeRecords( records: readonly unknown[], - target: ExternalImportTarget = "cloud", + target: ExternalImportTarget, snapshot = false, ): ExternalFindings { const result: ExternalFindings = { @@ -463,145 +338,17 @@ function normalizeRecords( findings: [], excluded: [], }; - const previous: ExternalFindings | undefined = snapshot - ? { read: records.length, findings: [], excluded: [] } - : undefined; - const checks: (() => void)[] = []; - const uncertainIds = new Set(); - const unidentifiedRows = new Set(); - let independentlyObserved = Infinity; - let ranRootSerializer = false; - let largeLocalSelection = false; - const potentialIds: string[] = []; - const prefixes = new Set(); const identities = new Set(); for (const [index, input] of records.entries()) { - const afterRootSerializer = ranRootSerializer; - const keySensitive = previous !== undefined && maySerialize(input); - ranRootSerializer ||= keySensitive; let exported = input; - const capture = previous ? captureGitHubRecord(input, target) : undefined; - const before = capture?.record; - const priorPrefix = previous ? codeOccurrencePrefix(input) : undefined; - let priorId = previous ? priorAdvisoryId(input) : undefined; - let prefix = priorPrefix; - const rowIds = new Set(); - let rowUncertain = false; - const priorRowCount = previous?.findings.length ?? 0; - function possible() { - rowUncertain = true; - // A caller getter can mutate a later row before its native evidence is seen. - independentlyObserved = Math.min(independentlyObserved, priorRowCount); - for (const id of rowIds) uncertainIds.add(id); - if (prefix) prefixes.add(prefix); - if (priorId) uncertainIds.add(priorId); - else if (!priorPrefix) unidentifiedRows.add(index); - } - // The old reader called a root serializer with "source_data" rather than its - // array index. That code could change a later occurrence before it was read. - if (afterRootSerializer) { - possible(); - unidentifiedRows.add(index); - } - function retainPrevious( - selection: ExternalFindings, - id: string, - evidence: ExternalFinding["evidence"], - ) { - selection.findings.push({ source_finding_id: id, evidence }); - potentialIds.push(id); - checks.push(() => { - if (!isDeepStrictEqual(before, asRecord(cloneWithoutGetters(input)))) - possible(); - }); - } - const exclusion = (record: unknown, reason: string) => ({ - position: index + 1, - source_finding_id: text(asRecord(record)?.["url"]), - reason, - }); - let possibleFreshId = false; try { - if (previous && capture) { - const source = exportedGitHubRecord( - input, - index, - target, - target === "local" && !capture.largeLocalRow, - ); - exported = source.record; - prefix ??= codeOccurrencePrefix(exported); - // A serializer or Buffer can expand a native record that fit the approved request. - const oversizedLocalRow = - target === "local" && - (capture.largeLocalRow || - (source.bytes > 256 * 1024 && - (!before || capture.plainBuffers?.length === 0))); - if (oversizedLocalRow || !before) { - if ( - !oversizedLocalRow && - source.previousInvalid && - !source.skippedRawValue && - canSnapshotWithoutGetters(input) - ) { - previous.excluded.push( - exclusion(exported, errorMessage(source.previousInvalid)), - ); - } else if ( - target !== "local" && - source.bytes + source.adjustment > 256 * 1024 && - canSnapshotWithoutGetters(input) && - !keySensitive - ) { - previous.excluded.push( - exclusion( - exported, - "Evidence exceeds the Cloud limit of 256 KiB.", - ), - ); - } else { - largeLocalSelection ||= oversizedLocalRow; - // An occurrence without independent native evidence can only stop a retry. - possible(); - possibleFreshId = true; - } - } else { - if (capture.hiddenFields || source.skippedRawValue) possible(); - try { - const row = mapRecord(before); - const knownOldId = row.source_finding_id; - priorId = knownOldId; - if (!afterRootSerializer) unidentifiedRows.delete(index); - rowIds.add(knownOldId); - if (rowUncertain) uncertainIds.add(knownOldId); - const evidence = validateExternalEvidenceFields( - row.evidence, - target, - ); - if (source.previousInvalid) throw source.previousInvalid; - if (keySensitive) possible(); - const reason = - keySensitive || target === "local" - ? undefined - : previousGitHubExclusion( - evidence, - source.bytes, - source.adjustment, - ); - if (reason) previous.excluded.push(exclusion(before, reason)); - else retainPrevious(previous, knownOldId, evidence); - } catch (error) { - previous.excluded.push(exclusion(before, errorMessage(error))); - } - } - if (source.invalid) throw source.invalid; + if (snapshot) { + const capture = exportedGitHubRecord(input, index, target); + exported = capture.record; + if (capture.invalid) throw capture.invalid; } - const mapped = mapRecord(exported), - id = mapped.source_finding_id; - rowIds.add(id); - if (rowUncertain) uncertainIds.add(id); - if (possibleFreshId) potentialIds.push(id); - const evidence = validateExternalEvidence(mapped.evidence, target); + const { source_finding_id: id, evidence: mapped } = mapRecord(exported); + const evidence = validateExternalEvidence(mapped, target); if (identities.has(id)) throw new CodexSecurityError( `Duplicate source finding id ${JSON.stringify(id)}. Select each GitHub occurrence once.`, @@ -610,83 +357,15 @@ function normalizeRecords( result.findings.push({ source_finding_id: id, evidence }); } catch (error) { if (error instanceof CodexSecurityError) throw error; - const reason = errorMessage(error); - result.excluded.push( - exclusion( - snapshot && exported === input - ? { url: ownData(input, "url") } - : exported, - reason, - ), - ); - if (previous && exported === input) { - if ( - error instanceof CloudEvidenceDepthError && - before && - // Extra Buffer fields were inspected by the old row validator, but - // structuredClone does not retain them as part of the typed array. - capture?.plainBuffers?.every( - (buffer) => Object.keys(buffer).length === buffer.length, - ) - ) { - // Exported JSON can be deeper than an approved native snapshot. - // Retain only the independently observed, valid prior selection. - try { - const row = mapRecord(before); - const evidence = validateExternalEvidence(row.evidence, target); - const savedSource = JSON.stringify(evidence.source_data); - // structuredClone turns each Buffer into a Uint8Array. Reconstruct its - // original serialized size on detached data without running caller hooks. - const oldSource = JSON.stringify( - evidence.source_data, - (_key, value: unknown) => - value instanceof Uint8Array - ? { type: "Buffer", data: [...value] } - : value, - ); - validateExternalEvidenceJson( - evidence, - target, - Buffer.byteLength(oldSource) - Buffer.byteLength(savedSource), - ); - retainPrevious(previous, row.source_finding_id, evidence); - } catch (priorError) { - previous.excluded.push(exclusion(before, errorMessage(priorError))); - } - continue; - } - // A serializer may throw at its array key while the old source_data - // key produced a valid record. Keep only evidence for refusing a match. - possible(); - if (before) - try { - const id = mapRecord(before).source_finding_id; - priorId = id; - if (!afterRootSerializer) unidentifiedRows.delete(index); - potentialIds.push(id); - uncertainIds.add(id); - } catch { - /* No old identity. */ - } - } - } - } - if (previous) { - for (const check of checks) check(); - const uncertain = independentlyObserved !== Infinity; - if (uncertain || !isDeepStrictEqual(previous.findings, result.findings)) { - const prior: LegacyReaderSelection = { sourceFindingIds: potentialIds }; - if (uncertain) { - prior.unprovenIdPrefixes = [...prefixes]; - prior.uncertainFindingIds = [...uncertainIds]; - if (unidentifiedRows.size) - prior.unprovenOccurrenceCount = unidentifiedRows.size; - if (!largeLocalSelection) - prior.knownFindings = previous.findings - .slice(0, independentlyObserved) - .filter((finding) => !uncertainIds.has(finding.source_finding_id)); - } else prior.snapshot = previous; - rememberPreviousSelection(result, prior); + const url = + snapshot && exported === input + ? ownData(input, "url") + : asRecord(exported)?.["url"]; + result.excluded.push({ + position: index + 1, + source_finding_id: text(url), + reason: errorMessage(error), + }); } } return result; diff --git a/sdk/typescript/src/hosted-scanner-v1.ts b/sdk/typescript/src/hosted-scanner-v1.ts deleted file mode 100644 index 5383815271..0000000000 --- a/sdk/typescript/src/hosted-scanner-v1.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { canonicalNumber } from "./external-import-contract.js"; -import { - hostedNumberToken, - parseHostedScannerJson, -} from "./hosted-scanner-http.js"; -import { isRecord } from "./record.js"; - -const jsonNumber = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:e[+-]?\d+)?$/iu; - -export function legacyDecimalInteger(token: string): string | undefined { - if (!jsonNumber.test(token)) return; - const value = Number(token); - // Rule out unbounded exponents cheaply. Rounded boundary values still need an exact check. - if (!Number.isInteger(value) || value < 1 || value > 2 ** 63) return; - const id = canonicalNumber(token); - return /^[1-9]\d{0,18}$/u.test(id) && - (id.length < 19 || id <= "9223372036854775807") - ? id - : undefined; -} - -/** A legacy export needs a type witness when JSON.parse cannot preserve a decimal. */ -export function isLegacyWitness(token: string): boolean { - if (!jsonNumber.test(token)) return false; - const value = Number(token); - if ( - !Number.isFinite(value) || - (Number.isInteger(value) && !Number.isSafeInteger(value)) - ) - return true; - // Distinguish zero from underflow without converting an unbounded exponent. - if (value === 0) return /[1-9]/u.test(token.split(/e/iu, 1)[0]!); - return canonicalNumber(token) !== String(value); -} - -function restoreField(value: unknown, witness: unknown): unknown { - if (typeof witness === "string") { - if (value !== witness || !isLegacyWitness(witness)) - throw new Error("Invalid exact scanner number."); - return parseHostedScannerJson(witness); - } - if ( - !isRecord(witness) || - hostedNumberToken(witness) !== undefined || - (!Array.isArray(value) && - (!isRecord(value) || hostedNumberToken(value) !== undefined)) - ) - throw new Error("Invalid scanner numeric fields."); - const restored = Array.isArray(value) ? [...value] : { ...value }; - const fields = restored as Record; - for (const [key, child] of Object.entries(witness)) { - if (!Object.hasOwn(restored, key)) - throw new Error("Invalid scanner numeric field."); - fields[key] = restoreField(fields[key], child); - } - return restored; -} - -/** Original hosted exports saved imprecise JSON numbers as text plus a sparse type witness. */ -export function restoreLegacyHostedNumbers( - document: Record, -): Record { - const fields = document["numericFields"]; - if (fields === undefined) return document; - if (!isRecord(fields) || Object.hasOwn(fields, "numericFields")) - throw new Error("Invalid scanner numeric fields."); - return restoreField(document, fields) as Record; -} diff --git a/sdk/typescript/src/inspector-findings.ts b/sdk/typescript/src/inspector-findings.ts index ee1b9a14ee..2717151672 100644 --- a/sdk/typescript/src/inspector-findings.ts +++ b/sdk/typescript/src/inspector-findings.ts @@ -519,7 +519,6 @@ function evidence( id: string, source: string, target: ExternalImportTarget, - previousProjection: boolean, ) { const { description, @@ -589,10 +588,7 @@ function evidence( description: requiredText(description, "Inspector description"), severity: scannerSeverity(level), url: text(pkg?.sourceUrl), - advisory_ids: - previousProjection && pkg?.relatedVulnerabilities - ? [...new Set(advisoryIds.concat(pkg.relatedVulnerabilities))] - : advisoryIds, + advisory_ids: advisoryIds, packages, locations, source_updated_at: updatedAt === undefined ? null : timestamp(updatedAt), @@ -602,18 +598,17 @@ function evidence( finding: structuredClone(finding), }, }; - const enrichment = - code && !previousProjection - ? { - details: { - kind: "sast", - repository: { url: source }, - rule: { id: code.detectorId, name: code.detectorName }, - weakness_ids: strings(code.cwes), - code: endLine === null ? null : { end_line: endLine }, - }, - } - : undefined; + const enrichment = code + ? { + details: { + kind: "sast", + repository: { url: source }, + rule: { id: code.detectorId, name: code.detectorName }, + weakness_ids: strings(code.cwes), + code: endLine === null ? null : { end_line: endLine }, + }, + } + : undefined; return validateEnrichedExternalEvidence(base, target, enrichment); } @@ -622,26 +617,6 @@ export function normalizeInspectorFindings( sourceRepositoryUrl: string, target: ExternalImportTarget = "cloud", projectMap?: ReadonlyMap, -): ExternalFindings { - return normalizeReport(input, sourceRepositoryUrl, target, projectMap, false); -} - -/** @internal Reconstruct the prior evidence when matching a pending publication. */ -export function normalizePreviousInspectorFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", - projectMap?: ReadonlyMap, -): ExternalFindings { - return normalizeReport(input, sourceRepositoryUrl, target, projectMap, true); -} - -function normalizeReport( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget, - projectMap: ReadonlyMap | undefined, - previousProjection: boolean, ): ExternalFindings { const source = repositoryUrl(sourceRepositoryUrl); if ( @@ -663,44 +638,29 @@ function normalizeReport( claims: repositoryClaims(finding), })); checkReportScope(rows, target, projectMap); - function checkRetainedScope( - result: ExternalFindings, - removedIds?: Set, - ) { - if (!result.excluded.length || !result.findings.length) return; - // Only surviving rows can supply shared provider and host evidence on replay. - const skipped = new Set(result.excluded.map((row) => row.position)); - const retained = rows.filter((row) => !skipped.has(row.position)); - checkReportScope(retained, target, projectMap, (row, error) => { - skipped.add(row.position); - result.excluded.push({ - position: row.position, - source_finding_id: identity(record(row.finding, "Inspector finding")) - .arn, - reason: errorMessage(error), - }); - }); - result.findings = result.findings.filter((finding, i) => { - if (!skipped.has(retained[i]!.position)) return true; - removedIds?.add(finding.source_finding_id); - return false; - }); - result.excluded.sort((a, b) => a.position - b.position); - } - return normalizeRecords( + const result = normalizeRecords( rows, - ({ finding }, id) => - evidence( - selectedFinding(finding), - id, - source, - target, - previousProjection, - ), + ({ finding }, id) => evidence(selectedFinding(finding), id, source, target), ({ finding }) => identity(record(finding, "Inspector finding")).arn, target, - { filter: checkRetainedScope }, ); + if (!result.excluded.length || !result.findings.length) return result; + // Only surviving rows can supply shared provider and host evidence on replay. + const skipped = new Set(result.excluded.map((row) => row.position)); + const retained = rows.filter((row) => !skipped.has(row.position)); + checkReportScope(retained, target, projectMap, (row, error) => { + skipped.add(row.position); + result.excluded.push({ + position: row.position, + source_finding_id: identity(record(row.finding, "Inspector finding")).arn, + reason: errorMessage(error), + }); + }); + result.findings = result.findings.filter( + (_, i) => !skipped.has(retained[i]!.position), + ); + result.excluded.sort((a, b) => a.position - b.position); + return result; } export async function readInspectorFindings( diff --git a/sdk/typescript/src/opengrep-findings.ts b/sdk/typescript/src/opengrep-findings.ts index 9aae0d0400..96827c1d90 100644 --- a/sdk/typescript/src/opengrep-findings.ts +++ b/sdk/typescript/src/opengrep-findings.ts @@ -1,5 +1,4 @@ import { - previousAdmission, validateExternalEvidenceJson, validateExternalReport, type ExternalImportTarget, @@ -15,7 +14,7 @@ import { sourceRepositoryUrl, } from "./native-json-findings.js"; import { - trackReaderSelection, + appendFinding, array, readScannerJson, record, @@ -68,14 +67,11 @@ export function normalizeOpenGrepFindings( excluded: [], report: structuredClone(report), }; - const selection = trackReaderSelection(output); type Observation = { position: number; parsed: ReturnType; }; function exclude(position: number, id: string | null, error: unknown) { - if (id && error instanceof Error && previousAdmission(error) !== undefined) - selection.uncertainIds.add(id); output.excluded.push({ position, source_finding_id: id, @@ -146,8 +142,6 @@ export function normalizeOpenGrepFindings( projectedBytes, ); } catch (error) { - if (error instanceof Error && previousAdmission(error) !== undefined) - selection.uncertainIds.add(id); singletonErrors.set(observation, errorMessage(error)); } } @@ -202,7 +196,7 @@ export function normalizeOpenGrepFindings( )) { if (observations.length === 1) { const { position } = observations[0]!; - selection.append(position, id, () => { + appendFinding(output, position, id, () => { const evidence = projectGroup(observations); positions.set(id, [position]); return evidence; @@ -218,11 +212,6 @@ export function normalizeOpenGrepFindings( validateExternalEvidenceJson( { source_data: { results: [result] } }, target, - 0, - (previousBytes, currentBytes) => { - if (previousBytes !== currentBytes) - selection.uncertainIds.add(id); - }, ); observation.parsed.result = result; return [{ observation, key: canonicalJson(result) }]; @@ -248,7 +237,6 @@ export function normalizeOpenGrepFindings( if (retained.length === ordered.length) throw error; evidence = projectGroup(retained); } - if (previousAdmission(evidence) === null) selection.uncertainIds.add(id); output.findings.push({ source_finding_id: id, evidence }); positions.set( id, @@ -261,7 +249,7 @@ export function normalizeOpenGrepFindings( } output.excluded.sort((left, right) => left.position - right.position); sourcePositions.set(output.findings, positions); - return selection.finish(); + return output; } export async function readOpenGrepFindings( diff --git a/sdk/typescript/src/private-finding-import.ts b/sdk/typescript/src/private-finding-import.ts index ee457d352b..5d32509979 100644 --- a/sdk/typescript/src/private-finding-import.ts +++ b/sdk/typescript/src/private-finding-import.ts @@ -1,12 +1,15 @@ import { hash } from "node:crypto"; -import { constants } from "node:fs"; -import { open } from "node:fs/promises"; import { setTimeout as delay } from "node:timers/promises"; import { z } from "zod"; import { CodexSecurityError } from "./errors.js"; -import type { ExternalFinding, ExternalFindings } from "./external-findings.js"; +import { + normalizeEndorFindings, + requiresStructuredEndorReport, +} from "./endor-findings.js"; +import type { ExternalFinding } from "./external-findings.js"; import type { ImportSource } from "./external-import-models.js"; import { canonicalJson, isRecord } from "./record.js"; +import { normalizeWizJson } from "./wiz-findings.js"; const structuredFormat = "security.findings.v1"; const privateFormat = z.enum([ @@ -89,6 +92,8 @@ const jobSchema = z.object({ export type PrivateImportContext = z.infer; export type PrivateImportJob = z.infer; +class PrivateReportSizeError extends CodexSecurityError {} + export function privateReport( findings: ExternalFinding[], limits: PrivateImportContext["limits"], @@ -117,14 +122,14 @@ export function privateReport( }, }; if (Buffer.byteLength(canonicalJson(record)) > limits.max_record_bytes) - throw new CodexSecurityError( + throw new PrivateReportSizeError( `Finding ${JSON.stringify(source_finding_id)} exceeds the private import record limit.`, ); return record; }); const bytes = canonicalJson({ format: structuredFormat, findings: records }); if (Buffer.byteLength(bytes) > limits.max_direct_file_bytes) - throw new CodexSecurityError( + throw new PrivateReportSizeError( "The selected report exceeds the Cloud direct-upload limit. Split this selection.", ); return { @@ -134,163 +139,95 @@ export function privateReport( }; } -const endorExport = z.strictObject({ - list: z.strictObject({ - objects: z.array(z.unknown()), - response: z - .strictObject({ - next_page_id: z.literal(["", null]).optional(), - next_page_token: z.literal(["", "0", 0, null]).optional(), - total_size: z.union([z.string(), z.number()]).optional(), - }) - .optional(), - }), -}); -const wizExport = z.strictObject({ - vulnerabilityFindings: z.strictObject({ - nodes: z.array(z.unknown()), - pageInfo: z - .strictObject({ - hasNextPage: z.literal(false).optional(), - hasPreviousPage: z.literal(false).optional(), - startCursor: z.string().nullable().optional(), - endCursor: z.string().nullable().optional(), - }) - .optional(), - }), -}); - -// Only complete, known envelopes can be sent byte for byte. Unreviewed sibling -// collections, metadata or partial pages keep the normalized projection. -function nativeRecords( - payload: unknown, - provider: string, -): unknown[] | undefined { - if (provider === "wiz" && Array.isArray(payload)) return payload; - if (!isRecord(payload)) return; - if (provider === "endor") { - if (isRecord(payload["spec"])) return [payload]; - return endorExport.safeParse(payload).data?.list.objects; - } - if (typeof payload["id"] === "string") return [payload]; - const data = - Object.keys(payload).length === 1 && "data" in payload - ? payload["data"] - : payload; - return wizExport.safeParse(data).data?.vulnerabilityFindings.nodes; -} - -export async function selectPrivateReport( - selection: ExternalFindings, +/** Encode only the reviewed occurrences, whether they came from a file or the workbench. */ +export function selectPrivateReport( + findings: ExternalFinding[], provider: string, - path: string | undefined, context: PrivateImportContext, - saved: string | undefined, - requiresStructuredReport = false, -): Promise> { - const checkpoint = - saved === undefined ? undefined : checkpointSchema.parse(JSON.parse(saved)); - const originalFormat = - checkpoint?.contextVersion === context.context_version - ? checkpoint.format - : undefined; - if (requiresStructuredReport && !originalFormat) { - if (!context.formats.includes(structuredFormat)) - throw new CodexSecurityError( - `This Cloud service must support ${structuredFormat} to publish the reviewed Endor fields.`, - ); - return privateReport(selection.findings, context.limits); - } - const format = + approvedFormat?: PrivateImportFormat, +): ReturnType { + const nativeFormat = provider === "endor" ? "endor.findings_json.v1" : provider === "wiz" ? "wiz.vulnerability_json.v1" : undefined; - if ( - format && - path && - selection.excluded.length === 0 && - selection.read <= 10_000 && - selection.findings.length === selection.read && - (originalFormat - ? originalFormat === format - : context.formats.includes(format)) - ) { - const handle = await open( - path, - constants.O_RDONLY | (constants.O_NONBLOCK ?? 0), - ); - let bytes: string | undefined; + let format = + approvedFormat ?? + (context.formats.includes(structuredFormat) + ? structuredFormat + : nativeFormat); + let sizeError: PrivateReportSizeError | undefined; + if (!nativeFormat || !format || format === structuredFormat) { try { - const info = await handle.stat(); + return privateReport(findings, context.limits); + } catch (error) { if ( - info.isFile() && - info.size > 0 && - info.size <= context.limits.max_direct_file_bytes - ) { - const buffer = Buffer.alloc(info.size + 1); - let count = 0; - while (count < buffer.length) { - const { bytesRead } = await handle.read( - buffer, - count, - buffer.length - count, - ); - if (bytesRead === 0) break; - count += bytesRead; - } - if (count === info.size) { - try { - bytes = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(buffer.subarray(0, count)); - } catch { - /* Binary exports use the normalized selection. */ - } - } - } - } finally { - await handle.close(); - } - if (bytes !== undefined) { - let payload: unknown; - try { - payload = JSON.parse(bytes.replace(/^\uFEFF/u, "")); - } catch { - /* JSONL and other exports use the normalized selection. */ - } - const records = nativeRecords(payload, provider); - if ( - records && - records.length === selection.findings.length && - records.every((raw, index) => { - const reviewed = selection.findings[index]!; - if (!isRecord(raw)) return false; - const record = canonicalJson(raw); - return ( - raw[provider === "endor" ? "uuid" : "id"] === - reviewed.source_finding_id && - reviewed.source_finding_id === reviewed.source_finding_id.trim() && - reviewed.evidence.source_data != null && - record === canonicalJson(reviewed.evidence.source_data) && - Buffer.byteLength(record) <= context.limits.max_record_bytes - ); - }) + !(error instanceof PrivateReportSizeError) || + approvedFormat || + !nativeFormat || + !context.formats.includes(nativeFormat) ) - return { format, bytes, digest: hash("sha256", bytes) }; + throw error; + sizeError = error; + format = nativeFormat; } } - if (originalFormat && originalFormat !== structuredFormat) + if (!approvedFormat && !context.formats.includes(format)) throw new CodexSecurityError( - "The original native report is required to resume this private import.", + "Private findings import is unavailable for this report format.", ); - return privateReport(selection.findings, context.limits); + + // The native endpoint derives its own evidence. Verify that it will receive + // the selected occurrences with exactly the fields the user reviewed. + const rows = findings.map(({ evidence }) => evidence.source_data); + const endor = provider === "endor"; + const payload = endor ? { list: { objects: rows } } : rows; + let totalRowBytes = 0; + if ( + format === nativeFormat && + findings.length > 0 && + findings.length <= 10_000 && + rows.every((row, i) => { + if (!isRecord(row)) return false; + const id = findings[i]!.source_finding_id; + const bytes = Buffer.byteLength(canonicalJson(row)); + totalRowBytes += bytes; + return ( + row[endor ? "uuid" : "id"] === id && + id === id.trim() && + (endor || !("source_finding_id" in row)) && + bytes <= context.limits.max_record_bytes && + totalRowBytes <= context.limits.max_direct_file_bytes + ); + }) + ) { + const projected = endor + ? normalizeEndorFindings(payload) + : normalizeWizJson(payload, "cloud"); + if ( + !projected.excluded.length && + !requiresStructuredEndorReport(projected) && + projected.findings.length === findings.length && + projected.findings.every( + (row, i) => + row.source_finding_id === findings[i]!.source_finding_id && + canonicalJson(row.evidence) === canonicalJson(findings[i]!.evidence), + ) + ) { + const bytes = canonicalJson(payload); + if (Buffer.byteLength(bytes) <= context.limits.max_direct_file_bytes) + return { format, bytes, digest: hash("sha256", bytes) }; + } + } + if (sizeError) throw sizeError; + throw new CodexSecurityError( + `The selection cannot be represented exactly within the limits of ${nativeFormat}. This Cloud service must support ${structuredFormat} to publish it.`, + ); } const checkpointSchema = z.object({ - format: privateFormat.default(structuredFormat), + format: privateFormat, apiBaseUrl: z.string(), accountId: z.string(), contextVersion: z.string(), @@ -299,9 +236,8 @@ const checkpointSchema = z.object({ repository: z.object({ id: z.string(), repo_connector_id: z.string() }), uploadRequestId: z.uuid(), jobRequestId: z.uuid(), - // Earlier checkpoints used the unversioned job route. jobApiVersion: jobVersion.optional(), - approvedRepositoryUrl: z.string().optional(), + approvedRepositoryUrl: z.string(), approvedRepositoryPlatform: z.literal("other").optional(), fileId: fileId.optional(), jobId: jobSchema.shape.id.optional(), @@ -333,12 +269,14 @@ export function unsuccessfulJob(job: PrivateImportJob | undefined) { return job?.status === "failed" || job?.status === "cancelled"; } -export function parsePrivateCheckpoint(serialized: string) { - return checkpointSchema.parse(JSON.parse(serialized)); +export function parsePrivateCheckpoint(serialized: string | undefined) { + return serialized === undefined + ? undefined + : checkpointSchema.parse(JSON.parse(serialized)); } export function readPrivateCheckpoint( - serialized: string | undefined, + checkpoint: ReturnType, expected: { format: PrivateImportFormat; apiBaseUrl: string; @@ -349,8 +287,7 @@ export function readPrivateCheckpoint( repository: { id: string; repo_connector_id: string }; }, ) { - if (serialized === undefined) return; - const checkpoint = parsePrivateCheckpoint(serialized); + if (!checkpoint) return; const { fileId, jobId, receipt, previousReceipt } = checkpoint; const { apiBaseUrl, accountId, source, repository } = checkpoint; const { contextVersion, format, sha256, ...scope } = expected; @@ -376,7 +313,7 @@ export function readPrivateCheckpoint( return sameContext ? checkpoint : undefined; } -export class PrivateImportRequestError extends CodexSecurityError { +class PrivateImportRequestError extends CodexSecurityError { constructor( readonly status: number, readonly code: string | undefined, diff --git a/sdk/typescript/src/reader-selection.ts b/sdk/typescript/src/reader-selection.ts deleted file mode 100644 index dbf7550a0b..0000000000 --- a/sdk/typescript/src/reader-selection.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { createHash } from "node:crypto"; -import { isDeepStrictEqual } from "node:util"; -import { isBoxedPrimitive, isProxy } from "node:util/types"; -import type { ExternalFindings } from "./external-findings.js"; -import { canonicalJson, cloneWithoutGetters } from "./record.js"; - -const changedCloudProjections = new WeakSet(); - -// Readers mark selections whose original Cloud request can have different evidence. -export function markPreviousCloudProjection(findings: ExternalFindings): void { - changedCloudProjections.add(findings); -} - -export function hasPreviousCloudProjection( - findings: ExternalFindings, -): boolean { - return changedCloudProjections.has(findings); -} - -export interface LegacyReaderSelection { - sourceFindingIds: string[]; - snapshot?: ExternalFindings; - unprovenIdPrefixes?: string[]; - uncertainFindingIds?: string[]; - unprovenOccurrenceCount?: number; - knownFindings?: ExternalFindings["findings"]; -} -const previousSelections = new WeakMap< - object, - { - exported?: string; - native?: ExternalFindings; - previous: LegacyReaderSelection; - } ->(); - -// The registry digest is process-local. Frame both ordered lists separately so a -// local import never needs one JSON string containing all its findings. -function exportedSelectionDigest( - result: ExternalFindings, - preserveScalar?: (value: object) => boolean, -): string { - const { findings, excluded, ...metadata } = result; - const digest = createHash("sha256"); - digest.update(canonicalJson(metadata, undefined, preserveScalar)); - for (const rows of [excluded, findings]) { - digest.update("["); - for (let index = 0; index < rows.length; index++) { - // Keep the array index supplied to toJSON. An omitted entry in an array - // is JSON null, just like an explicit null or a hole. - const entry = canonicalJson( - { [index]: rows[index] }, - undefined, - preserveScalar, - ); - digest.update(entry === "{}" ? `{"${index}":null}` : entry); - } - digest.update("]"); - } - return digest.digest("hex"); -} - -export function rememberPreviousSelection( - result: ExternalFindings, - prior: LegacyReaderSelection, -): void { - // Stop-only hints do not require hashing unbounded local evidence. - const exported = - prior.snapshot || prior.knownFindings - ? exportedSelectionDigest(result, isBoxedPrimitive) - : undefined; - const selection = { - exported, - native: exported ? cloneWithoutGetters(result) : undefined, - previous: prior, - }; - previousSelections.set(result, selection); - previousSelections.set(result.findings, selection); -} - -/** Only unchanged SDK output can identify an older, differently serialized selection. */ -export function previousReaderSelection( - input: ExternalFindings, - current: ExternalFindings, - native?: ExternalFindings | null, -): LegacyReaderSelection | undefined { - const selection = - previousSelections.get(input) ?? - (!isProxy(input) && - previousSelections.get( - Object.getOwnPropertyDescriptor(input, "findings")?.value, - )); - if (!selection) return undefined; - const observed = native === undefined ? cloneWithoutGetters(input) : native; - if (selection.native && observed && selection.exported) { - // Callers can attach a report after normalizing rows. Historical validation - // and the saved approval still bind the independently captured report. - const attachedReport = - !Object.hasOwn(selection.native, "report") && - !Object.hasOwn(selection.previous.snapshot ?? {}, "report") && - observed.report !== undefined; - const rows = (value: ExternalFindings) => { - if (!attachedReport) return value; - const { report: _report, ...readerOutput } = value; - return readerOutput; - }; - if ( - isDeepStrictEqual(selection.native, rows(observed)) && - exportedSelectionDigest(rows(current)) === selection.exported - ) { - const prior = selection.previous; - return attachedReport && prior.snapshot - ? { ...prior, snapshot: { ...prior.snapshot, report: observed.report } } - : prior; - } - } - // Without a verified snapshot these IDs can only stop a retry. Preserve - // the caller's order when they rearranged the same known occurrences. - let sourceFindingIds = selection.previous.sourceFindingIds; - if (current.findings.length === sourceFindingIds.length) { - const members = new Set(sourceFindingIds); - const ordered = current.findings.map((row) => row.source_finding_id); - if (ordered.every((id) => members.has(id))) sourceFindingIds = ordered; - } - return { - sourceFindingIds, - unprovenIdPrefixes: selection.previous.unprovenIdPrefixes, - uncertainFindingIds: selection.previous.uncertainFindingIds, - unprovenOccurrenceCount: selection.previous.unprovenOccurrenceCount, - }; -} diff --git a/sdk/typescript/src/record.ts b/sdk/typescript/src/record.ts index 6456e2c6d8..427369d3f7 100644 --- a/sdk/typescript/src/record.ts +++ b/sdk/typescript/src/record.ts @@ -1,60 +1,3 @@ -import { isBoxedPrimitive, isDate, isProxy } from "node:util/types"; - -/** Native capture reads enumerable own accessors and may run proxy traps. */ -export function canSnapshotWithoutGetters( - input: unknown, - observe?: (value: object) => boolean | void, -): boolean { - const seen = new Set(); - const pending: unknown[] = [input]; - while (pending.length) { - const value = pending.pop(); - if (!value || typeof value !== "object" || seen.has(value)) continue; - seen.add(value); - if (isProxy(value)) return false; - if (observe?.(value) === false) return false; - if (isBoxedPrimitive(value) || ArrayBuffer.isView(value) || isDate(value)) - continue; - const prototype: unknown = Object.getPrototypeOf(value); - if ( - !Array.isArray(value) && - prototype !== Object.prototype && - prototype !== null - ) { - if ( - !prototype || - typeof prototype !== "object" || - isProxy(prototype) || - Object.getPrototypeOf(prototype) !== Object.prototype || - !Object.getOwnPropertyNames(prototype).every( - (key) => key === "constructor" || key === "toJSON", - ) - ) - return false; - } - for (const key of Object.keys(value)) { - const field = Object.getOwnPropertyDescriptor(value, key)!; - if (!Object.hasOwn(field, "value")) return false; - if (field.value !== null && typeof field.value === "object") - pending.push(field.value); - } - } - return true; -} - -/** Capture historical data only when cloning cannot run a caller's getter. */ -export function cloneWithoutGetters( - value: T, - observe?: (value: object) => boolean | void, -): T | undefined { - if (!canSnapshotWithoutGetters(value, observe)) return; - try { - return structuredClone(value); - } catch { - return; - } -} - export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } @@ -63,16 +6,12 @@ export function asRecord(value: unknown): Record | undefined { return isRecord(value) ? value : undefined; } -export function canonicalJson( - value: unknown, - compareKeys?: (left: string, right: string) => number, - preserveScalar?: (value: object) => boolean, -): string { +export function canonicalJson(value: unknown): string { return JSON.stringify(value, (_key, child: unknown) => { - if (isRecord(child) && !preserveScalar?.(child)) { + if (isRecord(child)) { return Object.fromEntries( Object.keys(child) - .sort(compareKeys) + .sort() .map((key) => [key, child[key]]), ); } diff --git a/sdk/typescript/src/scanner-import.ts b/sdk/typescript/src/scanner-import.ts index 1141b464b8..2433ef269f 100644 --- a/sdk/typescript/src/scanner-import.ts +++ b/sdk/typescript/src/scanner-import.ts @@ -8,8 +8,6 @@ import { CodexSecurityError, errorMessage } from "./errors.js"; import type { ExternalFinding, ExternalFindings } from "./external-findings.js"; import { REPORTABLE_SEVERITIES, SCAN_SEVERITIES } from "./scan-settings.js"; import { asRecord, isRecord } from "./record.js"; -import { previousAdmission } from "./external-import-contract.js"; -import { rememberPreviousSelection } from "./reader-selection.js"; export function record(value: unknown, label: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) @@ -319,6 +317,14 @@ export function parseJsonOrJsonl(contents: string): unknown { } } +/** Bind a normalized JSON report reader to the SDK's file entrypoint. */ +export function scannerJsonReader( + normalize: (report: unknown, ...options: Options) => ExternalFindings, +) { + return async (path: string, ...options: Options): Promise => + normalize(await readScannerJson(path), ...options); +} + export async function readScannerJson( path: string | FileHandle, parse: (json: string) => unknown = JSON.parse, @@ -558,110 +564,22 @@ export function validateSarifRuleReference( ); } -interface SizeChange { - index: number; - position: number; - id: string; - previous: ExternalFinding["evidence"] | null; -} - -type SelectionFilter = ( - selection: ExternalFindings, - removedIds?: Set, -) => void; - -/** Track older size decisions while a single reader constructs its result. */ -export function trackReaderSelection(result: ExternalFindings) { - const changes: SizeChange[] = []; - const uncertainIds = new Set(); - - function append( - position: number, - id: string, - project: () => ExternalFinding["evidence"], - ) { - const index = result.findings.length; - let outcome: unknown; - try { - const evidence = project(); - outcome = evidence; - result.findings.push({ source_finding_id: id, evidence }); - } catch (error) { - outcome = error; - result.excluded.push({ - position, - source_finding_id: id, - reason: errorMessage(error), - }); - } - const previous = - outcome && typeof outcome === "object" - ? previousAdmission(outcome) - : undefined; - if (previous === "uncertain") uncertainIds.add(id); - else if (previous !== undefined) - changes.push({ index, position, id, previous }); - } - - function finish(filter?: SelectionFilter): ExternalFindings { - if (!changes.length && !uncertainIds.size) { - filter?.(result); - return result; - } - const findings: ExternalFinding[] = []; - const excluded: ExternalFindings["excluded"] = []; - const restoredPositions = new Set(); - let cursor = 0; - for (const { index, position, id, previous } of changes) { - while (cursor < index) findings.push(result.findings[cursor++]!); - if (previous) { - findings.push({ source_finding_id: id, evidence: previous }); - restoredPositions.add(position); - } else { - cursor++; - excluded.push({ - position, - source_finding_id: id, - reason: "Evidence exceeds the Cloud limit of 256 KiB.", - }); - } - } - while (cursor < result.findings.length) - findings.push(result.findings[cursor++]!); - for (const row of result.excluded) { - if (!restoredPositions.has(row.position)) excluded.push(row); - } - excluded.sort((a, b) => a.position - b.position); - const previous = { ...result, findings, excluded }; - // A removed sibling could have depended on an unproven row for its scope. - filter?.(previous, uncertainIds.size ? uncertainIds : undefined); - filter?.(result); - const sourceFindingIds = [ - ...new Set([ - ...previous.findings.map((row) => row.source_finding_id), - ...uncertainIds, - ]), - ]; - if (uncertainIds.size) { - rememberPreviousSelection(result, { - sourceFindingIds, - uncertainFindingIds: [...uncertainIds], - knownFindings: structuredClone( - previous.findings.filter( - (row) => !uncertainIds.has(row.source_finding_id), - ), - ), - }); - } else { - rememberPreviousSelection(result, { - sourceFindingIds, - snapshot: structuredClone(previous), - }); - } - return result; +/** Append the evidence or its exclusion at the original input position. */ +export function appendFinding( + result: ExternalFindings, + position: number, + id: string, + project: () => ExternalFinding["evidence"], +) { + try { + result.findings.push({ source_finding_id: id, evidence: project() }); + } catch (error) { + result.excluded.push({ + position, + source_finding_id: id, + reason: errorMessage(error), + }); } - - return { append, finish, uncertainIds }; } /** Exclude every colliding record: choosing the first could overwrite another occurrence. */ @@ -670,10 +588,7 @@ export function normalizeRecords( projectEvidence: (record: T, id: string) => ExternalFinding["evidence"], identify: (record: T) => string, target: "local" | "cloud" = "cloud", - options: { - report?: ExternalFindings["report"]; - filter?: SelectionFilter; - } = {}, + options: Pick = {}, ): ExternalFindings { const result: ExternalFindings = { read: records.length, @@ -681,7 +596,6 @@ export function normalizeRecords( excluded: [], ...(options.report === undefined ? {} : { report: options.report }), }; - const selection = trackReaderSelection(result); const identified: { position: number; value: T; id: string }[] = []; const counts = new Map(); for (const [index, value] of records.entries()) { @@ -717,8 +631,8 @@ export function normalizeRecords( }); continue; } - selection.append(position, id, () => projectEvidence(value, id)); + appendFinding(result, position, id, () => projectEvidence(value, id)); } result.excluded.sort((a, b) => a.position - b.position); - return selection.finish(options.filter); + return result; } diff --git a/sdk/typescript/src/semgrep-findings.ts b/sdk/typescript/src/semgrep-findings.ts index 6d524216c0..760b5ffa90 100644 --- a/sdk/typescript/src/semgrep-findings.ts +++ b/sdk/typescript/src/semgrep-findings.ts @@ -1,8 +1,6 @@ -import { markPreviousCloudProjection } from "./reader-selection.js"; import { normalizeSemgrepPlatformFindings, isSemgrepPlatformFormat, - semgrepPlatformLegacyFormat, semgrepPlatformFormat, } from "./semgrep-platform-findings.js"; import { isRecord } from "./record.js"; @@ -36,7 +34,6 @@ import { optionalArray, readScannerJson, record, - relativePath, requiredText, sarifRunRepositories, sarifRunResults, @@ -183,6 +180,10 @@ export function normalizeSemgrepFindings( options.sourceRepositories, ); } + if (document["format"] === "semgrep-platform-findings/v1") + throw new Error( + "This Semgrep Platform export uses the retired v1 format. Recollect the findings with cs import semgrep.", + ); if (isSemgrepPlatformFormat(document["format"])) { return normalizeSemgrepPlatformFindings( document, @@ -200,47 +201,25 @@ export function normalizeSemgrepFindings( return normalizeSemgrepJson(document, options, target); } -function nativeProjection(target: ExternalImportTarget, previous: boolean) { - let changed = false; - return { - location(raw: string) { - const oldPath = relativePath(raw); - if (previous) return oldPath; - // Semgrep retains its raw spelling; OpenGrep groups by the normalized path. - const path = nativeResultPath(raw, oldPath) === null ? null : raw; - if (path !== oldPath) changed = true; - return path; - }, - finish(findings: ExternalFindings, omittedExplanations = false) { - if (target === "cloud" && (changed || omittedExplanations)) - markPreviousCloudProjection(findings); - return findings; - }, - }; -} - -/** @internal Rebuild the native JSON or SARIF that an earlier reader approved. */ -export function normalizePreviousSemgrepFindings( - input: unknown, - options: Partial, -): ExternalFindings { - const document = record(input, "Semgrep report"); - return document["version"] === "2.1.0" && document["runs"] !== undefined - ? normalizeSarif(document, options, "cloud", true) - : normalizeSemgrepJson(document, options, "cloud", true); +function semgrepLocation(raw: string) { + // Semgrep retains the scanner's path spelling after validation. + return nativeResultPath(raw) === null ? null : raw; } function normalizeSemgrepJson( document: Record, options: Partial, target: ExternalImportTarget, - previous = false, ): ExternalFindings { const repositoryUrl = sourceRepository(options); requiredText(document["version"], "Semgrep version"); - const { results, paths: _paths, time: _time, ...context } = document; - const omitExplanations = !previous && Object.hasOwn(context, "explanations"); - if (omitExplanations) delete context["explanations"]; + const { + results, + paths: _paths, + time: _time, + explanations: _explanations, + ...context + } = document; const rows = array(results, "Semgrep results"); if (!rows.length && document["errors"] === undefined) throw new Error("An empty Semgrep report must include its errors array."); @@ -248,12 +227,11 @@ function normalizeSemgrepJson( throw new Error( "The Semgrep report contains scan errors; supply a successful report before importing findings.", ); - const projection = nativeProjection(target, previous); - const findings = normalizeRecords( + return normalizeRecords( rows, (value) => projectNativeJsonResult( - parseNativeJsonResult(value, "Semgrep", projection.location), + parseNativeJsonResult(value, "Semgrep", semgrepLocation), { origin: "Semgrep", repositoryUrl, @@ -265,7 +243,6 @@ function normalizeSemgrepJson( jsonIdentity, target, ); - return projection.finish(findings, omitExplanations); } export function isSemgrepSarifDriver(name: string): boolean { @@ -276,15 +253,6 @@ export function normalizeSemgrepSarif( input: unknown, options: SemgrepImportOptions, target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeSarif(input, options, target); -} - -function normalizeSarif( - input: unknown, - options: Partial, - target: ExternalImportTarget, - previous = false, ): ExternalFindings { const repositoryUrl = sourceRepository(options); const repositoryKey = cloudRepositoryUrlKey(repositoryUrl); @@ -309,8 +277,7 @@ function normalizeSarif( throw new Error("Expected a Semgrep SARIF producer."); return sarifRunResults(context, "Semgrep"); }); - const projection = nativeProjection(target, previous); - const findings = normalizeRecords( + return normalizeRecords( entries, ({ result: value, context }) => { const result = record(value, "Semgrep result"); @@ -363,7 +330,7 @@ function normalizeSarif( typeof artifact["uri"] === "string" && /^[a-z][a-z\d+.-]*:/iu.test(artifact["uri"]); const path = - rawPath === null || absoluteUri ? null : projection.location(rawPath); + rawPath === null || absoluteUri ? null : semgrepLocation(rawPath); return validateExternalEvidence( { title: ruleId, @@ -403,7 +370,6 @@ function normalizeSarif( ({ result, context }) => sarifIdentity(result, context.runContext), target, ); - return projection.finish(findings); } /** @internal Read the original file before resolving repository provenance. */ @@ -411,16 +377,12 @@ export async function readSemgrepReport(path: string): Promise { return readScannerJson(path, (json) => { // Pick a parser before allocating the findings array. Verify the root: // the member can be nested in a native report or escaped in a Platform one. - const parser = /"format"\s*:\s*"semgrep-platform-findings\/v[12]"/u.test( - json, - ) + const parser = /"format"\s*:\s*"semgrep-platform-findings\/v2"/u.test(json) ? parseHostedScannerJson : JSON.parse; const parsed: unknown = parser(json); const requiredParser = - isRecord(parsed) && - (parsed["format"] === semgrepPlatformLegacyFormat || - parsed["format"] === semgrepPlatformFormat) + isRecord(parsed) && parsed["format"] === semgrepPlatformFormat ? parseHostedScannerJson : JSON.parse; return parser === requiredParser ? parsed : requiredParser(json); diff --git a/sdk/typescript/src/semgrep-platform-findings.ts b/sdk/typescript/src/semgrep-platform-findings.ts index 874f46bbb9..e839e9d2eb 100644 --- a/sdk/typescript/src/semgrep-platform-findings.ts +++ b/sdk/typescript/src/semgrep-platform-findings.ts @@ -1,6 +1,5 @@ import { DEFAULT_CLOUD_STRING_BYTES, - validateCodeLineRange, canonicalNumber, validateExternalEvidence, type ExternalImportTarget, @@ -21,7 +20,6 @@ import { isGitHubCloudHost } from "./git-remote.js"; import { position, weaknessIds } from "./native-json-findings.js"; import { array, - hostedRepositoryPath, normalizeRecords, record, relativePath, @@ -43,33 +41,17 @@ import { declaredScannerRepositoryUrl, validScannerRepositoryUrl, } from "./scanner-repository-url.js"; -import { - legacyDecimalInteger, - restoreLegacyHostedNumbers, -} from "./hosted-scanner-v1.js"; -import { - semgrepV1Id, - upgradeSemgrepPlatformReport, -} from "./semgrep-platform-v1.js"; import type { SemgrepImportOptions } from "./semgrep-findings.js"; export const semgrepPlatformFormat = "semgrep-platform-findings/v2"; -export const semgrepPlatformLegacyFormat = "semgrep-platform-findings/v1"; export const SEMGREP_PLATFORM_PRODUCTS = ["sast", "sca", "ai_sast"] as const; export type SemgrepPlatformProduct = (typeof SEMGREP_PLATFORM_PRODUCTS)[number]; export const semgrepSelectionFormat = "semgrep-platform-selection/v1"; export function isSemgrepPlatformFormat( format: unknown, -): format is - | typeof semgrepSelectionFormat - | typeof semgrepPlatformFormat - | typeof semgrepPlatformLegacyFormat { - return ( - format === semgrepSelectionFormat || - format === semgrepPlatformFormat || - format === semgrepPlatformLegacyFormat - ); +): format is typeof semgrepSelectionFormat | typeof semgrepPlatformFormat { + return format === semgrepSelectionFormat || format === semgrepPlatformFormat; } function selectedProjectUrl(value: unknown, label: string) { @@ -91,20 +73,17 @@ export function semgrepPlatformReportScope( throw new Error( "A normalized Semgrep Platform report scope requires its source format.", ); - if ( - format === undefined && - provider === "semgrep" && - "sourceRepositoryUrl" in input && - input.sourceRepositoryUrl !== undefined && - (input.findings.length === 0 || - input.findings.some((row) => isSemgrepPlatformEvidence(row, provider))) - ) { - // The object-row SDK retained only the binding on the envelope, including empty reports. - return { - url: hostedRepositoryUrl(input.sourceRepositoryUrl, "Semgrep Platform"), - }; + if (!isSemgrepPlatformFormat(format)) { + if ( + provider === "semgrep" && + typeof format === "string" && + format.startsWith("semgrep-platform-") + ) + throw new Error( + "Unsupported Semgrep Platform format. Recollect the findings.", + ); + return; } - if (!isSemgrepPlatformFormat(format)) return; const scope = record( "semgrepPlatformScope" in input ? input.semgrepPlatformScope : undefined, "Normalized Semgrep report scope", @@ -132,11 +111,8 @@ export function semgrepPlatformReportScope( }; } -export function semgrepPlatformReportRepository( - input: ExternalFindings, - provider?: string, -) { - return semgrepPlatformReportScope(input, provider)?.url; +export function semgrepPlatformReportRepository(input: ExternalFindings) { + return semgrepPlatformReportScope(input)?.url; } export interface SemgrepPlatformScope { @@ -202,7 +178,10 @@ function sourceLine(value: unknown, coerceStrings: boolean): number | null { const token = hostedNumberToken(value); // A long fraction can round to an integer if it is converted before checking its decimal value. if (token !== undefined) - return position(Number(legacyDecimalInteger(token)), "Semgrep source line"); + return position( + Number(semgrepPlatformIdOrUndefined(wireJson.rawJSON(token))), + "Semgrep source line", + ); return position( coerceStrings ? Number(semgrepPlatformId(value)) : value, "Semgrep source line", @@ -244,16 +223,6 @@ export function semgrepPlatformId(value: unknown): string { return id; } -// Older selections retained full decimal ID tokens beyond Semgrep's native int64 range. -function archivedPlatformId(value: unknown): string { - const token = wireJson.isRawJSON(value) - ? canonicalNumber(value.rawJSON, Math.max(21, value.rawJSON.length)) - : value; - return typeof token === "string" && /^[1-9][0-9]*$/u.test(token) - ? token - : semgrepPlatformId(value); -} - function savedPlatformId(value: unknown): string { if (typeof value !== "string") return semgrepPlatformId(value); const canonical = semgrepPlatformId(wireJson.rawJSON(value)); @@ -287,7 +256,6 @@ export function projectAliases( name: string, url: string, references: SemgrepRepositoryReference = [], - earlyV1 = false, ) { const { hostname, pathname } = new URL(url); const path = pathname.slice(1).replace(/\/$/u, ""); @@ -318,53 +286,12 @@ export function projectAliases( ]), ); const shortName = name.slice(name.lastIndexOf("/") + 1); - const historicalPaths: string[] = []; - if (earlyV1) { - // Older exports used decodeURI, which preserves escaped reserved delimiters. - for (const historical of [path, decodePath(decodeURI)]) { - if (!publicGitHost(hostname)) historicalPaths.push(historical); - if (publicHost || source.platform === "github") - historicalPaths.push(historical.replace(/\.git$/u, "")); - } - } - const hasHistoricalPath = (label: string) => - historicalPaths.some( - (path) => - label === path || label === path.slice(path.lastIndexOf("/") + 1), - ); - return { - hasHistoricalPath, - has: (label: string) => - label === name || - label === shortName || - pathKeys.has(key(label)) || - hasHistoricalPath(label), - }; -} - -/** Older captures could derive a row label from their verified HTTP mapping for an SSH project. */ -export function legacyProjectAliases( - name: string, - url: string, - references: SemgrepRepositoryReference, - mapping: string | undefined, -) { - const native = projectAliases(name, url, references, true); - const mapped = - mapping && mapping !== url - ? projectAliases(name, mapping, references, true) - : undefined; return { has: (label: string) => - native.has(label) || !!mapped?.hasHistoricalPath(label), + label === name || label === shortName || pathKeys.has(key(label)), }; } -/** Compare HTTP origins canonically without folding paths on independent custom Git hosts. */ -export function semgrepRepositoryUrlKey(value: string): string { - return hostedRepositoryUrlKey(nativeHttp(value)); -} - export type SemgrepRepositoryReference = readonly ExternalSourceRepository[] | ((url: string) => string); @@ -468,13 +395,6 @@ export function semgrepNativeProjectUrl( return hostedNativeRepositoryUrl(value, label); } -function referenceRepositoryKey( - references: readonly ExternalSourceRepository[], - target: ExternalImportTarget = "local", -) { - return (url: string) => repositoryContext(url, references, target).key(url); -} - function repositoryContext( projectUrl: string, references: SemgrepRepositoryReference = [], @@ -812,24 +732,23 @@ export function normalizeSelectedSemgrepPlatformFindings( scope, "local", ), - sourceRepositoryUrl: scope.project.url, + ...reportScope(semgrepSelectionFormat, scope), }; } function sourceId( scope: SemgrepPlatformScope, finding: Record, - readId = semgrepPlatformId, ) { // Keep the same tuple for a selected import and a subsequent scoped Platform collection. const exact = (id: string) => Number.isSafeInteger(Number(id)) ? Number(id) : id; return stableId("semgrep-platform", [ scope.deployment, - exact(readId(wireJson.rawJSON(scope.project.id))), + exact(semgrepPlatformId(wireJson.rawJSON(scope.project.id))), scope.product, scope.ref, - exact(readId(finding["id"])), + exact(semgrepPlatformId(finding["id"])), ]); } @@ -847,9 +766,7 @@ function readScope( references: SemgrepRepositoryReference, target: ExternalImportTarget = "local", ) { - const complete = - input["format"] === semgrepPlatformFormat || - input["format"] === semgrepPlatformLegacyFormat; + const complete = input["format"] === semgrepPlatformFormat; const product = scopeProduct(input["product"], complete); const project = responseRecord(input["project"], "Semgrep project"); if ( @@ -925,11 +842,11 @@ function matchesNativeInteger( if (token === undefined) return value === expected; return expected === 0 ? /^-?0(?:\.0+)?(?:e[+-]?\d+)?$/iu.test(token) - : legacyDecimalInteger(token) === String(expected); + : canonicalNumber(token) === String(expected); } const generatedId = /^semgrep-platform:[0-9a-f]{64}$/u; -/** Current occurrences carry a provider-specific key; older v1 rows retain their native scope and finding. */ +/** Recognize reserved Platform IDs and native scope before accepting normalized evidence. */ export function isSemgrepPlatformEvidence( finding: ExternalFinding, provider?: string, @@ -958,152 +875,21 @@ export function isSemgrepPlatformEvidence( ); } -function upgradeOriginalSemgrepFinding( +function nativeOccurrence( data: Record, references: SemgrepRepositoryReference, target: ExternalImportTarget, -): Record { - const { - numericFields: _witness, - finding: original, - ...context - } = restoreLegacyHostedNumbers(data); - const converted = upgradeSemgrepPlatformReport( - { ...context, findings: [original], findingCount: 1 }, - undefined, - typeof references === "function" - ? references - : referenceRepositoryKey(references, target), - references, - ); - return { - ...converted, - semgrepPlatformFindingJson: array( - converted["findings"], - "Semgrep findings", - )[0], - }; -} - -/** Keep caller-supplied metadata while replacing the scanner's old wire representation. */ -function canonicalSourceDataWithExtensions( - retained: Record, - canonical: ReturnType, - original: boolean, ) { - const extensions = ( - original: Record, - verified: Record, - replaced: string[] = [], - ) => - Object.fromEntries( - Object.entries(original).filter( - ([key]) => !Object.hasOwn(verified, key) && !replaced.includes(key), - ), + if (!isSemgrepPlatformFormat(data["format"])) + throw new Error( + "Unsupported Semgrep finding format. Recollect the Platform findings.", ); - return { - ...canonical, - ...extensions(retained, canonical, [ - ...(original ? ["finding", "numericFields", "sourceRepositoryUrl"] : []), - "nativeProjectUrl", - "repositoryAliases", - "repositoryWitnessJson", - "idEncoding", - ]), - project: { - ...canonical.project, - ...extensions( - record(retained["project"], "Semgrep project"), - canonical.project, - ), - }, - }; -} - -function legacyWitnessKey( - scope: SemgrepPlatformScope, - projectUrlKey: string, - label: string, -) { - return JSON.stringify([ - scope.deployment, - scope.project.id, - scope.product, - scope.ref, - projectUrlKey, - label, - ]); -} - -function nativeOccurrence( - retained: Record, - references: SemgrepRepositoryReference, - target: ExternalImportTarget, -) { - let data = !isSemgrepPlatformFormat(retained["format"]) - ? { - ...retained, - format: - (!Object.hasOwn(retained, "semgrepPlatformFindingJson") && - "finding" in retained) || - retained["idEncoding"] === "v1-decimal" - ? semgrepPlatformLegacyFormat - : "repositoryAliases" in retained - ? semgrepPlatformFormat - : semgrepSelectionFormat, - } - : retained; - const legacy = data["format"] === semgrepPlatformLegacyFormat; - const original = legacy && !Object.hasOwn(data, "semgrepPlatformFindingJson"); - if (original) data = upgradeOriginalSemgrepFinding(data, references, target); const { scope, aliases } = readScope(data, references, target); const repository = repositoryContext(scope.project.url, references, target); const row = parseSemgrepResponse( requiredText(data["semgrepPlatformFindingJson"], "Semgrep finding JSON"), ); - return { data, legacy, original, scope, aliases, repository, row }; -} - -/** Original v1 occurrences could keep only the observed repository label on a row. */ -export function semgrepLegacyRepositoryWitnesses( - findings: readonly ExternalFinding[], - provider: string, - references: SemgrepRepositoryReference = [], - target: ExternalImportTarget = "local", -): ReadonlyMap { - const witnesses = new Map(); - // Current occurrences carry their own witness. Only original rows need a sibling lookup. - if ( - !findings.some(({ evidence }) => - isRecord(evidence.source_data?.["finding"]), - ) - ) - return witnesses; - for (const finding of findings) { - const data = finding.evidence.source_data; - if (!data || !isSemgrepPlatformEvidence(finding, provider)) continue; - const { scope, aliases, repository, row, legacy } = nativeOccurrence( - data, - references, - target, - ); - const repo = row["repository"]; - if (!isRecord(repo)) continue; - const label = text(repo["name"]); - if (!label || (repo["id"] == null && !text(repo["url"]))) continue; - checkSemgrepPlatformRow( - row, - scope, - aliases, - repository, - legacy ? semgrepV1Id : semgrepPlatformId, - ); - witnesses.set( - legacyWitnessKey(scope, repository.key(scope.project.url), label), - JSON.stringify({ id: scope.project.id, name: label }), - ); - } - return witnesses; + return { scope, aliases, repository, row }; } /** Recheck the source repository when an SDK caller supplies normalized evidence directly. */ @@ -1111,14 +897,8 @@ export function semgrepPlatformFindingRepository( finding: ExternalFinding, provider: string, references: SemgrepRepositoryReference = [], - legacyWitnesses?: ReadonlyMap, ): string | undefined { - return semgrepPlatformFindingBinding( - finding, - provider, - references, - legacyWitnesses, - )?.url; + return semgrepPlatformFindingBinding(finding, provider, references)?.url; } export function platformLocationPath(value: unknown) { @@ -1132,103 +912,28 @@ export function platformLocationPath(value: unknown) { function platformLocation( location: Record, product: SemgrepPlatformScope["product"], - selectable: boolean, - bulk?: { legacy: boolean }, + coerceStrings: boolean, ) { const path = platformLocationPath(location["file_path"]); - const oldPath = bulk?.legacy - ? hostedRepositoryPath(location["file_path"]) - : null; - const unprojected = { locations: [], endLine: null }; - if (!path && !oldPath) return unprojected; - // V1 retained coordinates for unprojected paths without validating them. - const newlyProjectable = !!bulk?.legacy && !oldPath; - const coerce = newlyProjectable || (!!bulk && !bulk.legacy && !selectable); - try { - const line = sourceLine(location["line"], coerce); - const endLine = - line !== null && product !== "sca" - ? sourceLine(location["end_line"], coerce) - : null; - if (bulk?.legacy && oldPath !== path) validateCodeLineRange(line, endLine); - return path ? { locations: [{ path, line }], endLine } : unprojected; - } catch (error) { - if (newlyProjectable) return unprojected; - throw error; - } + if (!path) return { locations: [], endLine: null }; + const line = sourceLine(location["line"], coerceStrings); + const endLine = + line !== null && product !== "sca" + ? sourceLine(location["end_line"], coerceStrings) + : null; + return { locations: [{ path, line }], endLine }; } -function platformSeverity(value: unknown, selectable: boolean) { +function platformSeverity( + value: unknown, + selectable: boolean, +): ExternalFinding["evidence"]["severity"] { const severity = text(value)?.toLowerCase(); return scannerSeverity( selectable && severity === "informational" ? null : severity, ); } -/** Upgrade only values produced by the original object-row reader; keep SDK-authored evidence. */ -function migrateLegacyProjection( - evidence: ExternalFinding["evidence"], - row: Record, - scope: SemgrepPlatformScope, - selectable: boolean, - references: SemgrepRepositoryReference, -) { - const severity = text(row["severity"]); - const oldSeverity = scannerSeverity( - severity === "informational" ? null : severity, - ); - const upgradeSeverity = - evidence.severity === oldSeverity && - oldSeverity !== scannerSeverity(severity?.toLowerCase()); - const location = isRecord(row["location"]) ? row["location"] : {}; - const oldPath = hostedRepositoryPath(location["file_path"]); - const first = evidence.locations?.[0]; - // Only replace the exact location and end line produced by the original object-row reader. - const oldLocation = - oldPath !== platformLocationPath(location["file_path"]) && - (oldPath - ? evidence.locations?.length === 1 && - first?.path === oldPath && - matchesNativeInteger(location["line"], first.line) - : evidence.locations?.length === 0) && - matchesNativeInteger( - oldPath && scope.product !== "sca" && first?.line != null - ? location["end_line"] - : null, - evidence.details?.code?.end_line, - ); - if (!upgradeSeverity && !oldLocation) return; - const projected = normalizePlatformRows( - [{ finding: row, selectable }], - scope, - "local", - { - legacy: true, - projectNames: projectAliases( - scope.project.name, - scope.project.url, - references, - ), - }, - ).findings[0]?.evidence; - if (!projected) return; - return { - ...(upgradeSeverity && { severity: projected.severity }), - ...(oldLocation && { locations: projected.locations }), - ...(oldLocation && - evidence.details?.code && - projected.details?.code && { - details: { - ...evidence.details, - code: { - ...evidence.details.code, - end_line: projected.details.code.end_line, - }, - }, - }), - }; -} - function qualifiedBranch(ref: string) { return ref.startsWith("refs/") ? ref : `refs/heads/${ref}`; } @@ -1275,19 +980,15 @@ export function semgrepPlatformFindingBinding( finding: ExternalFinding, provider: string, references: SemgrepRepositoryReference = [], - legacyWitnesses?: ReadonlyMap, reportScope?: ReturnType, projectedSources: readonly ( { url?: string; branch?: string } | undefined )[] = [], target: ExternalImportTarget = "local", - replayApprovedRequest = false, ) { const retained = finding.evidence.source_data; if (!isSemgrepPlatformEvidence(finding, provider)) { - // Previously approved Cloud requests keep the released projection and native checks below. if ( - !replayApprovedRequest && provider === "semgrep" && reportScope?.format !== undefined && hasRetainedSemgrepScope(retained?.["finding"]) @@ -1328,8 +1029,11 @@ export function semgrepPlatformFindingBinding( } if (!retained) throw new Error("Missing the original Semgrep finding evidence."); - const { data, legacy, original, scope, aliases, repository, row } = - nativeOccurrence(retained, references, target); + const { scope, aliases, repository, row } = nativeOccurrence( + retained, + references, + target, + ); checkProjectedProduct(finding, scope.product, true); if ( reportScope?.deployment !== undefined && @@ -1342,58 +1046,32 @@ export function semgrepPlatformFindingBinding( "The Semgrep report repository scope does not match the finding's deployment, project ID, ref, or product.", ); - const other = data["finding"]; - if ( - (legacy && Object.hasOwn(data, "finding")) || - (isRecord(other) && Object.hasOwn(other, "id")) - ) { - const restored = restoreLegacyHostedNumbers(data)["finding"]; - const originalRow = parseSemgrepResponse(JSON.stringify(restored)); - const id = originalRow["id"]; - const nativeId = - semgrepPlatformIdOrUndefined(id) ?? - (typeof id === "string" ? legacyDecimalInteger(id) : undefined); + const other = retained["finding"]; + if (isRecord(other) && Object.hasOwn(other, "id")) { + const original = parseSemgrepResponse(JSON.stringify(other)); if ( - (legacy || nativeId !== undefined) && - stringifySemgrepFinding(originalRow) !== stringifySemgrepFinding(row) + semgrepPlatformIdOrUndefined(original["id"]) !== undefined && + stringifySemgrepFinding(original) !== stringifySemgrepFinding(row) ) throw new Error("The retained Semgrep finding representations disagree."); } - const selectable = checkSemgrepPlatformRow( - row, - scope, - aliases, - repository, - legacy ? semgrepV1Id : semgrepPlatformId, - ); + const selectable = checkSemgrepPlatformRow(row, scope, aliases, repository); if (reportScope?.format === semgrepSelectionFormat && !selectable) throw new Error( "Semgrep finding does not belong to the selected project and ref.", ); - let occurrenceId = legacy ? semgrepV1Id : semgrepPlatformId; - const nativeId = sourceId(scope, row, occurrenceId); + const nativeId = sourceId(scope, row); if ( generatedId.test(finding.source_finding_id) && finding.source_finding_id !== nativeId - ) { - if ( - legacy || - finding.source_finding_id !== sourceId(scope, row, archivedPlatformId) - ) - throw new Error( - "The generated Semgrep finding ID does not match its native occurrence.", - ); - occurrenceId = archivedPlatformId; - } + ) + throw new Error( + "The generated Semgrep finding ID does not match its native occurrence.", + ); const { name, url } = scope.project; let namedOnly = aliases && repositoryWithOnlyAlias(row, projectAliases(name, url)); - const raw = original - ? namedOnly && - legacyWitnesses?.get( - legacyWitnessKey(scope, repository.key(url), namedOnly), - ) - : data["repositoryWitnessJson"]; + const raw = retained["repositoryWitnessJson"]; // Only rely on the reader's platform hint when there is no independent proof. if ( namedOnly && @@ -1401,13 +1079,7 @@ export function semgrepPlatformFindingBinding( projectAliases(name, url, references).has(namedOnly) ) namedOnly = undefined; - const legacyPath = - original && - namedOnly && - legacyProjectAliases(name, url, references, scope.sourceRepositoryUrl).has( - namedOnly, - ); - if (raw !== undefined || (namedOnly && !legacyPath)) { + if (raw !== undefined || namedOnly) { const label = text( record(row["repository"], "Semgrep finding repository")["name"], ); @@ -1427,9 +1099,7 @@ export function semgrepPlatformFindingBinding( scope, aliases ?? new Set([label]), repository, - legacy - ? semgrepV1Id - : (id) => semgrepPlatformId(wireJson.rawJSON(savedPlatformId(id))), + (id) => semgrepPlatformId(wireJson.rawJSON(savedPlatformId(id))), ); } const claimed = finding.evidence.details?.repository?.url; @@ -1448,57 +1118,36 @@ export function semgrepPlatformFindingBinding( row, scope, selectable, - aliases ? { legacy } : undefined, + aliases !== undefined, namedOnly, - occurrenceId, ); - let migration = original - ? migrateLegacyProjection( - finding.evidence, - row, - scope, - selectable, - references, - ) - : undefined; - if ( - !original && - data["format"] === semgrepPlatformFormat && - canonical.format === semgrepSelectionFormat - ) { - const selectedSeverity = platformSeverity(row["severity"], true); - if ( - selectedSeverity !== finding.evidence.severity && - finding.evidence.severity === platformSeverity(row["severity"], false) - ) - migration = { severity: selectedSeverity }; - } - // The original reader projected the HTTP mapping when the native project URL used SSH. - const oldMappedUrl = - original && - scope.product !== "sca" && - scope.nativeProjectUrl !== undefined && - claimed === scope.sourceRepositoryUrl && - claimed !== scope.project.url; - const details = migration?.details ?? finding.evidence.details; + const { + repositoryAliases: _aliases, + repositoryWitnessJson: _witness, + nativeProjectUrl: _native, + ...metadata + } = retained; + const selectedSeverity = platformSeverity(row["severity"], true); + const previousSeverity = platformSeverity(row["severity"], false); return { - url: scope.project.url, + url, nativeId, + // A reader-provided repository binding can promote a backlog row to a selected SAST row. migration: - oldMappedUrl && details?.repository - ? { - ...migration, - details: { - ...details, - repository: { ...details.repository, url: scope.project.url }, - }, - } - : migration, - sourceData: canonicalSourceDataWithExtensions( - retained, - canonical, - original, - ), + retained["format"] === semgrepPlatformFormat && + canonical.format === semgrepSelectionFormat && + finding.evidence.severity === previousSeverity && + previousSeverity !== selectedSeverity + ? { severity: selectedSeverity } + : undefined, + sourceData: { + ...metadata, + ...canonical, + project: { + ...record(retained["project"], "Semgrep project"), + ...canonical.project, + }, + }, }; } @@ -1513,20 +1162,10 @@ export function normalizeSemgrepPlatformFindings( ) { const declaredUrl = typeof options === "string" ? options : options.sourceRepositoryUrl; - const saved = record(report, "Semgrep Platform report"); - const sourceFormat = saved["format"]; + const input = record(report, "Semgrep Platform report"); + const sourceFormat = input["format"]; if (!isSemgrepPlatformFormat(sourceFormat)) throw new Error("Expected a Semgrep Platform selection or project export."); - const legacy = sourceFormat === semgrepPlatformLegacyFormat; - const readId = legacy ? semgrepV1Id : semgrepPlatformId; - const input = legacy - ? upgradeSemgrepPlatformReport( - saved, - declaredUrl, - referenceRepositoryKey(references, target), - references, - ) - : saved; const { scope, aliases } = readScope(input, references, target); const repository = selectionRepository(scope.project.url, references, target); if ( @@ -1556,7 +1195,6 @@ export function normalizeSemgrepPlatformFindings( scope, aliases, repository, - readId, ); if (aliases) { const repo = record( @@ -1574,17 +1212,8 @@ export function normalizeSemgrepPlatformFindings( "Incomplete Semgrep Platform export: finding count does not match.", ); if (aliases) { - // v1 also accepted decodeURI path labels. Keep a witness for their current-format replay. - const archiveNames = legacy - ? legacyProjectAliases( - scope.project.name, - scope.project.url, - references, - scope.sourceRepositoryUrl, - ) - : projectNames; for (const { finding } of rows) { - const name = repositoryWithOnlyAlias(finding, archiveNames); + const name = repositoryWithOnlyAlias(finding, projectNames); if (name && !verifiedLabels.has(name)) throw new Error( "Semgrep finding repository lacks a matching project name, ID or URL.", @@ -1596,21 +1225,19 @@ export function normalizeSemgrepPlatformFindings( rows, scope, target, - aliases - ? { - legacy, - projectNames: { - has: (name) => - (verifiedLabels.has(name) ? unhintedNames : projectNames).has( - name, - ), - }, - } - : undefined, - aliases ? readId : archivedPlatformId, + aliases && { + has: (name) => + (verifiedLabels.has(name) ? unhintedNames : projectNames).has(name), + }, ); + return { ...result, ...reportScope(sourceFormat, scope) }; +} + +function reportScope( + sourceFormat: typeof semgrepSelectionFormat | typeof semgrepPlatformFormat, + scope: SemgrepPlatformScope, +) { return { - ...result, sourceFormat, sourceRepositoryUrl: scope.project.url, semgrepPlatformScope: { @@ -1624,13 +1251,14 @@ export function normalizeSemgrepPlatformFindings( export function stringifySemgrepFinding( finding: Record, - idOf: (id: unknown) => string | undefined = semgrepPlatformIdOrUndefined, ): string { const repo = finding["repository"]; const nativeIds = [finding["id"], isRecord(repo) ? repo["id"] : undefined]; return JSON.stringify(finding, (_key, value: unknown) => { if (wireJson.isRawJSON(value)) { - const id = nativeIds.includes(value) ? idOf(value) : undefined; + const id = nativeIds.includes(value) + ? semgrepPlatformIdOrUndefined(value) + : undefined; return wireJson.rawJSON(id ?? canonicalNumber(value.rawJSON)); } if (isRecord(value)) @@ -1647,37 +1275,22 @@ function platformSourceData( finding: Record, scope: ResolvedScope, selectable: boolean, - bulk?: { legacy: boolean }, + bulk: boolean, namedOnly?: string | null, - idOf = semgrepPlatformId, ) { const repo = record( finding["repository"] ?? {}, "Semgrep finding repository", ); const label = text(repo["name"]); - // Ordinary v1 and v2 rows share one stored representation on replay. - // Preserve the legacy discriminator only when reopening the raw IDs needs it. const { nativeProjectUrl, ...evidenceScope } = scope; const nativeBulkScope = !selectable || scope.sourceRepositoryUrl !== undefined || nativeProjectUrl !== undefined; - const needsLegacyId = - bulk?.legacy && - [finding["id"], repo["id"]].some( - (id) => - typeof id === "string" && - semgrepPlatformIdOrUndefined(id) === undefined, - ); - // Native SAST rows with their own repository and ref replay identically in either import mode. - const needsBulkScope = bulk && (nativeBulkScope || needsLegacyId); + const needsBulkScope = bulk && nativeBulkScope; return { - format: needsLegacyId - ? semgrepPlatformLegacyFormat - : needsBulkScope - ? semgrepPlatformFormat - : semgrepSelectionFormat, + format: needsBulkScope ? semgrepPlatformFormat : semgrepSelectionFormat, ...evidenceScope, project: { ...scope.project, @@ -1685,7 +1298,6 @@ function platformSourceData( nativeProjectUrl ?? (scope.sourceRepositoryUrl ? null : scope.project.url), }, - ...(needsLegacyId && { idEncoding: "v1-decimal" }), ...(needsBulkScope && { repositoryAliases: label ? [label] : [] }), // Store the verified binding independently of the native fields retained on its proving row. ...(namedOnly && { @@ -1694,7 +1306,7 @@ function platformSourceData( name: namedOnly, }), }), - semgrepPlatformFindingJson: stringifySemgrepFinding(finding, idOf), + semgrepPlatformFindingJson: stringifySemgrepFinding(finding), }; } @@ -1703,13 +1315,8 @@ function normalizePlatformRows( rows: { finding: Record; selectable: boolean }[], scope: ResolvedScope, target: ExternalImportTarget, - bulk?: { - legacy: boolean; - projectNames: Pick, "has">; - }, - readId = bulk?.legacy ? semgrepV1Id : semgrepPlatformId, + projectNames?: Pick, "has">, ) { - const legacy = bulk?.legacy; return normalizeRecords( rows, ({ finding, selectable }) => { @@ -1724,11 +1331,10 @@ function normalizePlatformRows( const { locations, endLine } = platformLocation( location, scope.product, - selectable, - bulk, + !!projectNames && !selectable, ); const namedOnly = - bulk && repositoryWithOnlyAlias(finding, bulk.projectNames); + projectNames && repositoryWithOnlyAlias(finding, projectNames); const common = { description, severity: platformSeverity(finding["severity"], selectable), @@ -1738,23 +1344,22 @@ function normalizePlatformRows( finding, scope, selectable, - bulk, + projectNames !== undefined, namedOnly, - readId, ), }; let messageTitle = description ?? - (legacy || scope.product === "sast" + (scope.product === "sast" ? null - : `Semgrep finding ${readId(finding["id"])}`); + : `Semgrep finding ${semgrepPlatformId(finding["id"])}`); if ( target === "cloud" && description && Buffer.byteLength(description) > DEFAULT_CLOUD_STRING_BYTES ) { const id = finding["id"]; - messageTitle = `Semgrep finding ${text(id) ?? readId(id)}`; + messageTitle = `Semgrep finding ${text(id) ?? semgrepPlatformId(id)}`; } if (scope.product === "sca") { const dependency = responseRecord( @@ -1815,7 +1420,7 @@ function normalizePlatformRows( target, ); }, - ({ finding }) => sourceId(scope, finding, readId), + ({ finding }) => sourceId(scope, finding), target, ); } diff --git a/sdk/typescript/src/semgrep-platform-v1.ts b/sdk/typescript/src/semgrep-platform-v1.ts deleted file mode 100644 index 31f2a63037..0000000000 --- a/sdk/typescript/src/semgrep-platform-v1.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { hostedNumberToken } from "./hosted-scanner-http.js"; -import { hostedRepositoryUrl } from "./hosted-scanner-http.js"; -import { - legacyDecimalInteger, - restoreLegacyHostedNumbers, -} from "./hosted-scanner-v1.js"; -import { array, record, requiredText, text } from "./scanner-import.js"; -import { - legacyProjectAliases, - semgrepNativeProjectUrl, - semgrepProjectUrlEvidence, - type SemgrepRepositoryReference, - semgrepPlatformId, - semgrepRepositoryUrlKey, -} from "./semgrep-platform-findings.js"; - -/** Early v1 exports also admitted positive integral decimal spellings stored as text. */ -export function semgrepV1Id(value: unknown): string { - if (typeof value === "number" && Number.isSafeInteger(value)) - return semgrepPlatformId(String(value)); - const token = hostedNumberToken(value); - if (token !== undefined && !/^[1-9][0-9]*$/u.test(token)) - return semgrepPlatformId(legacyDecimalInteger(token)); - return semgrepPlatformId( - typeof value === "string" ? legacyDecimalInteger(value) : value, - ); -} - -function isLossyNumber(value: unknown): boolean { - return ( - typeof value === "number" && - (!Number.isFinite(value) || - (Number.isInteger(value) && !Number.isSafeInteger(value))) - ); -} - -/** Translate a saved object-row export into lossless per-finding JSON for the current reader. */ -export function upgradeSemgrepPlatformReport( - input: Record, - declaredUrl?: string, - repositoryKey = semgrepRepositoryUrlKey, - references: SemgrepRepositoryReference = [], -): Record { - const report = restoreLegacyHostedNumbers(input); - const project = record(report["project"], "Semgrep project"); - const name = requiredText(project["name"], "Semgrep project name"); - const mapping = report["sourceRepositoryUrl"] ?? declaredUrl; - const bound = - mapping === undefined - ? undefined - : hostedRepositoryUrl(mapping, "Semgrep source repository"); - const native = semgrepNativeProjectUrl(project["url"], references); - const url = hostedRepositoryUrl( - native ?? bound, - "Semgrep project repository", - ); - for (const other of [bound, declaredUrl]) { - if ( - other !== undefined && - repositoryKey(hostedRepositoryUrl(other, "Semgrep repository")) !== - repositoryKey(url) - ) - throw new Error( - "Semgrep project does not match the declared source repository URL.", - ); - } - // For a resolved native URL, retain only the mapping saved in the report. - const urlEvidence = semgrepProjectUrlEvidence( - project["url"], - native, - native === undefined || report["sourceRepositoryUrl"] != null - ? bound - : undefined, - ); - const entries = array(report["findings"], "Semgrep findings"); - let aliases = report["repositoryAliases"]; - if (aliases === undefined) { - const derived = legacyProjectAliases( - name, - url, - references, - urlEvidence.sourceRepositoryUrl, - ); - const observed = new Set(); - for (const entry of entries) { - const row = record(entry, "Semgrep finding"); - const repo = record( - row["repository"] ?? {}, - "Semgrep finding repository", - ); - const label = text(repo["name"]); - if (!label) continue; - // Without the later alias list, each row must independently identify the project. - if ( - repo["id"] == null && - (repo["url"] == null || repo["url"] === "") && - !derived.has(label) - ) - throw new Error( - "Semgrep finding repository does not match the selected project.", - ); - observed.add(label); - } - aliases = Array.from(observed); - } - return { - format: "semgrep-platform-findings/v2", - deployment: report["deployment"], - project: { id: semgrepV1Id(project["id"]), name, url }, - ...urlEvidence, - product: report["product"], - ref: report["ref"], - findingCount: report["findingCount"], - repositoryAliases: aliases, - findings: entries.map((entry) => { - const row = record(entry, "Semgrep finding"); - const repository = record( - row["repository"] ?? {}, - "Semgrep finding repository", - ); - if ([row["id"], repository["id"]].some(isLossyNumber)) - throw new Error("Expected an exact, positive Semgrep Platform ID."); - return JSON.stringify(row, (_key, value: unknown) => { - if (isLossyNumber(value)) - throw new Error( - "Cannot retain an unsafe Semgrep finding number. Read the original Semgrep export to preserve its JSON numbers.", - ); - return value; - }); - }), - }; -} diff --git a/sdk/typescript/src/snyk-findings.ts b/sdk/typescript/src/snyk-findings.ts index 56b757dd7c..e8efb15765 100644 --- a/sdk/typescript/src/snyk-findings.ts +++ b/sdk/typescript/src/snyk-findings.ts @@ -4,12 +4,6 @@ import type { ExternalFindings, ExternalSourceRepository, } from "./external-findings.js"; -import { - encodeSnykV1File, - isSnykRestFormat, - parseSnykReport, - SNYK_REST_ISSUES_V1, -} from "./snyk-rest-v1.js"; import type { SnykIssuesFindings } from "./snyk-rest-findings.js"; import { normalizeHostedGitRepositoryUrl } from "./hosted-git-repository.js"; import { isRecord } from "./record.js"; @@ -534,9 +528,9 @@ export function normalizeSnykJson( ); } -/** @internal Keep the native JSON snapshot for catalog-bound file publication. */ -export function readSnykReport(path: string): Promise { - return readScannerJson(path, parseSnykReport); +export const SNYK_REST_ISSUES_FORMAT = "snyk-rest-issues/v2"; +export function isSnykRestFormat(format: unknown): boolean { + return typeof format === "string" && format.startsWith("snyk-rest-issues/"); } export async function readSnykFindings( @@ -544,7 +538,7 @@ export async function readSnykFindings( target: ExternalImportTarget = "cloud", references: readonly ExternalSourceRepository[] = [], ): Promise> { - return normalizeSnykFile(await readSnykReport(path), target, references); + return normalizeSnykFile(await readScannerJson(path), target, references); } /** @internal Dispatch a parsed Snyk file using the same rules as readSnykFindings. */ @@ -557,13 +551,7 @@ export async function normalizeSnykFile( const document = record(input, "Snyk findings"); if (isSnykRestFormat(document["format"])) { const { normalizeSnykIssues } = await import("./snyk-rest-findings.js"); - return normalizeSnykIssues( - document["format"] === SNYK_REST_ISSUES_V1 - ? encodeSnykV1File(document) - : document, - target, - references, - ); + return normalizeSnykIssues(document, target, references); } if (document["format"] === "snyk-findings-export/v1") return normalizeSnykFindingsExport(document, target); diff --git a/sdk/typescript/src/snyk-rest-findings.ts b/sdk/typescript/src/snyk-rest-findings.ts index 95af2bbfe6..70ba224a6f 100644 --- a/sdk/typescript/src/snyk-rest-findings.ts +++ b/sdk/typescript/src/snyk-rest-findings.ts @@ -1,4 +1,5 @@ import { + canonicalNumber, validateExternalEvidence, type ExternalImportTarget, } from "./external-import-contract.js"; @@ -10,14 +11,10 @@ import { } from "./external-findings.js"; import type { ImportedPackage, Provider } from "./external-import-models.js"; import { - SNYK_REST_ISSUES_FORMAT, - encodeSnykV1File, isSnykRestFormat, - SNYK_REST_ISSUES_V1, - originalSnykEvidence, - readSnykRestDocument, -} from "./snyk-rest-v1.js"; -import { snykRepositoryUrls } from "./snyk-findings.js"; + SNYK_REST_ISSUES_FORMAT, + snykRepositoryUrls, +} from "./snyk-findings.js"; import { isGitHubCloudHost } from "./git-remote.js"; import { gitPathToHttpPath, @@ -38,7 +35,6 @@ import { type HostedScannerHttpOptions, } from "./hosted-scanner-http.js"; import { isRecord } from "./record.js"; -import { legacyDecimalInteger } from "./hosted-scanner-v1.js"; import { isSafeNonNegativeInteger } from "./value.js"; import { checkSnykTokenOrigin, snykApiToken } from "./snyk-token-origin.js"; import { regexes } from "zod/v4/core"; @@ -60,7 +56,7 @@ export interface SnykIssuesFindings extends ExternalFindings { sourceRepositoryPlatform?: "github" | "other"; snykRestContext?: Record; } -export { SNYK_REST_ISSUES_FORMAT, isSnykRestFormat } from "./snyk-rest-v1.js"; +export { isSnykRestFormat, SNYK_REST_ISSUES_FORMAT }; const issueTimestamp = new RegExp( regexes.datetime({ offset: true }).source, "i", @@ -359,8 +355,6 @@ function trustedScmBinding( return { url }; } -class SnykTargetRepositoryMismatch extends Error {} - function checkRemote( remote: unknown, selected: Scope, @@ -397,7 +391,7 @@ function checkRemote( gitlab, }; if (!snykRestRepositoryMatches(binding, url)) - throw new SnykTargetRepositoryMismatch( + throw new Error( "Snyk target URL does not match the selected source repository URL.", ); return { @@ -906,121 +900,21 @@ function linkedSnykResources( return {}; } -function retainedRelationship( - resource: Resource | undefined, - name: "target" | "integration" | "organization", -) { - const links = resource?.["relationships"]; - const relation = isRecord(links) ? links[name] : undefined; - const data = isRecord(relation) ? relation["data"] : undefined; - if (data === null) return null; - if ( - isRecord(data) && - (data["type"] === name || - (name === "organization" && data["type"] === "org")) - ) - return data; -} - -/** Read only vendor-typed claims still present in an already-approved request. */ -function retainedSnykAssets(data: Resource) { - const assets = []; - const id = (resource?: Resource | null) => { - const value = resource?.["id"]; - return isUuid(value) ? value.toLowerCase() : undefined; - }; - for (const json of [false, true]) { - const project = originalSnykResource(data, "project", json); - let expanded; - if (project?.["type"] === "project") { - const attributes = project["attributes"]; - const origin = isRecord(attributes) ? attributes["origin"] : undefined; - expanded = retainedRelationship(project, "target"); - assets.push({ - projectId: id(project), - product: projectProduct( - isRecord(attributes) ? attributes["type"] : undefined, - ), - targetId: expanded === null ? null : undefined, - organizationId: id(retainedRelationship(project, "organization")), - origin: - typeof origin === "string" && scmOrigins.has(origin) - ? origin - : undefined, - }); - } - for (const target of [ - expanded, - originalSnykResource(data, "target", json), - ]) { - if (target?.["type"] !== "target") continue; - const attributes = target["attributes"]; - const url = isRecord(attributes) ? attributes["url"] : undefined; - const integration = retainedRelationship(target, "integration"); - const integrationAttributes = integration?.["attributes"]; - const type = isRecord(integrationAttributes) - ? integrationAttributes["integration_type"] - : undefined; - assets.push({ - url: typeof url === "string" ? url : undefined, - targetId: id(target), - integrationId: id(integration), - deployment: - typeof type === "string" - ? concreteDeployment(scmOrigins.get(type)) - : undefined, - organizationId: id(retainedRelationship(target, "organization")), - }); - } - } - return assets; -} - function snykRowEvidence(finding: ExternalFinding, hostedReport = false) { const retained = finding.evidence.source_data; const marked = snykRestId.test(finding.source_finding_id) || isSnykRestFormat(retained?.["format"]); const hosted = hostedReport || marked; - const v1 = retained && linkedSnykResources(retained, false, hosted); - const v2 = retained && linkedSnykResources(retained, true, hosted); - if (v1 || v2 || marked) return { v1, v2 }; + const original = retained && linkedSnykResources(retained, true, hosted); + const retired = retained && linkedSnykResources(retained, false, hosted); + if (original || retired || marked) return { ...original, retired }; } export function isSnykRestFinding(finding: ExternalFinding): boolean { return snykRowEvidence(finding) !== undefined; } -/** Recover the native occurrence from an earlier normalized submission with a caller-assigned ID. */ -export function snykRestCheckpointFindingId( - finding: ExternalFinding, - references: readonly ExternalSourceRepository[], -) { - const data = finding.evidence.source_data; - const evidence = snykRowEvidence(finding); - if (!data || (!evidence?.v1?.issue && !evidence?.v2?.issue)) return; - const format = evidence.v2?.issue - ? SNYK_REST_ISSUES_FORMAT - : SNYK_REST_ISSUES_V1; - const document = readSnykRestDocument({ - ...data, - format: data["format"] ?? format, - }); - const { source } = savedContext(document, references); - const issue = parseHostedScannerJson( - requiredText(document["issueJson"], "Snyk original issue"), - ); - const { attributes } = selectedIssue(scopedIssue(issue, source), source); - return snykIssueId(attributes, source, data["snykRestIssueKey"]); -} - -export function snykRestFindingRepository( - finding: ExternalFinding, - references: readonly ExternalSourceRepository[] = [], -) { - return bindSnykFinding(finding, references, snykRowEvidence(finding)); -} - function bindSnykFinding( finding: ExternalFinding, references: readonly ExternalSourceRepository[], @@ -1032,31 +926,20 @@ function bindSnykFinding( finding.evidence.source_data, "Original Snyk REST issue evidence", ); - const format = - retained["format"] === undefined - ? originals?.v1 - ? SNYK_REST_ISSUES_V1 - : SNYK_REST_ISSUES_FORMAT - : retained["format"]; - if ( - isSnykRestFormat(format) && - (format === SNYK_REST_ISSUES_V1 ? originals?.v2 : originals?.v1) - ) + if (originals?.retired) throw new Error( - "A normalized Snyk row cannot claim both v1 and v2 native representations.", + "A Snyk REST finding must use the current collector's original JSON evidence.", ); - const document = readSnykRestDocument( - { ...retained, format }, - report?.fields, - ); + const format = retained["format"] ?? SNYK_REST_ISSUES_FORMAT; + const document: Resource = { ...retained, format }; const issue = - (format === SNYK_REST_ISSUES_FORMAT ? originals?.v2?.issue : undefined) ?? + originals?.issue ?? parseHostedScannerJson( requiredText(document["issueJson"], "Snyk original issue"), ); // A selected issue can retain any of its original monitor fields. Fill only those it omitted. let projected = false; - if (report && (originals?.v1?.issue || originals?.v2?.issue)) { + if (report && originals?.issue) { for (const [field, value] of Object.entries(report.fields)) { if (document[field] !== undefined) continue; document[field] = value; @@ -1079,28 +962,7 @@ function bindSnykFinding( ); verifySnykFindingId(finding, attributes, source); projectSnykIssue(attributes, asset, { ...retained, format }, "local"); - if (retain && format === SNYK_REST_ISSUES_V1) { - // The legacy witness stores the original decimal token, including its spelling. - const context: Resource = projected - ? encodeSnykV1File({ - ...repository.scope, - apiVersion: source.apiVersion, - format, - project: asset.project, - target: asset.target, - }) - : { format }; - retain(finding, { - ...retained, - ...context, - ...(isRecord(context["numericFields"]) && { - numericFields: { - ...context["numericFields"], - ...(isRecord(retained["numericFields"]) && retained["numericFields"]), - }, - }), - }); - } else if (retain && format === SNYK_REST_ISSUES_FORMAT) { + if (retain) { retain(finding, { ...retained, ...(projected && source), @@ -1166,18 +1028,17 @@ function problemIds(entries: Resource[], type?: string) { return [...ids]; } -function findingFilePath(value: unknown, v2: boolean) { +function findingFilePath(value: unknown) { const path = hostedRepositoryPath(value); - if (v2 && (path?.endsWith("/.") || !repositoryFilePath(path))) return null; + if (path?.endsWith("/.") || !repositoryFilePath(path)) return null; return path; } function packageEvidence( coordinates: Resource[], { manager, targetFile }: ReturnType, - v2: boolean, ) { - const manifest = findingFilePath(targetFile, v2); + const manifest = findingFilePath(targetFile); const packages: ImportedPackage[] = []; for (const coordinate of coordinates) { const fixed = records(coordinate, "remedies").flatMap((remedy) => { @@ -1214,11 +1075,16 @@ function packageEvidence( function codeStartLine(region: unknown): number | null { const start = isRecord(region) ? region["start"] : undefined; const token = isRecord(start) ? hostedNumberToken(start["line"]) : undefined; - const line = token && Number(legacyDecimalInteger(token)); - return isSafeNonNegativeInteger(line) && line > 0 ? line : null; + const line = Number(token); + return token && + isSafeNonNegativeInteger(line) && + line > 0 && + String(line) === canonicalNumber(token) + ? line + : null; } -function codeEvidence(coordinates: Resource[], scm: boolean, v2: boolean) { +function codeEvidence(coordinates: Resource[], scm: boolean) { const locations: { path: string; line?: number | null }[] = []; const revisions = new Set(); for (const coordinate of coordinates) { @@ -1228,11 +1094,9 @@ function codeEvidence(coordinates: Resource[], scm: boolean, v2: boolean) { if (value === undefined) continue; const location = record(value, "Snyk source location"); revisions.add(text(location["commit_id"])); - const path = findingFilePath(location["file"], v2); + const path = findingFilePath(location["file"]); if (scm && path) - locations.push( - v2 ? { path, line: codeStartLine(location["region"]) } : { path }, - ); + locations.push({ path, line: codeStartLine(location["region"]) }); } } return { @@ -1248,7 +1112,6 @@ function projectSnykIssue( target: ExternalImportTarget, ) { const code = source.product === "code"; - const v2 = sourceData["format"] !== SNYK_REST_ISSUES_V1; const status = attributes["status"]; if ( (status !== "open" && status !== "resolved") || @@ -1283,9 +1146,9 @@ function projectSnykIssue( severity === "info" ? "informational" : scannerSeverity(severity), advisory_ids: code ? [] : problemIds(problems), ...(code - ? codeEvidence(activeCoordinates, context.scm, v2) + ? codeEvidence(activeCoordinates, context.scm) : { - packages: packageEvidence(activeCoordinates, context, v2), + packages: packageEvidence(activeCoordinates, context), }), source_updated_at: Math.floor(updated / 1000), source_data: sourceData, @@ -1309,32 +1172,28 @@ export function normalizeSnykIssues( target: ExternalImportTarget = "local", references: readonly ExternalSourceRepository[] = [], ): SnykIssuesFindings { - const wire = record(input, "Snyk REST Issues export"); - const document = readSnykRestDocument(wire); + const document = record(input, "Snyk REST Issues export"); const asset = savedContext(document, references); const { source, repository } = asset; const issues = readIssues(document, source); - const legacy = originalSnykEvidence(wire, source); // normalizeRecords identifies every row before projecting any evidence. const resourceCounts = new Map(); const result = normalizeRecords( - issues.map((issue, index) => ({ issue, original: legacy?.rows[index] })), - ({ issue, original }) => { + issues, + (issue) => { if (resourceCounts.get(resourceId(issue, "issue"))! > 1) throw new Error( "The saved Snyk report contains overlapping issue IDs.", ); const attributes = record(issue["attributes"], "Snyk issue attributes"); - const sourceData = original - ? structuredClone(original) - : { - ...source, - snykRestIssueKey: attributes["key"], - issueJson: stringifyHostedEvidence(issue), - }; + const sourceData = { + ...source, + snykRestIssueKey: attributes["key"], + issueJson: stringifyHostedEvidence(issue), + }; return projectSnykIssue(attributes, asset, sourceData, target); }, - ({ issue }) => { + (issue) => { const identifier = resourceIdentifier(issue, "issue"); const raw = issue["attributes"]; const product = isRecord(raw) ? raw["type"] : undefined; @@ -1356,7 +1215,7 @@ export function normalizeSnykIssues( ...(repository.platform && { sourceRepositoryPlatform: repository.platform, }), - snykRestContext: legacy ? structuredClone(legacy.context) : source, + snykRestContext: source, }; } @@ -1367,15 +1226,12 @@ export function snykRestImportRepository( urls: Iterable = [], trustedSources: readonly ExternalSourceRepository[] = [], retain?: (finding: ExternalFinding, source: Resource) => void, - allowOpaqueLegacyRows = false, declaredCloudRepository?: string, ): RepositoryBinding | undefined { const project = input.snykRestContext !== undefined ? savedContext( - readSnykRestDocument( - record(input.snykRestContext, "Snyk report context"), - ), + record(input.snykRestContext, "Snyk report context"), trustedSources, ) : undefined; @@ -1394,33 +1250,10 @@ export function snykRestImportRepository( targetJson: JSON.stringify(project.target), }, }; - const retainedIssues: { finding: ExternalFinding; issue: Resource }[] = []; - const retainedRows: Resource[] = []; const boundRows = hosted - ? rows.flatMap(({ finding, originals }) => { - try { - return [ - bindSnykFinding( - finding, - trustedSources, - originals, - context, - retain, - ), - ]; - } catch (error) { - if ( - !allowOpaqueLegacyRows || - error instanceof SnykTargetRepositoryMismatch - ) - throw error; - for (const issue of [originals?.v1?.issue, originals?.v2?.issue]) - if (issue) retainedIssues.push({ finding, issue }); - if (originals && finding.evidence.source_data) - retainedRows.push(finding.evidence.source_data); - return []; - } - }) + ? rows.map(({ finding, originals }) => + bindSnykFinding(finding, trustedSources, originals, context, retain), + ) : []; const selected = project?.repository.scope ?? boundRows[0]?.scope; const bindings = project ? [...boundRows, project.repository] : boundRows; @@ -1440,20 +1273,9 @@ export function snykRestImportRepository( throw new Error( "The selected Snyk finding is missing the report's native target. Keep its full project or omit the project to use the report context.", ); - const retainedAssets = retainedRows.flatMap(retainedSnykAssets); - const monitorBindings: { - targetId?: string | null; - integrationId?: string; - deployment?: string; - }[] = [...bindings, ...retainedAssets]; for (const key of ["targetId", "integrationId", "deployment"] as const) { - // Sparse targets can contradict a verified SCM deployment but cannot establish one. - if (key === "deployment" && !bindings.some((binding) => binding.deployment)) - continue; const ids = new Set( - monitorBindings - .map((binding) => binding[key]) - .filter((id) => id !== undefined), + bindings.map((binding) => binding[key]).filter((id) => id !== undefined), ); if (ids.size > 1) throw new Error( @@ -1462,46 +1284,6 @@ export function snykRestImportRepository( } const scopes = boundRows.map(({ scope }) => scope); const claims = new Set(bindings.map((binding) => binding.url)); - const retainedScms = []; - for (const { - url, - origin, - product, - projectId, - organizationId, - } of retainedAssets) { - if (url) claims.add(url); - if (origin) retainedScms.push(reportedScm(repositoryUrl, origin)); - if (selected) - scopes.push({ - ...selected, - projectId: projectId ?? selected.projectId, - organizationId: organizationId ?? selected.organizationId, - product: product ?? selected.product, - }); - } - // Already-approved rows can lack the original project while retaining a - // monitored issue, partial scope, or source URL that contradicts a new claim. - if (selected) { - for (const { finding, issue } of retainedIssues) { - const { attributes } = selectedIssue( - scopedIssue(issue, selected), - selected, - ); - if (snykRestId.test(finding.source_finding_id)) - verifySnykFindingId(finding, attributes, selected); - } - } - for (const { sourceRepositoryUrl, ...fields } of retainedRows) { - if (sourceRepositoryUrl !== undefined) - claims.add( - requiredText(sourceRepositoryUrl, "Snyk source repository URL"), - ); - if (selected) { - const supplied = { ...selected, ...fields }; - scopes.push(scope(supplied, supplied.apiOrigin)); - } - } if ( bindings.some((binding) => binding.scm !== bindings[0]?.scm) || (selected && @@ -1541,12 +1323,10 @@ export function snykRestImportRepository( repositoryUrl, trustedSources, input.sourceRepositoryPlatform !== "other" && - retainedScms.every(({ github }) => github) && bindings.every( (binding) => binding.platform !== "other" && !binding.gitlab, ), input.sourceRepositoryPlatform !== "github" && - retainedScms.every(({ gitlab }) => gitlab) && bindings.every( (binding) => binding.gitlab || binding.platform === undefined, ), diff --git a/sdk/typescript/src/snyk-rest-v1.ts b/sdk/typescript/src/snyk-rest-v1.ts deleted file mode 100644 index 6e74315487..0000000000 --- a/sdk/typescript/src/snyk-rest-v1.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { - hostedNumberToken, - parseHostedScannerJson, -} from "./hosted-scanner-http.js"; -import { - isLegacyWitness, - restoreLegacyHostedNumbers, -} from "./hosted-scanner-v1.js"; -import { canonicalJson, isRecord } from "./record.js"; -import { array, record } from "./scanner-import.js"; - -type Resource = Record; -export const SNYK_REST_ISSUES_V1 = "snyk-rest-issues/v1"; -export const SNYK_REST_ISSUES_FORMAT = "snyk-rest-issues/v2"; -export function isSnykRestFormat(value: unknown): boolean { - return value === SNYK_REST_ISSUES_FORMAT || value === SNYK_REST_ISSUES_V1; -} - -/** Only v1 can carry native JSON numbers outside the lossless evidence strings. */ -export function parseSnykReport(text: string): unknown { - const document: unknown = JSON.parse(text); - return isRecord(document) && document["format"] === SNYK_REST_ISSUES_V1 - ? parseHostedScannerJson(text) - : document; -} - -function encodeNumbers(data: object): Resource | undefined { - const fields: Resource = Object.create(null); - const values = data as Resource; - for (const [key, value] of Object.entries(values)) { - const token = hostedNumberToken(value); - if (token !== undefined) { - if (isLegacyWitness(token)) { - values[key] = token; - fields[key] = token; - } else { - values[key] = Number(token); - } - } else if (value !== null && typeof value === "object") { - // restoreLegacyHostedNumbers shares branches that had no sidecar. - // Keep the captured input reusable for another repository projection. - const child = Array.isArray(value) ? [...value] : { ...value }; - values[key] = child; - const nested = encodeNumbers(child); - if (nested) fields[key] = nested; - } - } - return Object.keys(fields).length ? fields : undefined; -} - -/** Recover the exact v1 wire representation whether numbers were native or witnessed strings. */ -export function encodeSnykV1File(document: Resource): Resource { - const source = { ...restoreLegacyHostedNumbers(document) }; - delete source["numericFields"]; - const fields = encodeNumbers(source); - if (fields) source["numericFields"] = fields; - return source; -} - -/** Read v1 exports, retained source records, and empty-report project contexts. */ -export function readSnykRestDocument( - document: Resource, - context?: Resource, -): Resource { - if (document["format"] !== SNYK_REST_ISSUES_V1) return document; - const fields = document["numericFields"]; - if (context && isRecord(fields)) { - // A projected row may keep witnesses for assets that now come from its report. - const inline = { ...fields }; - for (const field of ["project", "target"]) { - const json = context[field + "Json"]; - if ( - document[field] !== undefined || - !(field in fields) || - typeof json !== "string" - ) - continue; - const asset = encodeSnykV1File({ [field]: parseHostedScannerJson(json) }); - if ( - canonicalJson(fields[field]) !== - canonicalJson(at(asset, "numericFields", field)) - ) - throw new Error( - "Invalid scanner numeric fields for the retained Snyk asset.", - ); - delete inline[field]; - document = { ...document, numericFields: inline }; - } - } - const exact = restoreLegacyHostedNumbers(document); - return { - ...exact, - format: SNYK_REST_ISSUES_FORMAT, - projectJson: JSON.stringify(exact["project"]), - targetJson: JSON.stringify(exact["target"]), - issueJson: JSON.stringify(exact["issue"]), - ...(exact["pages"] !== undefined && { - pages: array(exact["pages"], "Snyk issue pages").map((entry) => { - const page = record(entry, "Snyk saved page"); - return { - startingAfter: page["startingAfter"], - responseJson: JSON.stringify(page["response"]), - }; - }), - }), - }; -} - -function at(value: unknown, ...path: (string | number)[]): unknown { - for (const segment of path) - value = isRecord(value) ? value[segment] : undefined; - return value; -} - -/** Retain v1's evidence representation so replaying an existing import stays unchanged. */ -export function originalSnykEvidence(document: Resource, source: Resource) { - if (document["format"] !== SNYK_REST_ISSUES_V1) return; - const { projectJson: _project, targetJson: _target, ...scope } = source; - const fields = document["numericFields"]; - const assetFields: Resource = {}; - for (const key of ["project", "target"]) { - const field = at(fields, key); - if (field !== undefined) assetFields[key] = field; - } - const context = { - ...scope, - format: SNYK_REST_ISSUES_V1, - project: document["project"], - target: document["target"], - ...(Object.keys(assetFields).length > 0 && { numericFields: assetFields }), - }; - const rows = array(document["pages"], "Snyk issue pages").flatMap( - (value, i) => { - const page = record(value, "Snyk saved page"); - const response = record(page["response"], "Snyk issue page"); - return array(response["data"], "Snyk issue page data").map((issue, j) => { - const field = at(fields, "pages", i, "response", "data", j); - return { - ...context, - issue, - ...(field !== undefined && { - numericFields: { ...assetFields, issue: field }, - }), - }; - }); - }, - ); - return { context, rows }; -} diff --git a/sdk/typescript/src/trivy-findings.ts b/sdk/typescript/src/trivy-findings.ts index 91a8a74360..178cb67d47 100644 --- a/sdk/typescript/src/trivy-findings.ts +++ b/sdk/typescript/src/trivy-findings.ts @@ -381,31 +381,6 @@ export function normalizeTrivyFindings( scanRoot?: string, ): ExternalFindings { const locationLimit = target === "cloud" ? cloudLocationLimit : Infinity; - return normalizeReport(input, target, scanRoot, locationLimit); -} - -/** @internal Only a finding at the Cloud location limit can have a different prior selection. */ -export function needsPreviousTrivyProjection({ findings }: ExternalFindings) { - return findings.some( - ({ evidence }) => evidence.locations?.length === cloudLocationLimit, - ); -} - -/** @internal Reconstruct the original selection for an interrupted Cloud request. */ -export function normalizePreviousTrivyFindings( - input: unknown, - target: ExternalImportTarget = "cloud", - scanRoot?: string, -): ExternalFindings { - return normalizeReport(input, target, scanRoot, Infinity); -} - -function normalizeReport( - input: unknown, - target: ExternalImportTarget, - scanRoot: string | undefined, - locationLimit: number, -): ExternalFindings { const document = record(input, "Trivy report"); if (document["SchemaVersion"] !== 2) throw new Error( diff --git a/sdk/typescript/src/veracode-findings.ts b/sdk/typescript/src/veracode-findings.ts index ab25a3f9ea..5b82d40daa 100644 --- a/sdk/typescript/src/veracode-findings.ts +++ b/sdk/typescript/src/veracode-findings.ts @@ -1,5 +1,4 @@ import { z } from "zod"; -import { markPreviousCloudProjection } from "./reader-selection.js"; import { CodexSecurityError } from "./errors.js"; import type { Provider } from "./external-import-models.js"; import { @@ -382,10 +381,7 @@ function expandedDetails( }; } -function rowSelectionValidator( - query: URLSearchParams, - expand: typeof expandedDetails, -) { +function rowSelectionValidator(query: URLSearchParams) { const staticOnly = query.has("scan_type"); const violates = query.get("violates_policy"); const newOnly = query.get("new") === "true"; @@ -408,7 +404,6 @@ function rowSelectionValidator( }); return (row: Record) => { - let expandedFilter = false; if ( newOnly && record(row["finding_status"], "Veracode finding_status")["new"] !== true @@ -423,14 +418,11 @@ function rowSelectionValidator( "Veracode finding contradicts the include_annot selection.", ); if (idFilters.length || scoreFilters.length) { - const native = findingDetails(row); - const details = expand(native); + const details = expandedDetails(findingDetails(row)); for (const { field, ids } of idFilters) { if (row["scan_type"] !== "STATIC") continue; const entry = record(details[field], `Veracode ${field}`); const id = integer(entry["id"], `${field} ID`); - // Older readers required HAL descriptors for filtered IDs. - expandedFilter ||= details !== native && native[field] !== entry; if (!ids.has(String(id))) throw new Error( `Veracode finding contradicts the ${field} selection.`, @@ -459,11 +451,10 @@ function rowSelectionValidator( throw new Error( "Veracode finding contradicts the violates_policy selection.", ); - return expandedFilter; }; } -function readReport(input: unknown, expand: typeof expandedDetails) { +function readReport(input: unknown) { const pages = (Array.isArray(input) ? input : [input]) .map(readPage) .sort((left, right) => left.number - right.number); @@ -484,15 +475,14 @@ function readReport(input: unknown, expand: typeof expandedDetails) { "Supply every Veracode page exactly once with stable totals, selection, origin and application context.", ); const { scope } = first; - const checkSelection = rowSelectionValidator(first.query, expand); - let usedExpandedFilter = false; + const checkSelection = rowSelectionValidator(first.query); const seen = new Set(); const rows: Record[] = []; for (const page of pages) { for (const value of page.rows) { const row = record(value, "Veracode finding"); requireApplication(row, scope); - if (checkSelection(row)) usedExpandedFilter = true; + checkSelection(row); const issue = row["issue_id"]; if ( typeof row["scan_type"] === "string" && @@ -508,27 +498,24 @@ function readReport(input: unknown, expand: typeof expandedDetails) { rows.push(row); } } - return { scope, rows, usedExpandedFilter }; + return { scope, rows }; } -function normalizeReport( +export function normalizeVeracodeFindings( input: unknown, sourceRepositoryUrl: string, - target: ExternalImportTarget, - expand: typeof expandedDetails, + target: ExternalImportTarget = "cloud", ): ExternalFindings { const repository = declaredScannerRepositoryUrl( sourceRepositoryUrl, "Veracode", target, ); - const { scope, rows, usedExpandedFilter } = readReport(input, expand); - let admittedScalar = false; - const findings = normalizeRecords( + const { scope, rows } = readReport(input); + return normalizeRecords( rows, (row) => { - const native = findingDetails(row); - const details = expand(native); + const details = expandedDetails(findingDetails(row)); const status = record(row["finding_status"], "Veracode finding_status"); const cwe = record(details["cwe"], "Veracode CWE"); const weakness = `CWE-${integer(cwe["id"], "CWE ID", 1)}`; @@ -540,7 +527,7 @@ function normalizeReport( severityByScore[integer(details["severity"], "severity")]; if (!severity) throw new Error("Veracode severity must be on the native 0–5 scale."); - const evidence = validateExternalEvidence( + return validateExternalEvidence( { title: text(category?.["name"]) ?? text(cwe["name"]) ?? weakness, severity, @@ -562,37 +549,10 @@ function normalizeReport( }, target, ); - if (details !== native) admittedScalar = true; - return evidence; }, (row) => occurrenceId(row, scope), target, ); - if (target === "cloud" && admittedScalar && !usedExpandedFilter) - markPreviousCloudProjection(findings); - return findings; -} - -export function normalizeVeracodeFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeReport(input, sourceRepositoryUrl, target, expandedDetails); -} - -/** Reconstruct a saved publication made when only HAL descriptors were admitted. */ -export function normalizePreviousVeracodeFindings( - input: unknown, - sourceRepositoryUrl: string, - target: ExternalImportTarget = "cloud", -): ExternalFindings { - return normalizeReport( - input, - sourceRepositoryUrl, - target, - (details) => details, - ); } export async function readVeracodeFindings( diff --git a/sdk/typescript/src/wiz-findings.ts b/sdk/typescript/src/wiz-findings.ts index 02f5c33f93..5bd168d54e 100644 --- a/sdk/typescript/src/wiz-findings.ts +++ b/sdk/typescript/src/wiz-findings.ts @@ -139,7 +139,6 @@ type RepositoryMetadata = { type FindingInput = { records: InputRecord[]; repositories: Map; - legacyRepositories?: FindingInput["repositories"]; }; // Wiz's documented inventory platform establishes custom GitHub host semantics. @@ -180,42 +179,6 @@ function sourceRepository( }; } -function sourceBranch( - branch: Record | undefined, - repository: RepositoryMetadata, - legacy = false, -): string | null { - const name = text(branch?.["name"]); - if (!name) return null; - // Repository names and URL paths can both be literal branch prefixes. Keep - // the old mapping only for an already approved publication checkpoint. - if (!legacy) return name; - const qualifiers = repository.name ? [repository.name] : []; - - // Include the URL qualifier before selecting the longest repository prefix. - // Keep URL normalization and the branch suffix's original case and slashes. - const repositoryKey = repositoryUrlKey(repository.url).toLowerCase(); - const url = /^(https?:\/\/[^/?#]+)\/([^?#]+)$/iu.exec(repositoryKey); - if (url) { - const prefixParts = url[2]!.split("/").length; - const parts = name.split("/"); - const qualifier = parts.slice(0, prefixParts).join("/"); - if ( - parts.length > prefixParts && - repositoryUrlKey(url[1] + "/" + qualifier).toLowerCase() === repositoryKey - ) - qualifiers.push(qualifier); - } - for (const qualifier of qualifiers.sort( - (left, right) => right.length - left.length, - )) { - const prefix = qualifier + "/"; - if (name.toLowerCase().startsWith(prefix.toLowerCase())) - return name.slice(prefix.length) || null; - } - return url || repository.name ? name : null; -} - function updatedAt(value: unknown, field: string): number | null { const timestamp = text(value); if (!timestamp) return null; @@ -235,7 +198,6 @@ function repositoryFinding( kind: RepositoryFindingKind, record: Record, repositories: FindingInput["repositories"], - legacy = false, target: ExternalImportTarget = "cloud", ): VendorFinding { const id = sourceFindingId(record["id"], target); @@ -323,7 +285,7 @@ function repositoryFinding( description: text(record["description"]) ?? text(rule?.["description"]), url: text(record["wizUrl"]) ?? text(record["portalUrl"]), locations: path ? [{ path, line: line ?? null }] : [], - branch: sourceBranch(branch, repository, legacy), + branch: text(branch?.["name"]), code_revision: null, source_scan_id: null, source_updated_at: @@ -342,7 +304,6 @@ function repositoryFinding( function records(payload: unknown): FindingInput { const repositories: FindingInput["repositories"] = new Map(); - const legacyRepositories: FindingInput["repositories"] = new Map(); if (Array.isArray(payload)) return { records: payload.map((value) => ({ value })), repositories }; const envelope = asRecord(payload); @@ -412,10 +373,6 @@ function records(payload: unknown): FindingInput { : entry.name, platform: entry.platform ?? previous?.platform ?? null, }); - legacyRepositories.set(entry.id, { - ...entry, - platform: entry.platform ?? previous?.platform ?? null, - }); } for (const entry of entries) { const resolved = repositories.get(entry.id)!; @@ -462,7 +419,6 @@ function records(payload: unknown): FindingInput { return { records: selected, repositories, - legacyRepositories, }; throw new Error( "Unsupported Wiz export. Supply repository vulnerability CSV, vulnerability finding objects, named sastFindings/secretInstances/iacFindings collections, or normalized JSONL. Other CSV reports need an explicit normalized JSONL mapping.", @@ -509,6 +465,14 @@ export async function readVendorFindings( return normalizeFindings(await readFindingInput(path), target); } +/** @internal Reopen a native JSON selection with the same rules as a Wiz file. */ +export function normalizeWizJson( + payload: unknown, + target: ExternalImportTarget, +) { + return normalizeFindings(records(payload), target); +} + function inventorySources(input: FindingInput, current: VendorFindings) { return current.findings.flatMap(({ evidence }) => { const repository = evidence.details?.repository; @@ -524,70 +488,13 @@ function inventorySources(input: FindingInput, current: VendorFindings) { }); } -export async function readLocalWizFindings(path: string) { - const input = await readFindingInput(path); - const current = normalizeFindings(input, "local"); - return { current, sourceRepositories: inventorySources(input, current) }; -} - -/** @internal Preserve older inventory projections only for saved publication replay. */ -export async function readVendorFindingsForPublication( +export async function readWizSelection( path: string, - target: ExternalImportTarget = "cloud", -): Promise<{ - current: VendorFindings; - legacy: VendorFindings; - currentIdentity: unknown[]; - legacyIdentity: unknown[]; - sourceRepositories: ExternalSourceRepository[]; -}> { + target: ExternalImportTarget, +) { const input = await readFindingInput(path); - const normalized = new WeakSet(); - const current = normalizeFindings(input, target, normalized); - // Only repository finding families use historical inventory and branch mapping. - const hasRepositoryFindings = input.records.some( - ({ kind }) => kind === "sast" || kind === "secret" || kind === "iac", - ); - const legacy = hasRepositoryFindings - ? normalizeFindings( - { - ...input, - repositories: input.legacyRepositories ?? input.repositories, - }, - target, - normalized, - true, - ) - : current; - const identity = (parsed: VendorFindings) => - parsed.findings.map((finding) => { - if (normalized.has(finding)) return finding; - const repository = finding.evidence.details?.repository; - return { - source_finding_id: finding.source_finding_id, - evidence: { - // Raw vendor records retain every original branch/ref and source field. - // Inventory display aliases must not change an outstanding request's identity. - source_data: finding.evidence.source_data, - repository: repository - ? { - id: repository.id, - // Publication targets use Cloud's GitHub URL semantics. Input - // joins above still require provenance for custom-host folding. - url: repositoryUrlKey(repository.url).toLowerCase(), - } - : null, - }, - }; - }); - const currentIdentity = identity(current); - return { - current, - legacy, - currentIdentity, - legacyIdentity: legacy === current ? currentIdentity : identity(legacy), - sourceRepositories: inventorySources(input, current), - }; + const findings = normalizeFindings(input, target); + return { findings, sourceRepositories: inventorySources(input, findings) }; } async function readFindingInput(path: string): Promise { @@ -646,21 +553,16 @@ export function isWizOccurrence( ...validateExternalEvidence(finding.evidence, target), source_data: undefined, }); - const matches = (legacy: boolean) => { - const [row] = normalizeFindings(input, target, undefined, legacy).findings; - return ( - row !== undefined && - canonicalJson({ ...row.evidence, source_data: undefined }) === selected - ); - }; - return matches(false) || (scoped && matches(true)); + const [row] = normalizeFindings(input, target).findings; + return ( + row !== undefined && + canonicalJson({ ...row.evidence, source_data: undefined }) === selected + ); } function normalizeFindings( input: FindingInput, target: ExternalImportTarget, - normalized?: WeakSet, - legacy = false, ): VendorFindings { const result: VendorFindings = { read: input.records.length, @@ -705,7 +607,6 @@ function normalizeFindings( item.kind, record, input.repositories, - legacy, target, ); } else { @@ -727,8 +628,6 @@ function normalizeFindings( } identities.add(finding.source_finding_id); result.findings.push(finding); - if (item.kind !== "sca_csv" && "source_finding_id" in record) - normalized?.add(finding); } catch (error) { if (error instanceof CodexSecurityError) throw error; result.excluded.push({ diff --git a/sdk/typescript/tests-ts/blackduck-findings.test.ts b/sdk/typescript/tests-ts/blackduck-findings.test.ts index a31d473c92..0f91e391b5 100644 --- a/sdk/typescript/tests-ts/blackduck-findings.test.ts +++ b/sdk/typescript/tests-ts/blackduck-findings.test.ts @@ -4,9 +4,7 @@ import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { blackDuckReportRepositories, - needsPreviousBlackDuckProjection, normalizeBlackDuckFindings, - normalizePreviousBlackDuckFindings, readBlackDuckFindings, } from "../src/blackduck-findings.js"; import { @@ -147,15 +145,10 @@ test("keeps GitHub package coordinates separate from the repository that was sca componentVersionOriginId: "LibraryOrg/Library/v1.0", packageUrl: "pkg:github/LibraryOrg/Library@release%2Fv1.0", }; - const previous = normalizePreviousBlackDuckFindings([row], repository) - .findings[0]!; - expect(previous.evidence.packages).toEqual([]); for (const target of ["local", "cloud"] as const) { const current = normalizeBlackDuckFindings([row], repository, target); - expect(needsPreviousBlackDuckProjection(current)).toBe(true); expect(current.excluded).toEqual([]); expect(current.findings[0]).toMatchObject({ - source_finding_id: previous.source_finding_id, evidence: { packages: [ { @@ -170,10 +163,6 @@ test("keeps GitHub package coordinates separate from the repository that was sca } row.packageUrl = "pkg:github/LibraryOrg/Library@" + "v".repeat(513); const bounded = normalizeBlackDuckFindings([row], repository); - expect(needsPreviousBlackDuckProjection(bounded)).toBe(false); - expect(bounded).toEqual( - normalizePreviousBlackDuckFindings([row], repository), - ); expect(bounded.findings[0]!.evidence.source_data?.["row"]).toEqual(row); }); diff --git a/sdk/typescript/tests-ts/continuum-findings.test.ts b/sdk/typescript/tests-ts/continuum-findings.test.ts index c951d69230..791054e64f 100644 --- a/sdk/typescript/tests-ts/continuum-findings.test.ts +++ b/sdk/typescript/tests-ts/continuum-findings.test.ts @@ -2,10 +2,7 @@ import { expect, test } from "bun:test"; import { writeFile } from "node:fs/promises"; import { gzipSync } from "node:zlib"; import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; -import { - normalizeContinuumFindings, - normalizePreviousContinuumFindings, -} from "../src/continuum-findings.js"; +import { normalizeContinuumFindings } from "../src/continuum-findings.js"; import { createAndReplayFinding, localReportFixture, @@ -157,10 +154,6 @@ test("reads the weakness Continuum names in subCategory for its own findings", ( // Explicit IDs win, and other category text is not a weakness. expect(weaknesses({ ...native, cweIds: ["CWE-20"] })).toEqual(["CWE-20"]); expect(weaknesses({ ...native, subCategory: "Injection" })).toEqual([]); - expect( - normalizePreviousContinuumFindings(native, continuumRepository).findings[0]! - .evidence.details?.weakness_ids, - ).toEqual([]); }); test("an inferred weakness cannot exclude a Cloud finding at its evidence limit", () => { @@ -172,10 +165,9 @@ test("an inferred weakness cannot exclude a Cloud finding at its evidence limit" vulnerabilities: [], stepsToReproduce: Array(31).fill("x".repeat(8192)).concat(""), }; - const original = normalizePreviousContinuumFindings( - finding, - continuumRepository, - ).findings[0]!.evidence; + const original = normalize(finding).findings[0]!.evidence; + original.source_updated_at = null; + original.details!.weakness_ids = []; const bytes = Buffer.byteLength(JSON.stringify(original)); finding.stepsToReproduce[31] = "x".repeat(256 * 1024 - bytes); const result = normalize(finding); @@ -254,13 +246,10 @@ test("a Cloud finding that fits with native packages alone still imports", () => ], padding: "", }; - const previous = normalizePreviousContinuumFindings( - finding, - continuumRepository, - ); - const bytes = Buffer.byteLength( - JSON.stringify(previous.findings[0]!.evidence), - ); + const base = normalize(finding).findings[0]!.evidence; + base.packages = []; + base.source_updated_at = null; + const bytes = Buffer.byteLength(JSON.stringify(base)); finding.padding = "x".repeat(256 * 1024 - bytes); const result = normalize(finding); expect(result.excluded).toEqual([]); diff --git a/sdk/typescript/tests-ts/endor-findings.test.ts b/sdk/typescript/tests-ts/endor-findings.test.ts index 971dfd8561..f3de1adc0c 100644 --- a/sdk/typescript/tests-ts/endor-findings.test.ts +++ b/sdk/typescript/tests-ts/endor-findings.test.ts @@ -1,9 +1,7 @@ -import { hasPreviousCloudProjection } from "../src/reader-selection.js"; import { expect, test } from "bun:test"; import { normalizeEndorFindings, normalizeEndorSarif, - normalizePreviousEndorFindings, requiresStructuredEndorReport, } from "../src/endor-findings.js"; import { endorFinding, endorSarif } from "./support/endor-import-fixtures.js"; @@ -71,13 +69,6 @@ test.each([ }, }, }; - const previous = normalizePreviousEndorFindings(input); - expect(previous.findings[0]).toMatchObject({ - source_finding_id: input.uuid, - evidence: { packages: [], advisory_ids: originalIds, source_data: input }, - }); - expect(hasPreviousCloudProjection(previous)).toBe(false); - const current = normalizeEndorFindings(input); expect(current.excluded).toEqual([]); expect(current.findings[0]).toMatchObject({ @@ -92,13 +83,9 @@ test.each([ expect(current.findings[0]!.evidence.packages!.map((p) => p.name)).toEqual( oversizedPackage ? [] : spec.dependency_file_paths.map(() => name), ); - expect(hasPreviousCloudProjection(current)).toBe( - !oversizedPackage || !oversizedAliases, - ); expect(requiresStructuredEndorReport(current)).toBe( oversizedPackage || oversizedAliases, ); - if (oversizedPackage && oversizedAliases) expect(current).toEqual(previous); }, ); @@ -128,8 +115,6 @@ test("Endor keeps its original projection for duplicate or malformed malware met expect(current.findings[0]!.evidence.packages![0]!.name).toBe( base.spec.target_dependency_name, ); - expect(current).toEqual(normalizePreviousEndorFindings(input)); - expect(hasPreviousCloudProjection(current)).toBe(false); } }); diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 30822d3b03..1c52f25f4f 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -1,5 +1,3 @@ -import boxedNumberCheckpoints from "./fixtures/boxed-number-checkpoints.json" with { type: "json" }; -import boxedSonarCheckpoint from "./fixtures/boxed-sonar-checkpoint.json" with { type: "json" }; import { snykJson, snykPluginExport, @@ -22,10 +20,7 @@ import { } from "./support/semgrep-import-fixtures.js"; import { endorFinding, endorSarif } from "./support/endor-import-fixtures.js"; import { sonarGitTestEnvironment } from "./support/sonar-import-fixtures.js"; -import { - normalizeEndorFindings, - normalizePreviousEndorFindings, -} from "../src/endor-findings.js"; +import { normalizeEndorFindings } from "../src/endor-findings.js"; import { createServer, type ServerResponse } from "node:http"; import { execFile } from "node:child_process"; import { hash } from "node:crypto"; @@ -40,7 +35,7 @@ import { writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { basename, dirname, join } from "node:path"; +import { basename, join } from "node:path"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { gzipSync } from "node:zlib"; @@ -53,23 +48,17 @@ import { prepareNormalizedExternalPublication, type ExternalPublicationOptions, type ExternalPublicationProgress, - type ExternalPublicationResult, type NormalizedExternalPublicationOptions, type PreparedExternalPublication, } from "../src/external-findings-publish.js"; -import { canonicalJson } from "../src/record.js"; import { normalizeSemgrepPlatformFindings, selectSemgrepPlatformFindings, semgrepPlatformFormat, } from "../src/semgrep-platform-findings.js"; -import { - normalizePreviousSemgrepFindings, - normalizeSemgrepFindings, -} from "../src/semgrep-findings.js"; +import { normalizeSemgrepFindings } from "../src/semgrep-findings.js"; import { collectSemgrepPlatformFindings } from "../src/semgrep-platform-collector.js"; import { normalizeSnykIssues } from "../src/snyk-rest-findings.js"; -import releasedSnykCheckpoints from "./fixtures/snyk-released-checkpoints.json" with { type: "json" }; import { snykRestFixture } from "./support/snyk-rest-fixtures.js"; import { @@ -82,10 +71,7 @@ import { normalizeGitHubCodeScanningAlerts, readGitHubFindings, } from "../src/github-findings.js"; -import { - privateReport, - selectPrivateReport, -} from "../src/private-finding-import.js"; +import { privateReport } from "../src/private-finding-import.js"; import { readVendorFindings } from "../src/wiz-findings.js"; import { normalizeOpenGrepFindings } from "../src/opengrep-findings.js"; import { @@ -101,28 +87,17 @@ import type { } from "../src/external-import-models.js"; import { main } from "../src/cli.js"; import { dependencies } from "./cli-fixtures.js"; -import { normalizeGitHubDependabotAlerts } from "../src/github-dependabot-findings.js"; import { githubDependabotAlert } from "./support/github-dependabot-fixtures.js"; import { importedAlert, importedAdvisory, } from "./support/github-import-fixtures.js"; import { createCliTest } from "./support/cli-run.js"; -import { - normalizePreviousTrivyFindings, - normalizeTrivyFindings, -} from "../src/trivy-findings.js"; -import { - normalizeInspectorFindings, - normalizePreviousInspectorFindings, -} from "../src/inspector-findings.js"; -import { - normalizePreviousContinuumFindings, - readContinuumFindings, -} from "../src/continuum-findings.js"; +import { normalizeTrivyFindings } from "../src/trivy-findings.js"; +import { normalizeInspectorFindings } from "../src/inspector-findings.js"; +import {} from "../src/continuum-findings.js"; import { trivyReport, - trivyReportWithManyLocations, trivyRepositoryHeader, } from "./support/trivy-import-fixtures.js"; import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; @@ -137,14 +112,8 @@ import { continuumDetail, continuumRepository, } from "./support/continuum-import-fixtures.js"; -import { - normalizeBlackDuckFindings, - normalizePreviousBlackDuckFindings, -} from "../src/blackduck-findings.js"; -import { - normalizePreviousVeracodeFindings, - normalizeVeracodeFindings, -} from "../src/veracode-findings.js"; +import { normalizeBlackDuckFindings } from "../src/blackduck-findings.js"; +import { normalizeVeracodeFindings } from "../src/veracode-findings.js"; import { veracodeMixedPolicyPage, veracodePolicyPage, @@ -362,97 +331,6 @@ test("a Wiz Raw Event export explains the finding-report path before contacting expect(contactedCloud).toBe(false); }); -test("Dependabot keeps an older approval when an emitted boxed number excludes a row", async () => { - const f = await fixture(); - const selection = { - ...options, - provider: "github" as const, - sourceKey: "github.com/example/project/dependabot", - }; - const a = githubDependabotAlert(1); - const b = { ...githubDependabotAlert(2), score: new Number(NaN) }; - const prepare = (rows: unknown[], target: "local" | "cloud" = "cloud") => - prepareReleased( - f, - normalizeGitHubDependabotAlerts(rows, target), - selection, - ); - expect((await prepare([a, b])).preview.requests[0]!.items).toHaveLength(1); - - // The released reader admitted both rows and serialized the non-finite wrapper to null. - const request = { - ...(await prepare([a, { ...b, score: null }])).preview.requests[0]!, - request_id: "e1a703b7-7e69-4a6b-a6c5-02ff7a47fd45", - }; - expect(hash("sha256", JSON.stringify(request))).toBe( - "c26b480acaefaa832105d472abf05a372c9eb3d7c0749c15700e802466cb4b3a", - ); - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "a79e3af4845c4c7b5e0abffb5ed37054bb2debdf57b8a8b10acf3d7bd36a5e9f.pending.json", - requests: [request], - inputDigest: - "7b23cc522ea792cfa321415c62c5732421fb63f5c1efbba9fa6f81fb086c9e7f", - }); - - for (const target of ["local", "cloud"] as const) { - await expect(prepare([a, b], target)).rejects.toThrow( - "checkpoint was preserved", - ); - } - expect(f.posts).toHaveLength(0); - expect(await readFile(path, "utf8")).toBe(saved); - const changed = { - ...a, - security_advisory: { ...a.security_advisory, summary: "Changed advisory" }, - }; - expect((await prepare([changed, b])).preview.resumed).toBe(false); -}); - -test("Dependabot retries the approved subset when a numeric size correction admits another alert", async () => { - const f = await fixture(); - const selection = { - ...options, - provider: "github" as const, - sourceKey: "github.com/example/project/dependabot", - }; - const a = githubDependabotAlert(1); - const b = { - ...githubDependabotAlert(2), - score: new Number(0.00001), - pad: "", - }; - const bare = normalizeGitHubDependabotAlerts([b], "cloud").findings[0]! - .evidence; - b.pad = "x".repeat(256 * 1024 + 1 - Buffer.byteLength(JSON.stringify(bare))); - const normalized = normalizeGitHubDependabotAlerts([a, b], "cloud"); - const prepare = () => prepareReleased(f, normalized, selection); - const fresh = (await prepare()).preview.requests[0]!; - expect(fresh.items).toHaveLength(2); - - // Recorded from the released reader when only alert 1 fit its size calculation. - const request = { - ...fresh, - request_id: "95af968c-8f7c-4cb5-a79a-561eb06e39e8", - items: fresh.items.slice(0, 1), - }; - expect(hash("sha256", JSON.stringify(request))).toBe( - "bdec6e68ba764e63a60c83e14d7b691b03332c7af1c4ac031c2b5ca1cb92c69f", - ); - await stageReleasedCheckpoint(f, { - filename: - "c99d3e8e89437eeb0462959cc3504183d1c67b8bc5a07bfe0af58da058bca72d.pending.json", - requests: [request], - inputDigest: - "50c57ce23ee63e16bd06580d955f37110cdc827b26b4737c58ae13bfb88c2bca", - }); - - const resumed = await prepare(); - expect(resumed.preview.resumed).toBe(true); - expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request)]); -}); - test("Dependabot publishes on the existing GitHub provider and rejects foreign repositories before POST", async () => { const input = { ...githubDependabotAlert(), source: "extra" }; const correct = await fixture([input]); @@ -1472,15 +1350,14 @@ test("Cloud Semgrep reader references retain the Git HTTP host", async () => { test("a Semgrep file retry preserves the approved private repository's case", async () => { const url = "https://scm.example.test/Team/Project"; - const ref = "refs/heads/topic"; const report = { - format: "semgrep-platform-findings/v1", - deployment: "example", - project: { id: 10, name: "Team/Project", url }, - ref, - product: "sast", + ...selectedSemgrepReport( + { name: "Team/Project", url }, + { rule_name: "example.rule" }, + ), + format: semgrepPlatformFormat, findingCount: 1, - findings: [{ id: 12, ref, repository: { url }, rule_name: "example.rule" }], + repositoryAliases: [], }; const f = await hostedFixture("semgrep", report, url); const settings = { ...options, provider: "semgrep" as const }; @@ -1853,13 +1730,8 @@ test("a saved selection with Guardian evidence cannot resume under another provi const prepare = () => prepareExternalPublication(f.file, options, f.deps); const path = await f.loseResponse(await prepare()); const saved = JSON.parse(await readFile(path, "utf8")); - const { source_finding_id, evidence } = saved.requests[0].items[0]; + const { evidence } = saved.requests[0].items[0]; evidence.source_data = { format: "semgrep-guardian-selection/v1" }; - // An older reader can store a different projection of the same finding. - saved.inputProjection.savedEvidenceDigest = hash( - "sha256", - canonicalJson([{ source_finding_id, evidence }]), - ); const bytes = JSON.stringify(saved); await writeFile(path, bytes); await expect(prepare()).rejects.toThrow(/semgrep provider/); @@ -1923,92 +1795,6 @@ test("a normalized Semgrep report checks the provider before Cloud credentials", ).rejects.toThrow("requires the semgrep provider"); }); -test.each(["ticket", "HEAD", "release"] as const)( - "Cloud replays a previously approved Semgrep request with its original scope checks (%s)", - async (kind) => { - const url = "https://github.com/example/project"; - const other = "https://github.com/example/other"; - const ref = kind === "ticket" ? "refs/heads/release" : kind; - const scope = { - deployment: "example", - projectId: "14", - ref, - product: "sast", - }; - const capture = { - format: "semgrep-platform-findings/v1", - deployment: scope.deployment, - ref, - product: scope.product, - project: { id: 14, name: "example/project", url }, - findingCount: 1, - findings: [{ id: 23, rule_name: "example.rule", ref }], - }; - const f = await hostedFixture("semgrep", capture, url); - const report = normalizeSemgrepFindings(capture, {}, "cloud"); - const row = report.findings[0]!; - const bare = { - read: 1, - excluded: [], - findings: [row], - sourceRepositoryUrl: url, - }; - const marked = { - ...bare, - sourceFormat: "semgrep-platform-findings/v1" as const, - semgrepPlatformScope: scope, - }; - if (kind === "ticket") { - row.source_finding_id = "operator-assigned"; - row.evidence.source_data = { - format: "ticket/v1", - finding: { id: "TRACK-23", repository: { url: other } }, - }; - await expect(f.prepareNormalized(marked)).rejects.toThrow( - /original collection scope/iu, - ); - } - const path = await f.loseResponse( - await f.prepareNormalized(kind === "ticket" ? bare : marked), - ); - const saved = JSON.parse(await readFile(path, "utf8")); - // The released publisher stored whether a report was marked and the original evidence digests. - const { inputDigest, savedEvidenceDigest } = saved.inputProjection; - saved.inputProjection = { - retainedHostedSelection: true, - inputDigest, - savedEvidenceDigest, - }; - const bytes = JSON.stringify(saved); - await writeFile( - path, - JSON.stringify({ ...saved, accountId: "other-account" }), - ); - await expect(f.prepareNormalized(marked)).rejects.toThrow(/account/iu); - await writeFile(path, bytes); - await expect( - f.prepareNormalized({ ...marked, sourceRepositoryUrl: other }), - ).rejects.toThrow(/scope|repository/iu); - if (kind !== "ticket") { - await expect( - f.prepareNormalized({ - ...marked, - semgrepPlatformScope: { ...scope, ref: "refs/heads/" + ref }, - }), - ).rejects.toThrow(/scope|ref/iu); - } - expect(await readFile(path, "utf8")).toBe(bytes); - expect(f.posts).toHaveLength(1); - const retry = await f.prepareNormalized(marked); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts).toEqual([ - JSON.stringify(saved.requests[0]), - JSON.stringify(saved.requests[0]), - ]); - }, -); - test.each(["ticket metadata", "retained ref"] as const)( "a mixed Semgrep request retries with an unmarked row containing %s", async (kind) => { @@ -2074,7 +1860,7 @@ test("Cloud binds newly approved marked ticket context to its deployment", async const input = { ...report, sourceRepositoryUrl: url, - sourceFormat: "semgrep-platform-findings/v1" as const, + sourceFormat: semgrepPlatformFormat, semgrepPlatformScope: scope, }; const path = await f.loseResponse(await f.prepareNormalized(input)); @@ -2093,232 +1879,6 @@ test("Cloud binds newly approved marked ticket context to its deployment", async expect(f.posts[1]).toBe(f.posts[0]); }); -test.each(["marked", "unmarked"] as const)( - "Cloud preserves a Semgrep approval's scope when recovering the raw checkpoint key during %s retries", - async (retryKind) => { - const changedScope = retryKind === "marked"; - const url = "https://scm.example.test/Org/Repo"; - const capture = { - format: semgrepPlatformFormat, - repositoryAliases: ["org/repo"], - deployment: "example", - ref: "refs/heads/release", - product: "sast", - project: { id: "14", name: "Org/Repo", url }, - findingCount: 1, - findings: [ - JSON.stringify({ - id: 23, - rule_name: "sample.rule", - repository: { id: 14, name: "org/repo" }, - ref: "refs/heads/release", - }), - ], - }; - const f = await hostedFixture("semgrep", capture, url); - const native = normalizeSemgrepPlatformFindings(capture); - const ticket = { - source_finding_id: "custom", - evidence: { - title: "Ticket", - severity: "high" as const, - source_data: { finding: { ref: "refs/heads/other" } }, - }, - }; - const findings = [...native.findings, ticket]; - const bare = { - read: 2, - excluded: [], - findings, - sourceRepositoryPlatform: "other" as const, - sourceRepositoryUrl: url, - }; - const marked = { ...native, read: 2, findings }; - const mapped = { ...bare, sourceRepositoryPlatform: "github" as const }; - const path = await f.loseResponse(await f.prepareNormalized(bare)); - const bytes = await readFile(path, "utf8"); - - // The raw publisher installs its checkpoint after the mapped reader misses the raw key. - const staged = path + ".staged"; - await fs.rename(path, staged); - const originalRead = fs.readFile; - let installed = false; - const lookup = spyOn(fs, "readFile").mockImplementation((( - ...args: Parameters - ) => { - const result = originalRead(...args); - if (args[0] !== path || installed) return result; - installed = true; - return result.finally(() => fs.rename(staged, path)); - }) as typeof fs.readFile); - try { - if (changedScope) - await expect(f.prepareNormalized(marked)).rejects.toThrow( - /approved request/iu, - ); - else - expect((await f.prepareNormalized(mapped)).preview.resumed).toBe(true); - } finally { - lookup.mockRestore(); - } - expect(installed).toBe(true); - if (changedScope) expect(await readFile(path, "utf8")).toBe(bytes); - else { - await expect(fs.stat(path)).rejects.toMatchObject({ code: "ENOENT" }); - await expect(f.prepareNormalized(marked)).rejects.toThrow( - /approved request/iu, - ); - } - expect(f.posts).toHaveLength(1); - const resumed = await f.prepareNormalized(changedScope ? bare : mapped); - expect(resumed.preview.requests).toEqual(JSON.parse(bytes).requests); - expect(resumed.preview.resumed).toBe(true); - await resumed.publish(); - expect(f.posts[1]).toBe(f.posts[0]); - }, -); - -test("Cloud checks both native Semgrep rows when a projection retains both formats", async () => { - const url = "https://github.com/example/project"; - const f = await hostedFixture( - "semgrep", - selectedSemgrepReport( - { name: "example/project", url }, - { rule_name: "sample.rule" }, - ), - url, - ); - const { findings } = (await f.prepareFile()).preview; - const input = { read: 1, excluded: [], findings }; - const raw = findings[0]!.evidence.source_data!; - const json = String(raw["semgrepPlatformFindingJson"]); - const row = { ...JSON.parse(json), attempts: 1, scoreRatio: 1.25 }; - raw["finding"] = { annotation: "operator note" }; - expect((await f.prepareNormalized(input)).preview.findings).toHaveLength(1); - const score = "9007199254740993"; - raw["semgrepPlatformFindingJson"] = - `{"attempts":1.0,"scoreRatio":1.250,"score":${score},${json.slice(1)}`; - raw["finding"] = { ...row, score }; - raw["numericFields"] = { finding: { score } }; - raw["format"] = "semgrep-platform-findings/v1"; - raw["repositoryAliases"] = []; - expect((await f.prepareNormalized(input)).preview.findings).toHaveLength(1); - raw["finding"] = { - ...row, - score, - repository: { url: "https://github.com/foreign/project" }, - }; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - raw["format"] = "semgrep-platform-selection/v1"; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - raw["finding"] = { ...row, score, id: 31 }; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - expect(f.posts).toHaveLength(0); -}); - -test("textual and witnessed Semgrep finding IDs are checked even above the signed range", async () => { - const url = "https://github.com/example/project"; - const report = selectedSemgrepReport( - { name: "example/project", url }, - { rule_name: "sample.rule" }, - ); - const id = "1000000000000000000002"; - report.findings[0] = report.findings[0]!.replace('"id":23', '"id":' + id); - const f = await hostedFixture("semgrep", report, url); - const { findings } = (await f.prepareFile()).preview; - const input = { read: 1, excluded: [], findings }; - const raw = findings[0]!.evidence.source_data!; - const original = { - ...JSON.parse(String(raw["semgrepPlatformFindingJson"])), - id, - }; - raw["numericFields"] = { finding: { id } }; - raw["finding"] = { - ...original, - repository: { url: "https://github.com/foreign/project" }, - }; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - raw["finding"] = original; - expect((await f.prepareNormalized(input)).preview.findings).toHaveLength(1); - const otherId = "1000000000000000000003"; - raw["finding"] = { ...original, id: otherId }; - raw["numericFields"] = { finding: { id: otherId } }; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - const numericJson = raw["semgrepPlatformFindingJson"]; - delete raw["numericFields"]; - raw["semgrepPlatformFindingJson"] = JSON.stringify(original); - raw["finding"] = { - ...original, - repository: { url: "https://github.com/foreign/project" }, - }; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - raw["finding"] = original; - expect((await f.prepareNormalized(input)).preview.findings).toHaveLength(1); - raw["semgrepPlatformFindingJson"] = numericJson; - await expect(f.prepareNormalized(input)).rejects.toThrow( - /Semgrep.*disagree/i, - ); - expect(f.posts).toHaveLength(0); -}); - -test.each([true, false])( - "a new Semgrep reader mapping rechecks the original saved SCA row (exact case %s)", - async (exact) => { - const url = "https://scm.example.test/Team/Project"; - const report = { - format: semgrepPlatformFormat, - deployment: "example", - project: { id: "14", name: "Team/Project", url }, - product: "sca", - ref: "refs/heads/topic", - repositoryAliases: [], - findingCount: 1, - findings: [ - JSON.stringify({ - id: 23, - rule_name: "sample.rule", - repository: { url: exact ? url : url.toLowerCase() }, - }), - ], - }; - const input = normalizeSemgrepFindings( - report, - { sourceRepositories: [{ url, platform: "github" }] }, - "cloud", - ); - const f = await hostedFixture("semgrep", report, url); - const strict = () => - prepareNormalizedExternalPublication( - input, - { ...options, provider: "semgrep" }, - { - ...f.deps, - sourceRepositories: [{ url, platform: "gitlab" }], - }, - ); - if (!exact) await expect(strict()).rejects.toThrow(/repository/iu); - const pending = await f.loseResponse(await f.prepareNormalized(input)); - const approved = await readFile(pending, "utf8"); - if (exact) expect((await strict()).preview.resumed).toBe(true); - else await expect(strict()).rejects.toThrow(/repository/iu); - expect(await readFile(pending, "utf8")).toBe(approved); - await (await f.prepareNormalized(input)).publish(); - expect(f.posts).toEqual([f.posts[0]!, f.posts[0]!]); - }, -); - test("marked Semgrep rows require their provider while unmarked Wiz records can publish", async () => { const url = "https://github.com/example/project"; const f = await fixture([], url); @@ -2354,7 +1914,7 @@ test("marked Semgrep rows require their provider while unmarked Wiz records can f.deps, ); await expect( - prepare("1.2e1", null, "wiz", "semgrep-platform-findings/v1"), + prepare("1.2e1", null, "wiz", semgrepPlatformFormat), ).rejects.toThrow(/semgrep provider/i); expect((await prepare("1.2e1", null)).preview.findings).toHaveLength(1); await expect(prepare(null, null, "semgrep")).rejects.toThrow(/Semgrep/i); @@ -2367,16 +1927,6 @@ test("marked Semgrep rows require their provider while unmarked Wiz records can ); }); -async function useReleasedCheckpoint(path: string) { - const captured = await readFile(path, "utf8"); - const pending = JSON.parse(captured); - delete pending.approvedRepositoryUrl; - delete pending.approvedRepositoryPlatform; - await writeFile(path, JSON.stringify(pending)); - - return () => writeFile(path, captured); -} - test.each(["semgrep", "snyk"] as const)( "%s hosted findings bind the original repository and replay the approved request", async (provider) => { @@ -2465,7 +2015,7 @@ test.each(["semgrep", "snyk"] as const)( expect(f.posts).toHaveLength(0); f.state.repositoryUrl = url; - const pendingPath = await f.loseResponse(first); + await f.loseResponse(first); if (provider === "semgrep") { const { snykRestContext } = normalizeSnykIssues({ ...snykRestFixture("code", { @@ -2490,13 +2040,6 @@ test.each(["semgrep", "snyk"] as const)( expect(f.posts).toHaveLength(1); } f.state.repositoryUrl = "https://github.com/example/renamed"; - if (provider === "snyk") { - const restore = await useReleasedCheckpoint(pendingPath); - // The released publisher checked typed repository URLs only. The saved SCA project's - // original JSON does not establish which URL the catalog had at approval. - await expect(f.prepareFile()).rejects.toThrow(/does not match/); - await restore(); - } // A normalized caller can omit the redundant project/platform hints on the next try. const projected = await f.prepareNormalized(normalized); expect(projected.preview.resumed).toBe(true); @@ -2572,136 +2115,19 @@ test("Semgrep Cloud validates a normalized report's monitored scope on import an expect((await retry.publish()).verified).toBe(1); expect(f.posts[1]).toBe(f.posts[0]); - const projected = (title: string, scoped = true) => + const projectTicket = (title: string) => f.prepareNormalized({ - ...(scoped - ? { ...normalized, semgrepPlatformScope } - : { read: 1, excluded: [] }), + ...normalized, + semgrepPlatformScope, findings: [ { source_finding_id: "ticket", evidence: { title, severity: "high" } }, ], }); - await f.loseResponse(await projected("Initial title")); - await expect(projected("New title")).rejects.toThrow( - "pending hosted Cloud request", - ); - await expect(projected("New title", false)).rejects.toThrow( - "pending hosted Cloud request", - ); -}); - -test("Semgrep checks an alias witness before excluding it from the Cloud preview", async () => { - const url = "https://scm.example.test/Org/Project"; - const report = (witnessUrl: string) => ({ - format: semgrepPlatformFormat, - deployment: "example", - project: { id: "14", name: "Org/Project", url }, - product: "sast", - ref: "refs/heads/topic", - findingCount: 2, - repositoryAliases: ["display"], - findings: [ - JSON.stringify({ - id: 11, - rule_name: "sample.rule", - repository: { name: "display" }, - }), - JSON.stringify({ - id: 12, - rule_name: "x".repeat(513), - repository: { name: "display", url: witnessUrl }, - }), - ], - }); - const f = await hostedFixture("semgrep", report(url), url); - const matching = await f.prepareFile(); - expect(matching.preview.findings).toHaveLength(1); - expect(matching.preview.excluded).toHaveLength(1); - const wrong = report("https://scm.example.test:443/Org/Project"); - expect(normalizeSemgrepFindings(wrong, {}, "local").findings).toHaveLength(2); - await writeFile(f.file, JSON.stringify(wrong)); - await expect(f.prepareFile()).rejects.toThrow(/repository/i); - expect(f.posts).toHaveLength(0); -}); - -test("an original Semgrep v1 name-only finding needs a repository witness in the same batch and scope", async () => { - const f = await fixture(); - const url = f.state.repositoryUrl; - const project = { id: 10, name: "example/project", url }; - const ref = "refs/heads/topic"; - const label = "display"; - function rows(repository = project, branch = ref) { - const scope = { - format: "semgrep-platform-findings/v1", - deployment: "demo", - project: repository, - ref: branch, - product: "sast", - repositoryAliases: [label], - }; - const row = { ref: branch, rule: { name: "fixture.rule" } }; - const native = [ - { ...row, id: 12, repository: { name: label } }, - { - ...row, - id: 13, - repository: { - name: label, - id: String(repository.id), - url: repository.url, - }, - }, - ]; - const { findings } = normalizeSemgrepFindings( - { - ...scope, - findingCount: native.length, - findings: native, - }, - {}, - "cloud", - ); - for (const [i, finding] of findings.entries()) - finding.evidence.source_data = { ...scope, finding: native[i] }; - return findings; - } - const batch = rows(); - const prepare = (findings: typeof batch) => { - const input = { - findings, - read: findings.length, - excluded: [], - sourceRepositoryUrl: url, - }; - return prepareNormalizedExternalPublication( - input, - { ...options, provider: "semgrep" }, - f.deps, - ); - }; - for (const unrelated of [ - rows({ ...project, id: 11 }), - rows(project, "refs/heads/other"), - rows({ - ...project, - name: "other/project", - url: "https://github.com/other/project", - }), - ]) { - await expect(prepare([batch[0]!, unrelated[1]!])).rejects.toThrow( - /witness|alias/, - ); - } - expect(f.posts).toHaveLength(0); - const first = await prepare(batch); - const witness = - first.preview.findings[0]!.evidence.source_data?.["repositoryWitnessJson"]; - expect(JSON.parse(String(witness))).toEqual({ - name: label, - id: String(project.id), - }); - expect((await first.publish()).counts.created).toBe(2); - expect((await (await prepare(batch)).publish()).counts.unchanged).toBe(2); + const original = await projectTicket("Initial title"); + await f.loseResponse(original); + expect((await projectTicket("New title")).preview.resumed).toBe(false); + const again = await projectTicket("Initial title"); + expect(again.preview.requests).toEqual(original.preview.requests); }); test("Semgrep Supply Chain binds package evidence to its case-sensitive source project", async () => { @@ -2810,142 +2236,7 @@ test("Semgrep Supply Chain binds package evidence to its case-sensitive source p expect(f.posts).toHaveLength(0); }); -test.each(["open-source", "code"] as const)( - "a released Snyk %s checkpoint retains one request across projections and renames", - async (product) => { - const url = "https://github.com/example/project"; - const capture = snykRestFixture(product, { sourceRepositoryUrl: url }); - const input = normalizeSnykIssues(capture); - delete input.findings[0]!.evidence.source_data; - const f = await hostedFixture("snyk", capture, url); - Reflect.deleteProperty(f.environment, "CODEX_SECURITY_CLOUD_BASE_URL"); - let repository = url; - const bare = { - read: input.read, - findings: input.findings, - excluded: input.excluded, - }; - const projected = Object.assign( - Object.create({ - get sourceRepositoryUrl() { - return repository; - }, - }) as typeof input, - bare, - ); - const prepare = () => f.prepareNormalized(projected); - - await expect(prepare()).rejects.toThrow(/Snyk original issue/); - expect(f.posts).toHaveLength(0); - - // The released SDK approved these findings without retaining a native issue. - const { file, checkpoint } = releasedSnykCheckpoints[product]; - await f.writeCheckpoint(file, checkpoint); - const first = await f.prepareNormalized(bare); - expect(first.preview.resumed).toBe(true); - await f.loseResponse(first); - await expect(f.prepareFile()).rejects.toThrow( - "pending hosted Cloud request", - ); - expect(f.posts).toEqual([JSON.stringify(checkpoint.requests[0])]); - - f.state.repositoryUrl = "https://github.com/example/renamed"; - await expect(prepare()).rejects.toThrow(/does not match/); - expect((await f.prepareNormalized(bare)).preview.resumed).toBe(true); - f.state.repositoryUrl = url; - expect((await prepare()).preview.resumed).toBe(true); - const bareRetry = await f.prepareNormalized(bare); - expect(bareRetry.preview.requests).toEqual(first.preview.requests); - repository = "https://github.com/example/different"; - await expect(prepare()).rejects.toThrow(/does not match/); - repository = url; - // Binding while the catalog still matches establishes the URL for a future rename. - await f.loseResponse(await prepare()); - f.state.repositoryUrl = "https://github.com/example/renamed"; - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts).toHaveLength(3); - expect(new Set(f.posts).size).toBe(1); - }, -); - -test("a released Snyk checkpoint cannot authenticate the case of a typed URL", async () => { - const url = "https://scm.example.test/Team/Repo"; - const f = await hostedFixture("snyk", [], url.toLowerCase()); - const report = snykRestFixture("code", { sourceRepositoryUrl: url }); - const project = JSON.parse(report.projectJson); - project.attributes.origin = "gitlab"; - report.projectJson = JSON.stringify(project); - const { snykRestContext, findings } = normalizeSnykIssues(report); - findings[0]!.source_finding_id = "projected"; - delete findings[0]!.evidence.source_data; - const bare = { read: 1, excluded: [], findings }; - const pending = await f.loseResponse(await f.prepareNormalized(bare)); - await useReleasedCheckpoint(pending); - - await expect( - f.prepareNormalized({ ...bare, snykRestContext }), - ).rejects.toThrow(/does not match/); - expect((await f.prepareNormalized(bare)).preview.resumed).toBe(true); -}); - -test.each([ - ["Kit", "Kit"], - ["Kit", "Kit"], - ["åpp", "Åpp"], -])( - "released catalog spelling %s cannot authorize %s after a rename", - async (catalog, typed) => { - const root = "https://github.com/example/"; - const url = root + typed; - const f = await hostedFixture("snyk", [], root + catalog); - const { findings } = normalizeSnykIssues( - snykRestFixture("code", { sourceRepositoryUrl: url }), - ); - findings[0]!.source_finding_id = "reader-issue"; - delete findings[0]!.evidence.source_data; - const bare = { read: 1, excluded: [], findings }; - const pending = await f.loseResponse(await f.prepareNormalized(bare)); - await useReleasedCheckpoint(pending); - const original = await readFile(pending); - - f.state.repositoryUrl = root + "renamed"; - await expect( - f.prepareNormalized({ ...bare, sourceRepositoryUrl: url }), - ).rejects.toThrow(/does not match/); - expect(await readFile(pending)).toEqual(original); - const retry = await f.prepareNormalized(bare); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts[1]).toBe(f.posts[0]); - }, -); - -test("a rejected projected Black Duck row leaves a released Snyk checkpoint recoverable", async () => { - const url = "https://github.com/example/project"; - const f = await hostedFixture("snyk", [], url); - const input = normalizeBlackDuckFindings([blackDuckBomRow()], url); - input.findings[0]!.evidence.source_data = { sourceRepositoryUrl: url }; - const prepare = (sourceRepositoryUrl: string) => - f.prepareNormalized({ ...input, sourceRepositoryUrl }); - const pending = await f.loseResponse(await prepare(url)); - await useReleasedCheckpoint(pending); - const original = await readFile(pending); - - f.state.repositoryUrl = "https://github.com/example/renamed"; - await expect(prepare(f.state.repositoryUrl)).rejects.toThrow( - /does not match/, - ); - expect(await readFile(pending)).toEqual(original); - f.state.repositoryUrl = url; - const retry = await prepare(url); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts[1]).toBe(f.posts[0]); -}); - -test("a Snyk retry adopts case sensitivity when the operator publishes", async () => { +test("a Snyk retry adopts case sensitivity when the operator publishes", async () => { const url = "https://scm.example.test/Team/Repo"; const f = await hostedFixture("snyk", [], url); const bare = { @@ -2967,16 +2258,7 @@ test("a Snyk retry adopts case sensitivity when the operator publishes", async ( const alias = () => f.prepareNormalized({ ...bare, sourceRepositoryUrl: url.toLowerCase() }); const initial = await strict(); - const pending = await f.loseResponse(initial); - await expect(alias()).rejects.toThrow(/does not match/); - - // The released SDK did not record a repository URL or SCM policy. - await useReleasedCheckpoint(pending); - const strictRetry = await strict(); - const aliasRetry = await alias(); - expect(f.posts).toHaveLength(1); - await f.loseResponse(aliasRetry); - await f.loseResponse(strictRetry); + await f.loseResponse(initial); await expect(alias()).rejects.toThrow(/does not match/); f.state.repositoryUrl = "https://scm.example.test/Team/Renamed"; @@ -2993,7 +2275,7 @@ test("a Snyk retry adopts case sensitivity when the operator publishes", async ( initial.preview.requests, ); await retry.publish(); - expect(f.posts).toEqual(Array(4).fill(f.posts[0])); + expect(f.posts).toEqual([f.posts[0]!, f.posts[0]!]); }); test("Snyk Code accepts equivalent URL escapes and still rejects a foreign typed row", async () => { @@ -3053,90 +2335,6 @@ test.each(["semgrep", "snyk"] as const)( }, ); -test.each(["file", "operator"])( - "Semgrep v1 keeps the native project's explicit port when checking %s mappings", - async (mapping) => { - const url = "https://scm.example.test:443/team/project"; - const ref = "refs/heads/release"; - const report = { - format: "semgrep-platform-findings/v1", - deployment: "example", - project: { id: 14, name: "team/project", url }, - ref, - product: "sast", - findingCount: 1, - findings: [{ id: 23, ref, rule: { name: "sample.check" } }], - }; - const f = await fixture(report, url); - const prepare = async (sourceRepositoryUrl: string) => { - if (mapping === "file") - await writeFile( - f.file, - JSON.stringify({ ...report, sourceRepositoryUrl }), - ); - return prepareExternalPublication( - f.file, - { - ...options, - provider: "semgrep", - ...(mapping === "operator" && { sourceRepositoryUrl }), - }, - f.deps, - ); - }; - const withoutPort = url.replace(":443", ""); - for (const catalog of [withoutPort, url]) { - f.state.repositoryUrl = catalog; - await expect(prepare(withoutPort)).rejects.toThrow(/does not match/); - } - expect((await prepare(url)).preview.findings).toHaveLength(1); - expect(f.posts).toHaveLength(0); - }, -); - -test.each([true, false])( - "a Semgrep v1 private SSH file retries after a rename (operator mapping %s)", - async (mapped) => { - const url = "https://scm.example.test/Org/Repo"; - const ref = "refs/heads/release"; - const f = await fixture( - { - format: "semgrep-platform-findings/v1", - deployment: "example", - project: { - id: 14, - name: "Org/Repo", - url: "git@scm.example.test:org/repo.git", - }, - product: "sast", - ref, - findingCount: 1, - findings: [ - { id: 23, ref, rule: { name: "sample.rule" }, repository: { url } }, - ], - }, - url, - ); - const prepare = (sourceRepositoryUrl = mapped ? url : undefined) => - prepareExternalPublication( - f.file, - { ...options, provider: "semgrep", sourceRepositoryUrl }, - f.deps, - ); - await f.loseResponse(await prepare()); - f.state.repositoryUrl = "https://scm.example.test/Org/Renamed"; - await expect( - prepare(url.replace(".test/", ".test:443/")), - ).rejects.toThrow(); - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts).toHaveLength(2); - expect(f.posts[1]).toBe(f.posts[0]); - await expect(prepare()).rejects.toThrow(/repository|project/iu); - }, -); - test.each([true, false])( "an escaped projected repository in a Snyk checkpoint matches the current destination (declared %s)", async (declared) => { @@ -3212,7 +2410,6 @@ test.each(["semgrep", "snyk"] as const)( bare.findings[0]!.evidence.source_data = { sourceRepositoryUrl: url }; const declared = { ...bare, sourceRepositoryUrl: url }; const pending = await f.loseResponse(await f.prepareNormalized(bare)); - await useReleasedCheckpoint(pending); expect((await f.prepareNormalized(declared)).preview.resumed).toBe(true); const saved = await readFile(pending); @@ -3247,1136 +2444,505 @@ test("a private Snyk GitHub project uses its authenticated catalog alias", async expect(f.posts).toHaveLength(0); }); -test("an original Semgrep SDK row keeps the approved repository casing on retry", async () => { - const url = "https://scm.example.test/Team/Project"; - const report = selectedSemgrepReport( - { name: "Team/Project", url }, - { rule_name: "sample.rule" }, +function sonarCapture(repositoryUrl: string) { + return { + format: "sonar-issues/v1" as const, + server: "https://sonarcloud.io", + organization: "example", + project: "example_project", + branch: "main", + projectBase: "", + repositoryUrl, + pages: [ + { + paging: { pageIndex: 1, pageSize: 100, total: 1 }, + issues: [ + { + key: "issue-1", + project: "example_project", + component: "file", + rule: "example:R1", + message: "Review input handling", + type: "VULNERABILITY", + severity: "CRITICAL", + flows: [ + { + locations: [ + { + component: "file", + textRange: { startLine: 2, endLine: 3 }, + }, + ], + }, + ], + }, + ], + components: [{ key: "file", path: "src/example.ts" }], + }, + ], + }; +} + +test("Sonar replays the selected evidence locally and to Cloud without signing in to Sonar again", async () => { + const repo = "https://github.com/example/project"; + const source = sonarCapture(repo); + const f = await fixture(source, repo); + const checkout = join(f.root, "checkout"); + const git = promisify(execFile); + const environment = sonarGitTestEnvironment({ + ...f.cliDeps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: "https://cloud.example.test", + }); + await git("git", ["init", "--quiet", checkout], { env: environment }); + await git("git", ["-C", checkout, "remote", "add", "origin", repo], { + env: environment, + }); + const paths = new Set(); + const deps = { + ...f.cliDeps, + environment, + currentDirectory: () => checkout, + cloudFetch: (url: string, init: RequestInit) => { + paths.add(new URL(url).pathname); + return f.cliDeps.cloudFetch(url, init); + }, + }; + const first = createCliTest(main); + const previewExit = await first.runCli( + [ + "import", + "sonar", + "--sonar-project", + source.project, + "--sonar-branch", + source.branch, + "--sonar-issue", + "issue-1", + "--dry-run", + "--format", + "json", + ], + { + ...deps, + collectSonarIssues: async (selection) => { + expect(selection.issueKeys).toEqual(["issue-1"]); + return source; + }, + }, ); - const f = await hostedFixture("semgrep", report, url); - const { findings } = (await f.prepareFile()).preview; - const raw = findings[0]!.evidence.source_data!; - raw["format"] = "semgrep-platform-findings/v1"; - raw["finding"] = JSON.parse(report.findings[0]!); - delete raw["semgrepPlatformFindingJson"]; - raw["sourceRepositoryUrl"] = url; - const bare = { read: 1, findings, excluded: [] }; - const first = await f.prepareNormalized({ - ...bare, - sourceRepositoryPlatform: "other", + expect(previewExit, first.stderr.text()).toBe(0); + expect(f.posts).toHaveLength(0); + const capture: string = JSON.parse(first.stdout.text()).capture; + expect(JSON.parse(await readFile(capture, "utf8"))).toEqual(source); + const previewArgs = [ + "import", + "sonar", + capture, + "--to", + "cloud", + "--format", + "json", + ]; + + const outside = join(f.root, "without-remotes"); + await mkdir(outside); + const noRemotes = createCliTest(main); + expect( + await noRemotes.runCli([...previewArgs, "--dry-run"], { + ...deps, + currentDirectory: () => outside, + }), + ).toBe(2); + expect(noRemotes.stderr.text()).toContain("--source-repository-url"); + expect(f.posts).toHaveLength(0); + + const cloudPreview = createCliTest(main); + expect(await cloudPreview.runCli([...previewArgs, "--dry-run"], deps)).toBe( + 0, + ); + const reviewed = JSON.parse(cloudPreview.stdout.text()); + expect(reviewed.cloud.destination.url).toBe(repo); + expect(reviewed.capture).not.toBe(capture); + expect(f.posts).toHaveLength(0); + const args = [ + "import", + "sonar", + reviewed.capture, + "--to", + "cloud", + "--format", + "json", + ]; + + const unavailable = createCliTest(main); + expect( + await unavailable.runCli([...args, "--yes"], { + ...deps, + cloudFetch: async () => new Response("Unknown account", { status: 403 }), + }), + ).toBe(2); + expect(JSON.parse(unavailable.stdout.text()).local.counts.created).toBe(1); + + const replay = createCliTest(main); + expect(await replay.runCli([...args, "--yes"], deps)).toBe(0); + const result = JSON.parse(replay.stdout.text()); + expect(result.local.counts.unchanged).toBe(1); + expect(result.cloud.verified).toBe(1); + const posted = JSON.parse(f.posts[0]!); + expect(posted.source).toEqual({ + provider: "sonar", + source_key: result.sourceKey, }); - await f.loseResponse(first); - raw["sourceRepositoryUrl"] = url.toLowerCase(); - await expect(f.prepareNormalized(bare)).rejects.toThrow( - /original Semgrep input and SCM repository mapping/u, + expect(posted.items[0].source_finding_id).toBe("issue-1"); + const repository = f.destination(); + const reportsPath = `/external/v2/repository_connectors/${repository.repo_connector_id}/repositories/${repository.id}/source_reports`; + expect(paths).toEqual( + new Set([ + "/external/repositories", + reportsPath, + "/external/v2/finding_imports", + `${reportsPath}/${f.reports.get("issue-1")!.id}`, + ]), ); - raw["sourceRepositoryUrl"] = url; - expect((await f.prepareNormalized(bare)).preview.requests).toEqual( - first.preview.requests, + expect(posted.items[0].evidence.source_data.issue.flows).toEqual( + source.pages[0]!.issues[0]!.flows, ); }); -test("an original Semgrep SDK row keeps its approved GitHub alias after a rename", async () => { - const url = "https://scm.example.test/Org/Repo"; - const ref = "refs/heads/release"; - const scope = { - format: "semgrep-platform-findings/v1", - deployment: "example", - project: { id: 14, name: "Org/Repo", url }, - ref, - product: "sast", - }; +test("normalized Sonar publication checks retained bindings on preview and retry", async () => { + const f = await fixture(); + const repository = f.state.repositoryUrl; + const foreign = repository + "-other"; + const selected = { ...options, provider: "sonar" as const }; + const context = { format: "sonar-issues/v1", repositoryUrl: repository }; const finding = { - id: 23, - ref, - rule: { name: "sample.rule" }, - repository: { name: "org/repo", url }, + source_finding_id: "issue-1", + evidence: { + title: "Review input handling", + severity: "high" as const, + source_data: { ...context, issue: { key: "issue-1" } }, + }, }; - const f = await hostedFixture( - "semgrep", - { ...scope, findingCount: 1, findings: [finding] }, - url, + const rows = { read: 1, findings: [finding], excluded: [] }; + const named = { + source_finding_id: "issue-2", + evidence: { + ...finding.evidence, + branch: "main", + source_data: { ...context, branch: "main", issue: { key: "issue-2" } }, + }, + }; + const selection = { read: 2, findings: [finding, named], excluded: [] }; + const prepare = (findings = selection.findings) => + prepareNormalizedFixture( + f, + { ...selection, findings, report: { ...context, branch: "main" } }, + selected, + ); + await expect( + prepareNormalizedFixture(f, selection, selected), + ).rejects.toThrow("without a ref"); + await expect( + prepareNormalizedFixture( + f, + { + ...rows, + report: { ...context, branch: "main" }, + findings: [ + { + ...finding, + evidence: { ...finding.evidence, branch: "unrelated" }, + }, + ], + }, + selected, + ), + ).rejects.toThrow("retained Sonar branch"); + await expect( + prepareNormalizedFixture( + f, + { ...rows, report: { ...context, repositoryUrl: foreign } }, + selected, + ), + ).rejects.toThrow("does not match the selected Cloud repository"); + f.state.loseResponse = true; + await expect((await prepare()).publish()).rejects.toThrow( + "resume the saved request", ); - const first = await f.prepareFile(); - const findings = structuredClone(first.preview.findings); - findings[0]!.evidence.source_data = { ...scope, finding }; - const sdk = { read: 1, excluded: [], findings, sourceRepositoryUrl: url }; - await f.loseResponse(first); - f.state.repositoryUrl = "https://scm.example.test/Org/Renamed"; - const retry = await f.prepareNormalized(sdk); + await expect( + prepareNormalizedFixture( + f, + { ...selection, report: { ...context, branch: "other" } }, + selected, + ), + ).rejects.toThrow("conflicting branch"); + f.state.repositoryUrl = foreign; + await expect(prepare()).rejects.toThrow( + "does not match the selected Cloud repository", + ); + expect(f.posts).toHaveLength(1); + f.state.repositoryUrl = repository; + const retry = await prepare(); expect(retry.preview.resumed).toBe(true); await retry.publish(); + expect(f.posts).toHaveLength(2); expect(f.posts[1]).toBe(f.posts[0]); - await expect(f.prepareNormalized(sdk)).rejects.toThrow( - /repository|project/iu, - ); }); -test.each(["interrupted file", "completed SDK"])( - "Semgrep v1 file and original SDK rows reuse the same evidence (%s)", - async (first) => { - const url = "https://scm.example.test/Team/Project"; - const id = "9007199254740993"; - const scope = { - format: "semgrep-platform-findings/v1", - deployment: "example", - project: { id: 77, name: "Team/Project", url: null }, - sourceRepositoryUrl: url, - ref: "refs/heads/release", - product: "sast", - repositoryAliases: ["project"], - }; - const finding = { - id, - ref: scope.ref, - repository: { name: "project", url: url.toLowerCase() }, - rule: { name: "sample.check" }, - severity: "HIGH", - location: { file_path: "src\\service.ts", line: 3, end_line: 5 }, - }; - const report = { - ...scope, - findingCount: 1, - findings: [finding], - numericFields: { findings: { 0: { id } } }, - }; - const f = await hostedFixture("semgrep", report, url); - const { read, findings, excluded } = normalizeSemgrepFindings( - report, - { sourceRepositories: [{ url, platform: "github" }] }, - "cloud", - ); - const sdk = { read, findings, excluded }; - const evidence = findings[0]!.evidence; - evidence.severity = "unknown"; - evidence.locations = []; - evidence.details!.code!.end_line = null; - findings[0]!.evidence.source_data = { - ...scope, - finding, - numericFields: { finding: { id } }, - }; - const otherPort = structuredClone(sdk); - otherPort.findings[0]!.evidence.source_data!["sourceRepositoryUrl"] = - url.replace(".test/", ".test:443/"); - await expect(f.prepareNormalized(otherPort)).rejects.toThrow( - /repository/iu, - ); - if (first === "interrupted file") { - await f.loseResponse(await f.prepareFile()); - f.state.repositoryUrl = "https://scm.example.test/Team/Renamed"; - const retry = await f.prepareNormalized(sdk); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts[1]).toBe(f.posts[0]); - await expect(f.prepareNormalized(sdk)).rejects.toThrow( - /repository|project/i, - ); - } else { - expect( - (await (await f.prepareNormalized(sdk)).publish()).counts.created, - ).toBe(1); - expect((await (await f.prepareFile()).publish()).counts.unchanged).toBe( - 1, - ); - } - }, -); +test("Wiz occurrence mapping preserves original evidence without inventing repository provenance", async () => { + const record = { + id: "wiz-occurrence", + name: "CVE-2099-0001", + detailedName: "example-lib", + version: "1.2.0", + fixedVersion: "1.2.1", + vendorSeverity: "HIGH", + portalUrl: "https://example.test/finding", + locationPath: "/usr/lib/example", + lastDetectedAt: "2026-10-01T00:00:00Z", + projects: [{ id: "project-a" }, { id: "project-b" }], + packageManager: "legacy-manager", + vulnerableAsset: { id: "image-resource", imageDigest: "sha256:example" }, + }; + const f = await fixture([record]); + const parsed = await readVendorFindings(f.file); + expect(parsed.findings[0]).toMatchObject({ + source_finding_id: record.id, + evidence: { + severity: "high", + locations: [], + code_revision: null, + branch: null, + source_updated_at: null, + source_data: record, + image_digests: ["sha256:example"], + advisory_ids: [record.name], + packages: [{ name: "example-lib", ecosystem: "legacy-manager" }], + }, + }); +}); -test("a Semgrep selection with distinct submitted and native IDs remains independent", async () => { - const url = "https://github.com/example/project"; - const capture = selectedSemgrepReport( - { name: "example/project", url }, - { rule_name: "example.rule" }, - ); - const f = await hostedFixture("semgrep", capture, url); - const prepare = (nativeId: number, submittedId: string) => { - const findings = capture.findings.map((row) => - JSON.stringify({ ...JSON.parse(row), id: nativeId }), - ); - const input = normalizeSemgrepFindings( - { ...capture, findings }, - {}, - "cloud", - ); - input.findings[0]!.source_finding_id = submittedId; - return f.prepareNormalized(input); +test("Wiz API nodes preserve artifact ecosystems and distinguish container digests from VM image IDs", async () => { + const finding = { + name: "Vendor display advisory", + vulnerabilityExternalId: "CVE-2099-0001", + detailedName: "example-package", + version: "1.2.0", + vendorSeverity: "HIGH", }; - await f.loseResponse(await prepare(23, "first-ticket")); - await expect(prepare(23, "another-ticket")).rejects.toThrow( - "pending hosted Cloud request", + const records = [ + { + ...finding, + id: "container-package", + artifactType: { osPackageManager: "DPKG", codeLibraryLanguage: null }, + codeLibraryLanguage: "JAVA", + vulnerableAsset: { + id: "container-image", + type: "CONTAINER_IMAGE", + imageId: `sha256:${"a".repeat(64)}`, + }, + }, + { + ...finding, + id: "vm-library", + artifactType: { osPackageManager: null, codeLibraryLanguage: "PYTHON" }, + codeLibraryLanguage: "JAVA", + vulnerableAsset: { + id: "virtual-machine", + type: "VIRTUAL_MACHINE", + imageId: "ami-synthetic", + containerImageId: "ami-synthetic-alias", + }, + }, + { + ...finding, + id: "container-alias", + vulnerableAsset: { + id: "aliased-image", + type: "CONTAINER_IMAGE", + containerImageId: `sha256:${"d".repeat(64)}`, + }, + }, + { + ...finding, + id: "running-container", + vulnerableAsset: { + id: "running-image", + type: "CONTAINER", + ImageExternalId: `sha256:${"e".repeat(64)}`, + }, + }, + { + ...finding, + id: "explicit-metadata", + packageManager: "reported-manager", + imageDigest: `sha256:${"b".repeat(64)}`, + artifactType: { osPackageManager: "DPKG" }, + codeLibraryLanguage: "JAVA", + vulnerableAsset: { + id: "explicit-container", + type: "CONTAINER_IMAGE", + imageId: `sha256:${"c".repeat(64)}`, + }, + }, + { + ...finding, + id: "legacy-library", + codeLibraryLanguage: "JAVASCRIPT", + vulnerableAsset: { id: "legacy-image" }, + }, + ]; + const f = await fixture({ + data: { + vulnerabilityFindings: { + nodes: records, + pageInfo: { hasNextPage: false }, + }, + }, + }); + const parsed = await readVendorFindings(f.file); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]!.evidence.title).toContain(finding.name); + expect(parsed.findings.map(({ evidence }) => evidence.advisory_ids)).toEqual( + records.map(() => [finding.vulnerabilityExternalId]), ); - await expect(prepare(24, "first-ticket")).rejects.toThrow( - "pending hosted Cloud request", + expect( + parsed.findings.map(({ evidence }) => ({ + ecosystem: evidence.packages?.[0]?.ecosystem, + digests: evidence.image_digests, + })), + ).toEqual([ + { ecosystem: "DPKG", digests: [`sha256:${"a".repeat(64)}`] }, + { ecosystem: "PYTHON", digests: [] }, + { ecosystem: null, digests: [`sha256:${"d".repeat(64)}`] }, + { ecosystem: null, digests: [`sha256:${"e".repeat(64)}`] }, + { ecosystem: "reported-manager", digests: [`sha256:${"b".repeat(64)}`] }, + { ecosystem: "JAVASCRIPT", digests: [] }, + ]); + expect(parsed.findings.map(({ evidence }) => evidence.source_data)).toEqual( + records, ); - expect((await prepare(24, "another-ticket")).preview.resumed).toBe(false); - expect(f.posts).toHaveLength(1); }); -test.each(["semgrep", "snyk"] as const)( - "%s holds an overlapping native update until the initial request is recovered", - async (provider) => { - const url = "https://github.com/example/project"; - const report = - provider === "semgrep" - ? selectedSemgrepReport( - { name: "example/project", url }, - { rule_name: "example.rule" }, - ) - : snykRestFixture("open-source", { sourceRepositoryUrl: url }); - const f = await hostedFixture(provider, report, url); - const first = await f.prepareFile(); - const original = { - findings: first.preview.findings, - read: first.preview.findings.length, - excluded: [], - sourceRepositoryUrl: url, - report: { capture: "example" }, - }; - if (provider === "semgrep") - original.findings[0]!.source_finding_id = "caller-selected"; - const changed = structuredClone(original); - changed.findings[0]!.evidence.title += " updated"; - const stale = await f.prepareNormalized(changed); - f.state.marker = "next-generation"; - const challenger = await f.prepareNormalized(changed); - const winner = await f.prepareNormalized(original); - const observer = observeLockContention(); - const rename = fs.rename; - let competing: Promise | undefined; - const install = spyOn(fs, "rename").mockImplementationOnce( - async (...args) => { - // Start the other publisher after the winner's pending scan and before its atomic rename. - competing = challenger.publish().catch((error: unknown) => error); - const first = await Promise.race([ - observer.contended.then(() => "locked"), - competing.then(() => "completed"), - ]); - expect(first).toBe("locked"); - return rename(...args); - }, - ); - let pending: string; - try { - pending = await f.loseResponse(winner); - expect(String(await competing)).toContain("pending hosted Cloud request"); - } finally { - install.mockRestore(); - observer.restore(); - } - const bytes = await readFile(pending, "utf8"); - if (provider === "semgrep") { - const projected = structuredClone(changed); - delete projected.findings[0]!.evidence.source_data; - await expect(f.prepareNormalized(projected)).rejects.toThrow( - "pending hosted Cloud request", - ); - } - await expect(stale.publish()).rejects.toThrow("reset markers"); - - await expect(f.prepareFile()).rejects.toThrow( - "deployment could not be established", - ); - await expect(f.prepareNormalized(changed)).rejects.toThrow( - "pending hosted Cloud request", - ); - expect(await readFile(pending, "utf8")).toBe(bytes); - expect(f.posts).toHaveLength(1); - const captured = await f.prepareNormalized(original); - await (await f.prepareNormalized(original)).publish(); - expect(f.posts[1]).toBe(f.posts[0]); - const newer = await f.loseResponse(await f.prepareNormalized(changed)); - const approved = await readFile(newer, "utf8"); - await expect(captured.publish()).rejects.toThrow( - "pending hosted Cloud request", - ); - expect(f.posts).toHaveLength(3); - expect(await readFile(newer, "utf8")).toBe(approved); - expect( - (await (await f.prepareNormalized(changed)).publish()).counts.updated, - ).toBe(1); - }, -); - -test.each([ - ["semgrep", "native"], - ["semgrep", "SDK metadata and assigned IDs"], - ["semgrep", "released assigned IDs"], - ["semgrep", "released ID only"], - ["semgrep", "released assigned JSON row"], - ["snyk", "released assigned JSON row"], - ["snyk", "released assigned IDs"], - ["semgrep", "per-finding metadata"], - ["semgrep", "projected metadata"], - ["snyk", "released unmarked with metadata"], - ["snyk", "released compact issue JSON"], - ["semgrep", "released relabeled with metadata"], - ["semgrep", "released normalized SDK"], - ["semgrep", "released partial object row"], - ["semgrep", "released partial JSON row"], - ["semgrep", "released partial native project URL"], - ["semgrep", "released without format"], - ["snyk", "released without format"], - ["semgrep", "released unrelated evidence"], - ["snyk", "released unrelated evidence"], - ["snyk", "native"], - ["snyk", "projected evidence"], -] as const)( - "%s preserves the pending v1 request across a reader upgrade (%s)", - async (provider, mode) => { - const url = "https://github.com/example/project"; - const project = { id: 14, name: "example/project", url }; - const ref = "refs/heads/release"; - const finding = { - id: 23, - ref, - repository: { url }, - severity: "high", - rule_name: "example.rule", - }; - const nativeFindings = [finding, { ...finding, id: 24 }]; - const { projectJson, targetJson, pages, ...scope } = snykRestFixture( - "code", - { sourceRepositoryUrl: url }, - ); - const semgrep = { - format: "semgrep-platform-findings/v1", - deployment: "example", - project: mode === "native" ? { ...project, url: null } : project, - ...(mode === "native" && { sourceRepositoryUrl: url }), - product: "sast", - ref, - repositoryAliases: [], - }; - const report = - provider === "semgrep" - ? { - ...semgrep, - findingCount: nativeFindings.length, - findings: nativeFindings, - } - : { - ...scope, - format: "snyk-rest-issues/v1", - project: JSON.parse(projectJson), - target: JSON.parse(targetJson), - pages: pages.map(({ startingAfter, responseJson }) => ({ - startingAfter, - response: JSON.parse(responseJson), - })), - }; - const f = await hostedFixture( - provider, - provider === "snyk" && mode === "released assigned JSON row" - ? snykRestFixture("code", { sourceRepositoryUrl: url }) - : report, - url, - ); - // The legacy filename below belongs to the default Cloud deployment. - Reflect.deleteProperty(f.environment, "CODEX_SECURITY_CLOUD_BASE_URL"); - const prior = await f.prepareFile(); - const current = await f.loseResponse(prior); - const dir = dirname(current); - const pending = JSON.parse(await readFile(current, "utf8")); - delete pending.approvedRepositoryUrl; - delete pending.approvedRepositoryPlatform; - const request = pending.requests[0] as FindingImportRequest; - const rows = request.items.map(({ source_finding_id, evidence }, index) => { - if (provider === "semgrep" && mode !== "released assigned JSON row") - evidence.source_data = { ...semgrep, finding: nativeFindings[index] }; - return { source_finding_id, evidence }; - }); - const omittedMetadata = - mode === "released compact issue JSON" || - (provider === "semgrep" && mode === "released without format"); - const metadata = - mode === "SDK metadata and assigned IDs" || omittedMetadata - ? { batch: "example" } - : undefined; - const assigned = - metadata || - mode === "projected metadata" || - mode === "released assigned IDs" || - mode === "released assigned JSON row" || - mode === "released unrelated evidence"; - if (assigned) - rows.forEach((row, index) => { - request.items[index]!.source_finding_id = row.source_finding_id = - "custom-" + index; - }); - const released = mode.startsWith("released "); - const enrichedNative = - mode === "released unmarked with metadata" || - mode === "released relabeled with metadata"; - if (mode === "released compact issue JSON") - for (const { evidence } of rows) - evidence.source_data = { - issueJson: JSON.stringify(evidence.source_data!["issue"]), - }; - if (mode === "released without format") - for (const { evidence } of rows) delete evidence.source_data!["format"]; - if (enrichedNative) - for (const { evidence } of rows) - evidence.source_data = - provider === "snyk" - ? { issue: evidence.source_data!["issue"] } - : { ...evidence.source_data, format: "normalized/v1" }; - if (mode === "released unrelated evidence") - for (const { evidence } of rows) - evidence.source_data = { - project: { label: "example" }, - [provider === "semgrep" ? "finding" : "issue"]: { note: "custom" }, - }; - if ( - mode === "projected evidence" || - mode === "projected metadata" || - mode === "released ID only" - ) - for (const { evidence } of rows) delete evidence.source_data; - const partial = - mode === "released partial object row" || - mode === "released partial JSON row" || - mode === "released partial native project URL"; - if (partial) { - for (const { evidence } of rows) { - delete evidence.details; - const nativeProjectUrl = "git@github.com:example/project.git"; - if (mode === "released partial native project URL") { - evidence.source_data = { - format: "semgrep-platform-findings/v2", - nativeProjectUrl, - semgrepPlatformFindingJson: JSON.stringify({ - ref, - rule_name: "example.rule", - }), - }; - continue; - } - const finding = { ref, repository: { url: nativeProjectUrl } }; - evidence.source_data = { - format: semgrep.format, - ...(mode === "released partial object row" - ? { finding } - : { semgrepPlatformFindingJson: JSON.stringify(finding) }), - }; - } - } - const originalRows = - mode === "per-finding metadata" || - mode === "projected metadata" || - enrichedNative - ? rows.map((row, index) => ({ ...row, ticket: "example-" + index })) - : rows; - const identity = metadata - ? { findings: originalRows, report: metadata } - : originalRows; - const digest = (value: unknown) => hash("sha256", canonicalJson(value)); - pending.inputProjection = { - savedEvidenceDigest: digest(rows), - inputDigest: released - ? digest(identity) - : digest([ - identity, - [url, provider === "semgrep" ? null : false, false], - ]), - }; - const oldKey = digest([ - pending.accountId, - request.repository.id, - request.repository.repo_connector_id, - request.source, - identity, - ]); - const oldPath = join(dir, oldKey + ".pending.json"); - await rm(current); - let bytes = JSON.stringify(pending); - await writeFile(oldPath, bytes, { mode: 0o600 }); - - const input = { - read: rows.length, - excluded: [], - findings: structuredClone(prior.preview.findings), - sourceRepositoryUrl: url, - ...(metadata && { report: metadata }), - }; - if (assigned) - input.findings.forEach((row, index) => { - row.source_finding_id = rows[index]!.source_finding_id; - }); - const prepareRows = (findings: ExternalFindings["findings"]) => - f.prepareNormalized({ ...input, read: findings.length, findings }); - const retry = metadata ? () => f.prepareNormalized(input) : f.prepareFile; - if (mode === "released ID only") { - const assigned = input.findings.map((row, i) => ({ - ...row, - source_finding_id: "ticket-" + i, - })); - await expect(prepareRows(assigned)).rejects.toThrow("pending hosted"); - expect(await readFile(oldPath, "utf8")).toBe(bytes); - expect(f.posts).toHaveLength(1); - return; - } - if (mode === "released assigned IDs" || omittedMetadata) { - // The scanner now generates different IDs for the original native rows. - await expect(f.prepareFile()).rejects.toThrow("pending hosted"); - expect(await readFile(oldPath, "utf8")).toBe(bytes); - expect(f.posts).toHaveLength(1); - return; - } - if (mode === "released unrelated evidence") { - expect((await retry()).preview.resumed).toBe(false); - expect(await readFile(oldPath, "utf8")).toBe(bytes); - return; - } - if (partial) { - const bare = { read: rows.length, excluded: [], findings: rows }; - await expect( - f.prepareNormalized({ - ...bare, - sourceFormat: semgrep.format, - sourceRepositoryUrl: url, - semgrepPlatformScope: { - deployment: semgrep.deployment, - projectId: String(project.id), - product: semgrep.product, - ref: "refs/heads/other", - }, - }), - ).rejects.toThrow(/scope/iu); - f.state.repositoryUrl = "https://github.com/example/renamed"; - await expect( - f.prepareNormalized({ - ...bare, - sourceRepositoryUrl: f.state.repositoryUrl, - }), - ).rejects.toThrow(/repository|project/i); - expect((await f.prepareNormalized(bare)).preview.requests).toEqual( - pending.requests, - ); - f.state.repositoryUrl = url; - expect((await prepareRows(rows)).preview.requests).toEqual( - pending.requests, - ); - return; - } - if (mode === "released normalized SDK") { - expect((await prepareRows(rows)).preview.resumed).toBe(true); - bytes = await readFile(oldPath, "utf8"); - expect(JSON.parse(bytes)).toMatchObject({ - approvedRepositoryUrl: url, - requests: pending.requests, - inputProjection: { - ...pending.inputProjection, - retainedHostedSelection: true, - }, - }); - } - const older = "Resume it with the SDK build that created it"; - if (mode === "projected metadata") { - const edited = structuredClone(originalRows).reverse(); - edited[0]!.evidence.title += " updated"; - await expect(prepareRows(edited)).rejects.toThrow(older); - expect((await prepareRows(rows)).preview.resumed).toBe(false); - expect(await readFile(oldPath, "utf8")).toBe(bytes); - return; - } - await expect(retry()).rejects.toThrow(older); - // The exact projected path has a bare filename but a repository-bound payload digest. - if (mode === "projected evidence") - await expect(prepareRows(rows)).rejects.toThrow(older); - const changed = structuredClone(input.findings).reverse(); - changed[0]!.evidence.title += " changed"; - await expect(prepareRows(changed)).rejects.toThrow(older); - const unrelated = { - source_finding_id: "unrelated", - evidence: { title: "Another finding", severity: "high" as const }, - }; - if (mode === "native") { - const other = "https://github.com/example/another"; - f.state.repositoryUrl = other; - const otherInput = { - ...input, - read: 1, - findings: [unrelated], - sourceRepositoryUrl: other, - }; - const fresh = await prepareNormalizedExternalPublication( - otherInput, - { ...options, provider }, - { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: - "http://127.0.0.1:4200/backend-api/aardvark", - }, - }, - ); - expect(fresh.preview.resumed).toBe(false); - f.state.repositoryUrl = url; - } - if (enrichedNative) { - await expect( - prepareRows([input.findings[0]!, unrelated]), - ).rejects.toThrow(older); - } - const disjoint = prepareRows([unrelated]); - if (released || mode === "per-finding metadata") - expect((await disjoint).preview.resumed).toBe(false); - else await expect(disjoint).rejects.toThrow(older); - expect(f.posts).toHaveLength(1); - expect(await readFile(oldPath, "utf8")).toBe(bytes); - f.state.marker = "generation-2"; - if (mode === "per-finding metadata" || enrichedNative) { - await expect(retry()).rejects.toThrow( - "deployment could not be established", - ); - expect(await readFile(oldPath, "utf8")).toBe(bytes); - } - const afterReset = - mode === "projected evidence" || - mode === "per-finding metadata" || - enrichedNative - ? () => prepareRows(originalRows) - : retry; - await expect(afterReset()).rejects.toThrow("retired without uploading"); - expect( - (await readdir(dir)).filter((name) => name.endsWith(".pending.json")), - ).toEqual([]); - expect((await retry()).preview.resumed).toBe(false); - expect(f.posts).toHaveLength(1); - }, -); - -function sonarCapture(repositoryUrl: string) { - return { - format: "sonar-issues/v1" as const, - server: "https://sonarcloud.io", - organization: "example", - project: "example_project", - branch: "main", - projectBase: "", - repositoryUrl, - pages: [ - { - paging: { pageIndex: 1, pageSize: 100, total: 1 }, - issues: [ - { - key: "issue-1", - project: "example_project", - component: "file", - rule: "example:R1", - message: "Review input handling", - type: "VULNERABILITY", - severity: "CRITICAL", - flows: [ - { - locations: [ - { - component: "file", - textRange: { startLine: 2, endLine: 3 }, - }, - ], - }, - ], - }, - ], - components: [{ key: "file", path: "src/example.ts" }], - }, - ], +test("Wiz network-scan findings are excluded without requiring a detection method on package exports", async () => { + const supported = { + id: "package-occurrence", + name: "CVE-2099-0001", + detailedName: "example-package", + vendorSeverity: "HIGH", + vulnerableAsset: { id: "synthetic-asset" }, }; -} - -test("Sonar replays the selected evidence locally and to Cloud without signing in to Sonar again", async () => { - const repo = "https://github.com/example/project"; - const source = sonarCapture(repo); - const f = await fixture(source, repo); - const checkout = join(f.root, "checkout"); - const git = promisify(execFile); - const environment = sonarGitTestEnvironment({ - ...f.cliDeps.environment, - CODEX_SECURITY_CLOUD_BASE_URL: "https://cloud.example.test", - }); - await git("git", ["init", "--quiet", checkout], { env: environment }); - await git("git", ["-C", checkout, "remote", "add", "origin", repo], { - env: environment, - }); - const paths = new Set(); - const deps = { - ...f.cliDeps, - environment, - currentDirectory: () => checkout, - cloudFetch: (url: string, init: RequestInit) => { - paths.add(new URL(url).pathname); - return f.cliDeps.cloudFetch(url, init); - }, + const network = { + ...supported, + id: "network-occurrence", + name: "CWE-89", + detailedName: "GET /synthetic-endpoint", + detectionMethod: "EXTERNAL_NETWORK_SCAN", }; - const first = createCliTest(main); - const previewExit = await first.runCli( - [ - "import", - "sonar", - "--sonar-project", - source.project, - "--sonar-branch", - source.branch, - "--sonar-issue", - "issue-1", - "--dry-run", - "--format", - "json", - ], - { - ...deps, - collectSonarIssues: async (selection) => { - expect(selection.issueKeys).toEqual(["issue-1"]); - return source; - }, - }, - ); - expect(previewExit, first.stderr.text()).toBe(0); - expect(f.posts).toHaveLength(0); - const capture: string = JSON.parse(first.stdout.text()).capture; - expect(JSON.parse(await readFile(capture, "utf8"))).toEqual(source); - const previewArgs = [ - "import", - "sonar", - capture, - "--to", - "cloud", - "--format", - "json", - ]; - - const outside = join(f.root, "without-remotes"); - await mkdir(outside); - const noRemotes = createCliTest(main); + const f = await fixture([supported, network]); + const prepared = await prepareExternalPublication(f.file, options, f.deps); expect( - await noRemotes.runCli([...previewArgs, "--dry-run"], { - ...deps, - currentDirectory: () => outside, + prepared.preview.findings.map((finding) => finding.source_finding_id), + ).toEqual([supported.id]); + expect(prepared.preview.excluded).toEqual([ + expect.objectContaining({ + source_finding_id: network.id, + reason: expect.stringContaining("external network"), }), - ).toBe(2); - expect(noRemotes.stderr.text()).toContain("--source-repository-url"); - expect(f.posts).toHaveLength(0); + ]); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.reports.has(network.id)).toBe(false); +}); - const cloudPreview = createCliTest(main); - expect(await cloudPreview.runCli([...previewArgs, "--dry-run"], deps)).toBe( - 0, - ); - const reviewed = JSON.parse(cloudPreview.stdout.text()); - expect(reviewed.cloud.destination.url).toBe(repo); - expect(reviewed.capture).not.toBe(capture); - expect(f.posts).toHaveLength(0); - const args = [ - "import", - "sonar", - reviewed.capture, - "--to", - "cloud", - "--format", - "json", - ]; - - const unavailable = createCliTest(main); - expect( - await unavailable.runCli([...args, "--yes"], { - ...deps, - cloudFetch: async () => new Response("Unknown account", { status: 403 }), - }), - ).toBe(2); - expect(JSON.parse(unavailable.stdout.text()).local.counts.created).toBe(1); - - const replay = createCliTest(main); - expect(await replay.runCli([...args, "--yes"], deps)).toBe(0); - const result = JSON.parse(replay.stdout.text()); - expect(result.local.counts.unchanged).toBe(1); - expect(result.cloud.verified).toBe(1); - const posted = JSON.parse(f.posts[0]!); - expect(posted.source).toEqual({ - provider: "sonar", - source_key: result.sourceKey, - }); - expect(posted.items[0].source_finding_id).toBe("issue-1"); - const repository = f.destination(); - const reportsPath = `/external/v2/repository_connectors/${repository.repo_connector_id}/repositories/${repository.id}/source_reports`; - expect(paths).toEqual( - new Set([ - "/external/repositories", - reportsPath, - "/external/v2/finding_imports", - `${reportsPath}/${f.reports.get("issue-1")!.id}`, - ]), - ); - expect(posted.items[0].evidence.source_data.issue.flows).toEqual( - source.pages[0]!.issues[0]!.flows, +test("unsupported records are visible exclusions; duplicate identities and unfinished pages stop publication", async () => { + const f = await fixture([normalized(), normalized("unknown", "unsupported")]); + const parsed = await readVendorFindings(f.file); + expect(parsed.findings).toHaveLength(1); + expect(parsed.excluded).toHaveLength(1); + await writeFile(f.file, JSON.stringify([normalized(), normalized()])); + await expect(readVendorFindings(f.file)).rejects.toThrow("Duplicate source"); + await writeFile( + f.file, + JSON.stringify({ + data: { + vulnerabilityFindings: { + nodes: [normalized()], + pageInfo: { hasNextPage: true }, + }, + }, + }), ); + await expect(readVendorFindings(f.file)).rejects.toThrow("another page"); }); -test("normalized Sonar publication checks retained bindings on preview and retry", async () => { - const f = await fixture(); - const repository = f.state.repositoryUrl; - const foreign = repository + "-other"; - const selected = { ...options, provider: "sonar" as const }; - const context = { format: "sonar-issues/v1", repositoryUrl: repository }; - const finding = { - source_finding_id: "issue-1", - evidence: { - title: "Review input handling", - severity: "high" as const, - source_data: { ...context, issue: { key: "issue-1" } }, - }, - }; - const rows = { read: 1, findings: [finding], excluded: [] }; - const named = { - source_finding_id: "issue-2", - evidence: { - ...finding.evidence, - branch: "main", - source_data: { ...context, branch: "main", issue: { key: "issue-2" } }, - }, - }; - const selection = { read: 2, findings: [finding, named], excluded: [] }; - const prepare = (findings = selection.findings) => - prepareReleased( - f, - { ...selection, findings, report: { ...context, branch: "main" } }, - selected, - ); - await expect(prepareReleased(f, selection, selected)).rejects.toThrow( - "without a ref", - ); - await expect( - prepareReleased( - f, - { - ...rows, - report: { ...context, branch: "main" }, - findings: [ - { - ...finding, - evidence: { ...finding.evidence, branch: "unrelated" }, - }, - ], - }, - selected, - ), - ).rejects.toThrow("retained Sonar branch"); - await expect( - prepareReleased( - f, - { ...rows, report: { ...context, repositoryUrl: foreign } }, - selected, - ), - ).rejects.toThrow("does not match the selected Cloud repository"); - f.state.loseResponse = true; - await expect((await prepare()).publish()).rejects.toThrow( - "resume the saved request", - ); - await expect( - prepareReleased( - f, - { ...selection, report: { ...context, branch: "other" } }, - selected, - ), - ).rejects.toThrow("conflicting branch"); - f.state.repositoryUrl = foreign; - await expect(prepare()).rejects.toThrow( - "does not match the selected Cloud repository", - ); - expect(f.posts).toHaveLength(1); - f.state.repositoryUrl = repository; - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - await retry.publish(); - expect(f.posts).toHaveLength(2); - expect(f.posts[1]).toBe(f.posts[0]); - - // This old boxed update relies on the report for issue-1's branch on every retry. - const { filename, checkpoint } = boxedSonarCheckpoint; - const savedPath = join( - f.root, - "state", - "external-finding-publications", - filename, - ); - await writeFile(savedPath, JSON.stringify(checkpoint), { mode: 0o600 }); - const withEffort = (effort: unknown) => { - const updated = { - ...finding, - evidence: { - ...finding.evidence, - source_data: { ...finding.evidence.source_data, effort }, - }, - }; - return prepare([updated, named]); - }; - f.state.throttle = true; - const boxed = await withEffort(new Number(2.5)); - expect(boxed.preview.resumed).toBe(true); - await expect(boxed.publish()).rejects.toThrow("Slow down"); - f.state.throttle = false; - const primitive = await withEffort(2.5); - expect(primitive.preview.resumed).toBe(true); - expect((await primitive.publish()).verified).toBe(2); - const approved = JSON.stringify(checkpoint.requests[0]); - expect(f.posts.slice(2)).toEqual([approved, approved]); +test("Cloud evidence constraints exclude invalid normalized records before approval", async () => { + let nested: unknown = "leaf"; + for (let index = 0; index < 20; index++) nested = { child: nested }; + const invalid = [ + { title: "é".repeat(300) }, + { description: "" }, + { packages: [{ name: "example", installed_version: "1.0\0" }] }, + { advisory_ids: ["é".repeat(300)] }, + { locations: [{ path: "../outside.ts" }] }, + { url: "https://user@example.test/finding" }, + { source_data: { content: "x".repeat(6 * 1024 * 1024) } }, + { source_data: { nested } }, + { url: "https://[example.test" }, + ].map((evidence, index) => ({ + source_finding_id: `invalid-${index}`, + evidence: { ...normalized().evidence, ...evidence }, + })); + const f = await fixture([normalized(), ...invalid]); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + const preview = JSON.parse(cli.stdout.text()); + expect(preview.findings).toHaveLength(1); + expect(preview.excluded).toHaveLength(invalid.length); + expect(preview.excluded[6].reason).toContain("256 KiB"); + expect(f.posts).toHaveLength(0); + await writeFile(f.file, JSON.stringify(invalid)); + const rejected = createCliTest(main); + expect(await rejected.runCli([...f.command, "--yes"], f.cliDeps)).toBe(2); + expect(rejected.stderr.text()).toContain("No supported findings"); + expect(f.posts).toHaveLength(0); + expect(await readdir(join(f.root, "state")).catch(() => [])).toEqual([]); }); -test("Wiz occurrence mapping preserves original evidence without inventing repository provenance", async () => { - const record = { - id: "wiz-occurrence", +test("raw Wiz records apply Cloud string and URL constraints before approval", async () => { + const wiz = { + id: "wiz-valid", name: "CVE-2099-0001", - detailedName: "example-lib", - version: "1.2.0", - fixedVersion: "1.2.1", + detailedName: "example-package", vendorSeverity: "HIGH", - portalUrl: "https://example.test/finding", - locationPath: "/usr/lib/example", - lastDetectedAt: "2026-10-01T00:00:00Z", - projects: [{ id: "project-a" }, { id: "project-b" }], - packageManager: "legacy-manager", - vulnerableAsset: { id: "image-resource", imageDigest: "sha256:example" }, + vulnerableAsset: { id: "example-image" }, }; - const f = await fixture([record]); - const parsed = await readVendorFindings(f.file); - expect(parsed.findings[0]).toMatchObject({ - source_finding_id: record.id, - evidence: { - severity: "high", - locations: [], - code_revision: null, - branch: null, - source_updated_at: null, - source_data: record, - image_digests: ["sha256:example"], - advisory_ids: [record.name], - packages: [{ name: "example-lib", ecosystem: "legacy-manager" }], - }, - }); + const f = await fixture([ + wiz, + { ...wiz, id: "wiz-title", title: "é".repeat(300) }, + { ...wiz, id: "é".repeat(300) }, + { ...wiz, id: "wiz\0id" }, + { ...wiz, id: "wiz-url", portalUrl: "https://@example.test" }, + ]); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + const preview = JSON.parse(cli.stdout.text()); + expect(preview.findings).toHaveLength(1); + expect(preview.excluded).toHaveLength(4); + expect(f.posts).toHaveLength(0); }); -test("Wiz API nodes preserve artifact ecosystems and distinguish container digests from VM image IDs", async () => { - const finding = { - name: "Vendor display advisory", - vulnerabilityExternalId: "CVE-2099-0001", - detailedName: "example-package", - version: "1.2.0", - vendorSeverity: "HIGH", - }; - const records = [ - { - ...finding, - id: "container-package", - artifactType: { osPackageManager: "DPKG", codeLibraryLanguage: null }, - codeLibraryLanguage: "JAVA", - vulnerableAsset: { - id: "container-image", - type: "CONTAINER_IMAGE", - imageId: `sha256:${"a".repeat(64)}`, - }, - }, - { - ...finding, - id: "vm-library", - artifactType: { osPackageManager: null, codeLibraryLanguage: "PYTHON" }, - codeLibraryLanguage: "JAVA", - vulnerableAsset: { - id: "virtual-machine", - type: "VIRTUAL_MACHINE", - imageId: "ami-synthetic", - containerImageId: "ami-synthetic-alias", - }, - }, - { - ...finding, - id: "container-alias", - vulnerableAsset: { - id: "aliased-image", - type: "CONTAINER_IMAGE", - containerImageId: `sha256:${"d".repeat(64)}`, - }, - }, - { - ...finding, - id: "running-container", - vulnerableAsset: { - id: "running-image", - type: "CONTAINER", - ImageExternalId: `sha256:${"e".repeat(64)}`, - }, - }, - { - ...finding, - id: "explicit-metadata", - packageManager: "reported-manager", - imageDigest: `sha256:${"b".repeat(64)}`, - artifactType: { osPackageManager: "DPKG" }, - codeLibraryLanguage: "JAVA", - vulnerableAsset: { - id: "explicit-container", - type: "CONTAINER_IMAGE", - imageId: `sha256:${"c".repeat(64)}`, - }, - }, - { - ...finding, - id: "legacy-library", - codeLibraryLanguage: "JAVASCRIPT", - vulnerableAsset: { id: "legacy-image" }, - }, - ]; - const f = await fixture({ - data: { - vulnerabilityFindings: { - nodes: records, - pageInfo: { hasNextPage: false }, - }, - }, - }); - const parsed = await readVendorFindings(f.file); - expect(parsed.excluded).toEqual([]); - expect(parsed.findings[0]!.evidence.title).toContain(finding.name); - expect(parsed.findings.map(({ evidence }) => evidence.advisory_ids)).toEqual( - records.map(() => [finding.vulnerabilityExternalId]), - ); - expect( - parsed.findings.map(({ evidence }) => ({ - ecosystem: evidence.packages?.[0]?.ecosystem, - digests: evidence.image_digests, - })), - ).toEqual([ - { ecosystem: "DPKG", digests: [`sha256:${"a".repeat(64)}`] }, - { ecosystem: "PYTHON", digests: [] }, - { ecosystem: null, digests: [`sha256:${"d".repeat(64)}`] }, - { ecosystem: null, digests: [`sha256:${"e".repeat(64)}`] }, - { ecosystem: "reported-manager", digests: [`sha256:${"b".repeat(64)}`] }, - { ecosystem: "JAVASCRIPT", digests: [] }, - ]); - expect(parsed.findings.map(({ evidence }) => evidence.source_data)).toEqual( - records, - ); -}); - -test("Wiz network-scan findings are excluded without requiring a detection method on package exports", async () => { - const supported = { - id: "package-occurrence", - name: "CVE-2099-0001", - detailedName: "example-package", - vendorSeverity: "HIGH", - vulnerableAsset: { id: "synthetic-asset" }, - }; - const network = { - ...supported, - id: "network-occurrence", - name: "CWE-89", - detailedName: "GET /synthetic-endpoint", - detectionMethod: "EXTERNAL_NETWORK_SCAN", - }; - const f = await fixture([supported, network]); - const prepared = await prepareExternalPublication(f.file, options, f.deps); - expect( - prepared.preview.findings.map((finding) => finding.source_finding_id), - ).toEqual([supported.id]); - expect(prepared.preview.excluded).toEqual([ - expect.objectContaining({ - source_finding_id: network.id, - reason: expect.stringContaining("external network"), - }), - ]); - expect((await prepared.publish()).counts.created).toBe(1); - expect(f.reports.has(network.id)).toBe(false); -}); - -test("unsupported records are visible exclusions; duplicate identities and unfinished pages stop publication", async () => { - const f = await fixture([normalized(), normalized("unknown", "unsupported")]); - const parsed = await readVendorFindings(f.file); - expect(parsed.findings).toHaveLength(1); - expect(parsed.excluded).toHaveLength(1); - await writeFile(f.file, JSON.stringify([normalized(), normalized()])); - await expect(readVendorFindings(f.file)).rejects.toThrow("Duplicate source"); - await writeFile( - f.file, - JSON.stringify({ - data: { - vulnerabilityFindings: { - nodes: [normalized()], - pageInfo: { hasNextPage: true }, - }, - }, - }), - ); - await expect(readVendorFindings(f.file)).rejects.toThrow("another page"); -}); - -test("Cloud evidence constraints exclude invalid normalized records before approval", async () => { - let nested: unknown = "leaf"; - for (let index = 0; index < 20; index++) nested = { child: nested }; - const invalid = [ - { title: "é".repeat(300) }, - { description: "" }, - { packages: [{ name: "example", installed_version: "1.0\0" }] }, - { advisory_ids: ["é".repeat(300)] }, - { locations: [{ path: "../outside.ts" }] }, - { url: "https://user@example.test/finding" }, - { source_data: { content: "x".repeat(6 * 1024 * 1024) } }, - { source_data: { nested } }, - { url: "https://[example.test" }, - ].map((evidence, index) => ({ - source_finding_id: `invalid-${index}`, - evidence: { ...normalized().evidence, ...evidence }, - })); - const f = await fixture([normalized(), ...invalid]); - const cli = createCliTest(main); - expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); - const preview = JSON.parse(cli.stdout.text()); - expect(preview.findings).toHaveLength(1); - expect(preview.excluded).toHaveLength(invalid.length); - expect(preview.excluded[6].reason).toContain("256 KiB"); - expect(f.posts).toHaveLength(0); - await writeFile(f.file, JSON.stringify(invalid)); - const rejected = createCliTest(main); - expect(await rejected.runCli([...f.command, "--yes"], f.cliDeps)).toBe(2); - expect(rejected.stderr.text()).toContain("No supported findings"); - expect(f.posts).toHaveLength(0); - expect(await readdir(join(f.root, "state")).catch(() => [])).toEqual([]); -}); - -test("raw Wiz records apply Cloud string and URL constraints before approval", async () => { - const wiz = { - id: "wiz-valid", - name: "CVE-2099-0001", - detailedName: "example-package", - vendorSeverity: "HIGH", - vulnerableAsset: { id: "example-image" }, - }; - const f = await fixture([ - wiz, - { ...wiz, id: "wiz-title", title: "é".repeat(300) }, - { ...wiz, id: "é".repeat(300) }, - { ...wiz, id: "wiz\0id" }, - { ...wiz, id: "wiz-url", portalUrl: "https://@example.test" }, - ]); - const cli = createCliTest(main); - expect(await cli.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); - const preview = JSON.parse(cli.stdout.text()); - expect(preview.findings).toHaveLength(1); - expect(preview.excluded).toHaveLength(4); - expect(f.posts).toHaveLength(0); -}); - -test("Cloud validation preserves allowed evidence boundaries and raw vendor strings", () => { - let nested: unknown = "leaf"; - for (let index = 0; index < 18; index++) nested = { child: nested }; - const evidence = { - title: "é".repeat(256), - severity: "high" as const, - description: "é".repeat(32768), - url: "https://example.test:non-numeric/finding", - advisory_ids: [" ", "\0", "é".repeat(256)], - packages: [{ name: "example", manifest_path: "../vendor-manifest" }], - locations: [{ path: "src\\example.ts" }], - source_data: { nested, title: "", path: "\0", content: "x".repeat(4096) }, +test("Cloud validation preserves allowed evidence boundaries and raw vendor strings", () => { + let nested: unknown = "leaf"; + for (let index = 0; index < 18; index++) nested = { child: nested }; + const evidence = { + title: "é".repeat(256), + severity: "high" as const, + description: "é".repeat(32768), + url: "https://example.test:non-numeric/finding", + advisory_ids: [" ", "\0", "é".repeat(256)], + packages: [{ name: "example", manifest_path: "../vendor-manifest" }], + locations: [{ path: "src\\example.ts" }], + source_data: { nested, title: "", path: "\0", content: "x".repeat(4096) }, }; expect(validateExternalEvidence(evidence)).toEqual(evidence); for (const url of [ @@ -4655,70 +3221,6 @@ test("Continuum Cloud publication checks repository and provider and replays an expect(f.posts).toEqual([f.posts[0]!, f.posts[0]!]); }); -test.each(["single detail", "selected array"])( - "a Continuum %s reuses its interrupted 0.9.x request after CWE enrichment", - async (shape) => { - const { cweIds: _, ...detail } = continuumDetail(); - const native = { - ...detail, - source: "CONTINUUM", - type: "CONTINUUM_CODE_VULN", - vulnerabilities: [], - subCategory: "CWE-89: SQL Injection", - }; - const f = await fixture( - shape === "single detail" ? native : [native], - continuumRepository, - ); - const settings = { - ...options, - provider: "continuum" as const, - sourceRepositoryUrl: continuumRepository, - }; - // 0.9.x retained the status timestamp but did not infer a structured CWE. - const released = normalizePreviousContinuumFindings( - native, - continuumRepository, - ); - released.findings[0]!.evidence.source_updated_at = - Date.parse(native.statusUpdatedAt) / 1000; - expect(hash("sha256", canonicalJson(released.findings))).toBe( - "2d1bca6f9e46758b8b7d9d679e5e7912a9dd8f99ca4c3cb6889b40eaa4b2506e", - ); - const original = await prepareNormalizedExternalPublication( - released, - settings, - f.deps, - ); - const pending = await f.loseResponse(original); - - const prepare = () => prepareExternalPublication(f.file, settings, f.deps); - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests).toEqual(original.preview.requests); - expect(await f.loseResponse(retry)).toBe(pending); - - const sdk = await readContinuumFindings(f.file, continuumRepository); - const prepareSdk = (input: ExternalFindings) => - prepareNormalizedExternalPublication(input, settings, f.deps); - const edited = structuredClone(sdk); - edited.findings[0]!.evidence.details!.weakness_ids = ["CWE-20"]; - expect((await prepareSdk(edited)).preview.resumed).toBe(false); - - const sdkRetry = await prepareSdk(sdk); - expect(sdkRetry.preview.requests).toEqual(original.preview.requests); - expect(await f.loseResponse(sdkRetry)).toBe(pending); - const restored = await prepareSdk(JSON.parse(JSON.stringify(sdk))); - expect(restored.preview.resumed).toBe(true); - expect((await restored.publish()).verified).toBe(1); - expect(f.posts).toEqual(Array(4).fill(f.posts[0])); - expect(await Bun.file(pending).exists()).toBe(false); - expect( - (await prepare()).preview.findings[0]!.evidence.details?.weakness_ids, - ).toEqual(["CWE-89"]); - }, -); - test("Inspector publication checks its source and reuses the finding ARN", async () => { const { f, input, prepare } = await inspectorPublicationFixture(); const prepared = await prepare(); @@ -4785,11 +3287,6 @@ test.each(["Inspector", "Trivy"] as const)( if (!(error instanceof ExternalPublicationError)) throw error; expect(f.reports.get(record.id)?.evidence.source_data).toEqual(record); - // Older checkpoints can contain the original requests without their input projection. - const path = error.result.savedSubmission!; - const saved = JSON.parse(await readFile(path, "utf8")); - delete saved.inputProjection; - await writeFile(path, JSON.stringify(saved)); const retry = await prepare(); expect(retry.preview.resumed).toBe(true); expect((await retry.publish()).verified).toBe(1); @@ -4921,149 +3418,7 @@ test.each(["inspector", "sonar"] as const)( }, ); -function inspectorReportWithRelatedCves() { - const pkg = inspectorPackageFinding(); - // The previous reader excluded this row because it projected 101 advisory IDs. - pkg.packageVulnerabilityDetails.relatedVulnerabilities = Array.from( - { length: 100 }, - (_, i) => `CVE-2099-${3000 + i}`, - ); - return { findings: [inspectorCodeFinding(), pkg] }; -} - const exampleRepository = "https://github.com/example/project"; -const githubOrigin = { - ...blackDuckBomRow(), - componentVersionOriginName: "github", - componentVersionOriginId: "example/library/v1", - packageUrl: "pkg:github/Example/Library@v1", -}; -const nativeSemgrep = semgrepJson(); - -const awsReaderUpgrades = [ - { - provider: "inspector", - raw: inspectorReportWithRelatedCves(), - normalizePrevious: normalizePreviousInspectorFindings, - sourceRepositoryUrl: inspectorRepositoryUrl, - previousDigest: - "4e90748318e5952a1fa90ff9c25153396340a097db6112f27bdf89a3a092dade", - expectedCount: 2, - previousSha256: - "91b5a4d309daee520029a3d54e809ad84d8b47ee97df5fd54d56323fc246f640", - }, - { - provider: "continuum", - raw: continuumDetail(), - normalizePrevious: normalizePreviousContinuumFindings, - sourceRepositoryUrl: continuumRepository, - previousDigest: - "22307dedb51cefecf56d11fd68f45324dceb5226bfddfbd48c635a462bb68fb5", - expectedCount: 1, - previousSha256: - "98f888f8b748bb54a209b19de53af40867f5f129049a99da0387e697895aa320", - }, -] as const; - -test.each([ - ...awsReaderUpgrades, - { - provider: "blackduck", - raw: blackDuckBomReport([githubOrigin]), - normalizePrevious: normalizePreviousBlackDuckFindings, - sourceRepositoryUrl: exampleRepository, - previousDigest: - "ade8411242d3cac1fb490a86159d1b4a3b57a7d7f18bee64cadc971e3433185e", - expectedCount: 1, - }, - { - provider: "veracode", - raw: veracodeMixedPolicyPage(), - normalizePrevious: normalizePreviousVeracodeFindings, - sourceRepositoryUrl: exampleRepository, - previousDigest: - "0ca5d464a797a1a17e18a69f9b4cad172a458be2955c8a24048df78e2798b8b6", - expectedCount: 2, - }, - { - provider: "semgrep", - raw: { - ...nativeSemgrep, - explanations: [ - { - op: "And", - matches: [{ ...nativeSemgrep.results[0]!, path: "src/other.ts" }], - children: [], - }, - ], - }, - normalizePrevious: (raw: unknown, sourceRepositoryUrl: string) => - normalizePreviousSemgrepFindings(raw, { sourceRepositoryUrl }), - sourceRepositoryUrl: exampleRepository, - previousDigest: - "2265f4b64025fe061d950e8b20eba222563001a781b645846646000fbbcdc4a0", - expectedCount: 1, - }, - { - provider: "trivy", - raw: trivyReportWithManyLocations(), - normalizePrevious: (raw: unknown) => - normalizePreviousTrivyFindings(raw, "cloud", "."), - sourceRepositoryUrl: exampleRepository, - previousDigest: - "b4158738af7b92798b6f2b77d8f07343a9422cfb0ffafe4db6410a2bfe7d3b1e", - expectedCount: 2, - }, -] as const)( - "a raw $provider retry preserves the repository request from the previous reader", - async ({ - provider, - raw, - normalizePrevious, - sourceRepositoryUrl, - previousDigest, - expectedCount, - }) => { - const f = await fixture(raw, sourceRepositoryUrl); - const settings = { - ...options, - provider, - ...(provider === "trivy" - ? { trivyScanRoot: "." } - : { sourceRepositoryUrl }), - }; - const prepare = () => prepareExternalPublication(f.file, settings, f.deps); - - // Round-trip the previous SDK's output through JSON as a saved caller artifact. - const previous = JSON.parse( - JSON.stringify(normalizePrevious(raw, sourceRepositoryUrl)), - ); - expect(hash("sha256", canonicalJson(previous.findings))).toBe( - previousDigest, - ); - const original = await prepareNormalizedExternalPublication( - previous, - settings, - f.deps, - ); - const pending = await f.loseResponse(original); - const request = f.posts[0]!; - - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests).toEqual(original.preview.requests); - expect(await f.loseResponse(retry)).toBe(pending); - expect((await (await prepare()).publish()).verified).toBe(1); - expect(f.posts).toEqual([request, request, request]); - - const fresh = await prepare(); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.findings).toHaveLength(expectedCount); - expect(fresh.preview.findings).not.toEqual(original.preview.findings); - expect((await fresh.publish()).verified).toBe(expectedCount); - }, -); - test("publishes a filtered numeric Veracode CWE that the previous reader rejected", async () => { const raw = veracodeMixedPolicyPage(); raw._links.self.href += "&cwe=79"; @@ -5073,279 +3428,16 @@ test("publishes a filtered numeric Veracode CWE that the previous reader rejecte provider: "veracode" as const, sourceRepositoryUrl: exampleRepository, }; - expect(() => - normalizePreviousVeracodeFindings(raw, exampleRepository), - ).toThrow("Veracode cwe must be an object"); const prepared = await prepareExternalPublication(f.file, settings, f.deps); expect((await prepared.publish()).verified).toBe(2); }); -test("a fresh AWS publication waits for the previous reader to save its request", async () => { - const raw = continuumDetail(); - const f = await fixture(raw, continuumRepository); - const settings = { - ...options, - provider: "continuum" as const, - sourceRepositoryUrl: continuumRepository, - }; - const previous = await prepareNormalizedExternalPublication( - normalizePreviousContinuumFindings(raw, continuumRepository), - settings, - f.deps, - ); - const fresh = await prepareExternalPublication(f.file, settings, f.deps); - const observer = observeLockContention(); - const rename = fs.rename; - let competitor: Promise | undefined; - let outcome: string | undefined; - const install = spyOn(fs, "rename").mockImplementationOnce( - async (...args) => { - // The earlier publisher holds its lock but has not installed its checkpoint yet. - competitor = fresh.publish().catch((error: unknown) => error); - outcome = await Promise.race([ - observer.contended.then(() => "locked"), - competitor.then(() => "finished"), - ]); - return rename(...args); - }, - ); - try { - f.state.throttle = true; - await expect(previous.publish()).rejects.toThrow("Slow down"); - expect(outcome).toBe("locked"); - expect(String(await competitor)).toContain( - "Another publication prepared this input", - ); - expect(f.posts).toHaveLength(1); - } finally { - install.mockRestore(); - observer.restore(); - } -}); - -const savedBlackDuckEvidence = { - data: { format: "blackduck-bom-6" }, - requiredProvider: "blackduck", - digest: "d3001de83673b0f42b1d1d42cb19ea546869d41ed99504ca213b303267bb1346", -}; - -const savedDependabotData = { - ...githubDependabotAlert(), - sourceRepositoryUrl: "https://github.com/example/different", -}; - -const checkmarxFormat = { format: "cxsast-xml" }; const otherRepository = "https://github.com/example/other"; -const savedEndorData = (repositoryUri: string) => ({ - ...checkmarxFormat, - runContext: { versionControlProvenance: ["opaque", { repositoryUri }] }, -}); -const endorCheckpoint = { - requiredProvider: "checkmarx", - selected: { ...options, provider: "endor" as const }, -}; - const wizBranchAsset = { nativeType: "github#repositoryBranch", cloudProviderURL: exampleRepository + "/tree/main", }; -test.each([ - { - data: { ...checkmarxFormat, vulnerableAsset: wizBranchAsset }, - selected: options, - requiredProvider: "checkmarx", - filename: - "44c97129ef37ba88ccf7d6a1c43c3ef4c051080c0e746145b1f37dc6216f0181.pending.json", - digest: "7c90e7bf4a7d9c64c7cbbc15fdcdf6a3ff0e14f50cda77c9b523d834cf337dd7", - }, - { - ...endorCheckpoint, - data: savedEndorData(exampleRepository), - filename: - "4d59df5533b5d0acbbcc2e28884e597a52d8b55e354eb15539d6cfba37c79011.pending.json", - digest: "d19b090ff4cd18329b37e74e674394deb2670c1d0dfc7bc088c02edcbb088cad", - }, - { - ...endorCheckpoint, - data: { - ...savedEndorData(exampleRepository), - sourceRepositoryUrl: exampleRepository, - }, - filename: - "624c404a8fcf0e8241c30d9b1a434d78aaa2c1e87d211465e0698821ab1c509f.pending.json", - digest: "f77bce9f03733812b5fda88ae988193b54462baf78be95ba6b4de49658b103d9", - }, - { - ...endorCheckpoint, - data: { - ...savedEndorData(otherRepository), - sourceRepositoryUrl: exampleRepository, - }, - conflictingUrl: otherRepository, - filename: - "70e7f4be8d0d9eb0306f18ea1a2b256641bee797b3626a8dfc2920d0a15abcd2.pending.json", - digest: "8fedf62999d4c5f510706db4647ad3d847e8e5f6f5f3d75eb365da193c16ea1a", - }, - { - data: { format: "snyk-rest-issues/v2", issueJson: "{}" }, - requiredProvider: "snyk", - digest: "e5b609dd6f1d5846f1b84457c319fb02505e55385ed2a4823f2a359fef7e3f33", - selected: { ...options, provider: "snyk" as const }, - filename: - "d7c57e64c210ec93849be17563ba9ae4e9abbd5bd614fbc99a27346e72ce80ed.pending.json", - }, - { - data: { format: "semgrep-platform-findings/v1" }, - requiredProvider: "semgrep", - digest: "03781aba18f0321eef4e0aeccb512116387931f166fa42c148587085933cd48b", - selected: { ...options, provider: "veracode" as const }, - filename: - "b1fda8b80a88dc44670dcd566e5b4e13f8a3cc1ddd776973a047ffe45e998987.pending.json", - }, - { - ...savedBlackDuckEvidence, - selected: options, - filename: - "2dfdb44c2c7494f4eed20e59244df967598fc3a284589ec2f6c3a24bfff7b2d0.pending.json", - }, - { - ...savedBlackDuckEvidence, - selected: { ...options, provider: "inspector" as const }, - filename: - "d0ba21fbe6c5927206db937694c98f69ad59e33fc390cae6eb6bd01249b5ca10.pending.json", - }, - { - selected: { ...options, provider: "github" as const }, - data: savedDependabotData, - requiredProvider: "blackduck", - filename: - "2dc327e13b7b9548e828b0fe3193988a26c53c9c956b364c69990abdb418c298.pending.json", - digest: "abe417c10599f27dc9adc599465af29f693069deaca7942f79c952871216070e", - }, - { - selected: { ...options, provider: "inspector" as const }, - data: { - sourceRepositoryUrl: inspectorRepositoryUrl, - applicationUrl: veracodePolicyPage()._links.application.href, - row: veracodePolicyRow(), - }, - requiredProvider: "veracode", - filename: - "7d21ed6ce9acbc4bf00c5c7d7ed02b8e7937e92efa796e6218d57273eb9ce2f4.pending.json", - digest: "13b472f7b050ef94705eed1fb4487b206386130f988d3ee6f87824f02f4d79f0", - }, -])( - "checks a saved $selected.provider request from before $requiredProvider checks", - async (saved) => { - const { selected, data, requiredProvider, filename, digest } = saved; - const f = await fixture(); - const environment: NodeJS.ProcessEnv = { ...f.environment }; - // These persisted keys came from the default deployment. - delete environment["CODEX_SECURITY_CLOUD_BASE_URL"]; - delete environment["CODEX_SECURITY_CLOUD_WEB_URL"]; - const alert = - selected.provider === "github" ? savedDependabotData : undefined; - const row = normalized(); - const finding = { - source_finding_id: alert - ? "blackduck-bom:" + "a".repeat(64) - : row.source_finding_id, - evidence: validateExternalEvidence( - alert - ? { - title: "Dependabot alert", - severity: "high", - url: alert.html_url, - source_data: data, - } - : { ...row.evidence, source_data: data }, - ), - }; - const input = { read: 1, excluded: [], findings: [finding] }; - const prepare = (sourceRepositoryUrl?: string) => - prepareNormalizedExternalPublication( - { ...input, ...(sourceRepositoryUrl && { sourceRepositoryUrl }) }, - selected, - { - ...f.deps, - environment, - }, - ); - - // The previous publisher accepted this opaque evidence. Keep its checkpoint - // filename and digests so this exercises recovery across the provider change. - const directory = join( - f.environment.CODEX_SECURITY_STATE_DIR, - "external-finding-publications", - ); - const request = { - request_id: "efbc3c2e-0462-4411-89aa-9ced1d34223c", - repository: { - id: selected.repository, - repo_connector_id: "github", - environment_id: f.state.environmentId, - reset_marker: f.state.marker, - }, - source: { provider: selected.provider, source_key: selected.sourceKey }, - items: [{ client_id: "item-1", ...finding, expected_version: 0 }], - }; - await mkdir(directory, { recursive: true, mode: 0o700 }); - await writeFile( - join(directory, filename), - JSON.stringify({ - accountId: "synthetic-account", - requests: [request], - inputProjection: { inputDigest: digest, savedEvidenceDigest: digest }, - }), - { mode: 0o600 }, - ); - - if ("conflictingUrl" in saved && saved.conflictingUrl) { - await expect(prepare()).rejects.toThrow( - `belongs to "${saved.conflictingUrl}"`, - ); - expect(f.posts).toHaveLength(0); - return; - } - if (alert) { - // The original publisher selected the alert's repository. Changing the - // catalog to match the projection must not permit replay to another repo. - const originalRepository = f.state.repositoryUrl; - f.state.repositoryUrl = alert.sourceRepositoryUrl; - await expect(prepare()).rejects.toThrow( - `belongs to "${originalRepository}"`, - ); - expect(f.posts).toHaveLength(0); - return; - } - if (requiredProvider === "checkmarx") { - const original = f.state.repositoryUrl; - f.state.repositoryUrl += "-other"; - await expect(prepare()).rejects.toThrow(`belongs to "${original}"`); - expect(f.posts).toHaveLength(0); - f.state.repositoryUrl = original; - } - if (selected.provider === "snyk") { - const other = "https://github.com/example/other"; - await expect(prepare(other)).rejects.toThrow(/does not match/); - expect((await prepare(f.state.repositoryUrl)).preview.resumed).toBe(true); - expect((await prepare(f.state.repositoryUrl)).preview.resumed).toBe(true); - await expect(prepare(other)).rejects.toThrow(/does not match/); - } - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).counts.created).toBe(1); - expect(f.posts.map((body) => JSON.parse(body))).toEqual([request]); - if (selected.provider === requiredProvider) - await expect(prepare()).rejects.toThrow(/Snyk/i); - else - await expect(prepare()).rejects.toThrow( - `requires the ${requiredProvider} provider`, - ); - }, -); - test.each(["wiz", "inspector"] as const)( "rechecks a projected Black Duck repository on a saved %s publication", async (provider) => { @@ -6559,68 +4651,6 @@ test.each(["request", "counts", "length", "schema"])( }, ); -test("an opaque retry preserves a released partial submission whose final row exceeds the corrected limit", async () => { - const f = await fixture(); - const rows = Array.from({ length: 101 }, (_, i) => { - const row = normalized(`item-${i}`); - return { ...row, evidence: { ...row.evidence, severity: "high" as const } }; - }); - const pad = "x".repeat(261872); - Object.assign(rows[100]!.evidence, { - source_data: { - get score() { - return Object.assign(new Number(1e-7), { toJSON: () => 0 }); - }, - pad, - }, - }); - const prepare = (selected = rows, sourceKey = options.sourceKey) => - prepareReleased( - f, - { read: selected.length, findings: selected, excluded: [] }, - { ...options, sourceKey }, - ); - const ready = await prepare(); - const [first, last] = ready.preview.requests; - const [acknowledged] = (await (await prepare(rows.slice(0, 100))).publish()) - .receipts; - - // Rebuild the released SDK's checkpoint, including its first 100 acknowledged items. - first!.request_id = "353fb2dc-0e4a-4be5-b825-5755e4678aa5"; - last!.request_id = "66da2566-e3c5-4557-aef2-10ffd179982d"; - last!.items[0]!.evidence.source_data = { - score: 1e-7, - pad, - }; - const digest = - "ae9fd2d8c0f71c489d99604663201894d8d2fe91550bb949035177c8bd0595ef"; - const saved = JSON.stringify({ - accountId: "synthetic-account", - requests: [first, last], - receipts: [{ ...acknowledged, id: first!.request_id }], - inputProjection: { inputDigest: digest, savedEvidenceDigest: digest }, - }); - expect(hash("sha256", saved)).toBe( - "4ca49c21887a29fdb56ea1bcb8197ef5fed02c68f7aacfaa0cc06d9c9b5a128a", - ); - const path = join( - f.root, - "state", - "external-finding-publications", - "b8d2c1dda3201683c738fb810639279ba8e30a5dfd1402288a5d7017f7b572e8.pending.json", - ); - await writeFile(path, saved, { mode: 0o600 }); - - await expect(prepare()).rejects.toThrow("checkpoint was preserved"); - await expect(ready.publish()).rejects.toThrow("checkpoint was preserved"); - f.reports.clear(); // The separate source has no prior findings. - expect((await prepare(rows, "tenant-example/other")).preview.resumed).toBe( - false, - ); - expect(f.posts).toHaveLength(1); - expect(await readFile(path, "utf8")).toBe(saved); -}); - test("a held retry reports the acknowledged batch after another resumer retires the checkpoint", async () => { const f = await fixture( Array.from({ length: 101 }, (_, index) => normalized(`item-${index}`)), @@ -6916,2416 +4946,1276 @@ test.each([ }; const f = await fixture({ data: { - vulnerabilityFindings: { - nodes: [record], - pageInfo: { hasNextPage: false }, - }, - }, - }); - const parsed = await readVendorFindings(f.file); - expect(parsed.excluded).toEqual([]); - expect(parsed.findings[0]).toMatchObject({ - source_finding_id: record.id, - evidence: { - packages: [ - { name: "example-lib", ecosystem, installed_version: "1.0.0" }, - ], - image_digests: [digest], - source_data: record, - }, - }); - }, -); - -test.each([false, true])( - "receipt client IDs correlate reordered results (cached: %p)", - async (cached) => { - const f = await fixture([ - normalized("vendor-first", "high"), - normalized("vendor-second", "low"), - ]); - const deps = { - ...f.deps, - fetch: async (url: string, init: RequestInit) => { - const response = await f.deps.fetch(url, init); - if (init.method !== "POST" || !response.ok) return response; - const receipt = (await response.json()) as FindingImportReceipt; - receipt.results.reverse(); - return new Response(JSON.stringify(receipt), { - headers: { "Content-Type": "application/json" }, - }); - }, - }; - let prepared = await prepareExternalPublication(f.file, options, deps); - if (cached) { - f.state.brokenReadback = true; - await expect(prepared.publish()).rejects.toThrow("Readback unavailable"); - f.state.brokenReadback = false; - f.state.postBudget = 0; - prepared = await prepareExternalPublication(f.file, options, deps); - expect(prepared.preview.resumed).toBe(true); - } - const result = await prepared.publish(); - expect(result.counts.created).toBe(2); - expect(result.receipts[0]!.results.map((item) => item.client_id)).toEqual([ - "item-2", - "item-1", - ]); - expect(f.posts).toHaveLength(1); - expect(f.receipts.size).toBe(1); - }, -); - -test.each(["duplicate", "missing", "unexpected"])( - "receipt client IDs reject %s results before checkpointing", - async (invalid) => { - const f = await fixture([ - normalized("vendor-first"), - normalized("vendor-second"), - ]); - const deps = { - ...f.deps, - fetch: async (url: string, init: RequestInit) => { - const response = await f.deps.fetch(url, init); - if (init.method !== "POST" || !response.ok) return response; - const receipt = (await response.json()) as FindingImportReceipt; - if (invalid === "duplicate") - receipt.results[1]!.client_id = receipt.results[0]!.client_id; - if (invalid === "missing") receipt.results.pop(); - if (invalid === "unexpected") - receipt.results[0]!.client_id = "unexpected-client"; - return new Response(JSON.stringify(receipt), { - headers: { "Content-Type": "application/json" }, - }); - }, - }; - await expect( - (await prepareExternalPublication(f.file, options, deps)).publish(), - ).rejects.toThrow("Cloud returned a receipt for a different publication"); - const directory = join(f.root, "state", "external-finding-publications"); - const pending = (await readdir(directory)).find((name) => - name.endsWith(".pending.json"), - )!; - expect( - JSON.parse(await readFile(join(directory, pending), "utf8")).receipts ?? - [], - ).toEqual([]); - expect(f.posts).toHaveLength(1); - }, -); - -test.each(["replacement-environment", null])( - "saved publication preserves its environment when discovery changes to %p", - async (environmentId) => { - const f = await fixture(); - f.state.loseResponse = true; - await expect( - (await prepareExternalPublication(f.file, options, f.deps)).publish(), - ).rejects.toThrow("resume the saved request"); - const original = f.posts[0]!; - f.state.environmentId = environmentId; - const retry = await prepareExternalPublication(f.file, options, f.deps); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests[0]!.repository.environment_id).toBe( - "environment-example", - ); - expect((await retry.publish()).counts.created).toBe(1); - expect(f.posts.at(-1)).toBe(original); - }, -); - -test("mixed environments preserve input order and are shown before confirmation", async () => { - const f = await fixture([normalized("existing-a"), normalized("existing-c")]); - await (await prepareExternalPublication(f.file, options, f.deps)).publish(); - const originalIds = ["existing-a", "existing-c"].map( - (id) => f.reports.get(id)!.canonical_finding_id, - ); - f.state.environmentId = "replacement-environment"; - await writeFile( - f.file, - JSON.stringify([ - normalized("existing-a", "critical"), - normalized("new-b"), - normalized("existing-c", "critical"), - ]), - ); - const prepared = await prepareExternalPublication(f.file, options, f.deps); - expect( - prepared.preview.requests.map( - (request) => request.repository.environment_id, - ), - ).toEqual([ - "environment-example", - "replacement-environment", - "environment-example", - ]); - expect( - prepared.preview.requests.flatMap((request) => - request.items.map((item) => [ - item.source_finding_id, - item.expected_version, - ]), - ), - ).toEqual([ - ["existing-a", 1], - ["new-b", 0], - ["existing-c", 1], - ]); - f.state.throttle = true; - await expect(prepared.publish()).rejects.toThrow("Retry-After"); - f.state.throttle = false; - f.state.environmentId = "changed-default"; - const cli = createCliTest(main); - expect( - await cli.runCli(f.command, { - ...f.cliDeps, - externalPublicationPrompt: { - isInteractive: () => true, - confirm: async () => { - expect(cli.stderr.text()).toContain( - "Environment: environment-example, replacement-environment", - ); - expect(cli.stderr.text()).not.toContain("changed-default"); - expect(f.posts).toHaveLength(2); - return true; - }, - }, - }), - ).toBe(0); - expect( - JSON.parse(cli.stdout.text()).receipts.map( - (receipt: FindingImportReceipt) => receipt.id, - ), - ).toEqual(prepared.preview.requests.map((request) => request.request_id)); - expect(f.posts[2]).toBe(f.posts[1]); - expect(JSON.parse(cli.stdout.text()).counts).toEqual({ - created: 1, - updated: 2, - unchanged: 0, - error: 0, - }); - expect( - ["existing-a", "new-b", "existing-c"].map( - (id) => f.reports.get(id)!.environment_id, - ), - ).toEqual([ - "environment-example", - "replacement-environment", - "environment-example", - ]); - expect( - ["existing-a", "existing-c"].map( - (id) => f.reports.get(id)!.canonical_finding_id, - ), - ).toEqual(originalIds); -}); - -test("existing source updates do not need a newly discovered default environment", async () => { - const f = await fixture(); - await (await prepareExternalPublication(f.file, options, f.deps)).publish(); - f.state.environmentId = null; - await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); - const prepared = await prepareExternalPublication(f.file, options, f.deps); - expect(prepared.preview.requests[0]!.repository.environment_id).toBe( - "environment-example", - ); - expect((await prepared.publish()).counts.updated).toBe(1); - const posts = f.posts.length; - await writeFile(f.file, JSON.stringify([normalized("new-vendor")])); - await expect( - prepareExternalPublication(f.file, options, f.deps), - ).rejects.toThrow("Configure an authorized Cloud environment"); - expect(f.posts).toHaveLength(posts); -}); - -test.each([ - ["id", `aif_${"0".repeat(64)}`], - ["environment_id", "unexpected-environment"], -])("source readback must preserve acknowledged %s", async (field, value) => { - const f = await fixture(); - const prepared = await prepareExternalPublication(f.file, options, { - ...f.deps, - fetch: async (url, init) => { - const response = await f.deps.fetch(url, init); - if (new URL(url).pathname.includes("/source_reports/")) { - return Response.json({ - ...(await response.json()), - [field]: value, - }); - } - return response; - }, - }); - await expect(prepared.publish()).rejects.toThrow( - "Cloud readback did not match the saved finding identity", - ); - expect(f.posts).toHaveLength(1); - const retry = await prepareExternalPublication(f.file, options, f.deps); - expect((await retry.publish()).counts.created).toBe(1); - expect(f.posts).toHaveLength(1); -}); - -test("malformed bracketed URLs exclude only their own finding before upload", async () => { - const invalid = normalized("invalid-url"); - const f = await fixture([ - normalized("valid-url"), - { - ...invalid, - evidence: { ...invalid.evidence, url: "https://[127.0.0.1]/finding" }, - }, - ]); - const prepared = await prepareExternalPublication(f.file, options, f.deps); - expect( - prepared.preview.findings.map((finding) => finding.source_finding_id), - ).toEqual(["valid-url"]); - expect(prepared.preview.excluded).toHaveLength(1); - expect(prepared.preview.excluded[0]!.reason).toContain("HTTP(S)"); - expect((await prepared.publish()).counts.created).toBe(1); - expect(f.reports.has("invalid-url")).toBe(false); -}); - -test.each([false, true])( - "publication waits through the server budget while retaining cancellation: %p", - async (cancel) => { - const f = await fixture(); - const arrived = Promise.withResolvers(); - const release = Promise.withResolvers(); - const controller = new AbortController(); - const prepared = await prepareExternalPublication(f.file, options, { - ...f.deps, - signal: controller.signal, - fetch: async (url, init) => { - if (init.method === "POST") { - arrived.resolve(); - await release.promise; - } - return f.deps.fetch(url, init); - }, - }); - const deadlines: { milliseconds: number; controller: AbortController }[] = - []; - const timeout = spyOn(AbortSignal, "timeout").mockImplementation( - (milliseconds) => { - const controller = new AbortController(); - deadlines.push({ milliseconds, controller }); - return controller.signal; - }, - ); - const completion = prepared.publish(); - try { - await Promise.race([ - arrived.promise, - completion.then(() => { - throw new Error( - "Publication completed before the POST was released.", - ); - }), - ]); - // Advance a controlled clock through the server's publication budget. - for (const deadline of deadlines) { - if (deadline.milliseconds <= 45_000) - deadline.controller.abort( - new DOMException("Publication deadline elapsed", "TimeoutError"), - ); - } - if (cancel) controller.abort(new Error("Synthetic cancellation")); - release.resolve(); - if (cancel) { - await expect(completion).rejects.toThrow("Synthetic cancellation"); - expect(f.posts).toHaveLength(0); - const retry = await prepareExternalPublication(f.file, options, f.deps); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests[0]!.request_id).toBe( - prepared.preview.requests[0]!.request_id, - ); - expect((await retry.publish()).counts.created).toBe(1); - expect(f.posts[0]).toBe(JSON.stringify(prepared.preview.requests[0])); - } else { - expect((await completion).counts.created).toBe(1); - expect(f.posts).toHaveLength(1); - } - } finally { - release.resolve(); - await completion.catch(() => undefined); - timeout.mockRestore(); - } - }, -); - -test("numeric evidence size excludes an oversized item and preserves a valid mixed-format sibling", async () => { - const oversized = { - ...normalized("oversized-numbers"), - evidence: { - ...normalized().evidence, - source_data: { metrics: Array(48_000).fill(1e-7) }, - }, - }; - const accepted = { - ...normalized("accepted-numbers"), - evidence: { - ...normalized().evidence, - source_data: { - metrics: [...Array(10_000).fill(1e-7), ...Array(30_000).fill(1e-6)], - }, - }, - }; - expect(Buffer.byteLength(JSON.stringify(accepted.evidence))).toBeGreaterThan( - 256 * 1024, - ); - const f = await fixture([oversized, accepted]); - const prepared = await prepareExternalPublication(f.file, options, f.deps); - expect( - prepared.preview.findings.map((finding) => finding.source_finding_id), - ).toEqual(["accepted-numbers"]); - expect(prepared.preview.excluded).toHaveLength(1); - expect(prepared.preview.excluded[0]!.reason).toContain("256 KiB"); - expect((await prepared.publish()).counts.created).toBe(1); - expect(f.posts).toHaveLength(1); - expect(JSON.parse(f.posts[0]!).items[0].evidence.source_data).toEqual( - accepted.evidence.source_data, - ); - expect(f.reports.has("oversized-numbers")).toBe(false); -}); - -test.each([0xff, 0xfe])( - "invalid UTF-8 byte %p cannot change a source identity", - async (byte) => { - const f = await fixture(); - await writeFile( - f.file, - Buffer.concat([ - Buffer.from('{"source_finding_id":"vendor-'), - Buffer.from([byte]), - Buffer.from('","evidence":{"title":"Example","severity":"high"}}'), - ]), - ); - await expect( - prepareExternalPublication(f.file, options, f.deps), - ).rejects.toThrow("UTF-8"); - expect(f.posts).toHaveLength(0); - await writeFile( - f.file, - `\uFEFF${JSON.stringify(normalized("vendor-\uFFFD"))}`, - ); - expect( - (await readVendorFindings(f.file)).findings[0]!.source_finding_id, - ).toBe("vendor-\uFFFD"); - }, -); - -test("unsafe evidence integers are rejected while exact strings and safe integers survive", async () => { - const f = await fixture(); - await writeFile( - f.file, - `[ - {"source_finding_id":"unsafe-a","evidence":{"title":"Example","severity":"high","source_data":{"counter":9007199254740992}}}, - {"source_finding_id":"unsafe-b","evidence":{"title":"Example","severity":"high","source_data":{"counter":9007199254740993}}}, - {"source_finding_id":"unsafe-negative","evidence":{"title":"Example","severity":"high","source_data":{"counter":-9007199254740993}}}, - {"source_finding_id":"exact","evidence":{"title":"Example","severity":"high","source_data":{"counter":"9007199254740993","safe":9007199254740991,"fraction":0.125}}} - ]`, - ); - const parsed = await readVendorFindings(f.file); - expect(parsed.excluded.map((item) => item.source_finding_id)).toEqual([ - "unsafe-a", - "unsafe-b", - "unsafe-negative", - ]); - expect( - parsed.excluded.every((item) => item.reason.includes("safe integer")), - ).toBe(true); - expect(parsed.findings[0]!.evidence.source_data).toEqual({ - counter: "9007199254740993", - safe: Number.MAX_SAFE_INTEGER, - fraction: 0.125, - }); -}); - -test("mixed valid and rejected input reports partial completion with exit code 1", async () => { - const f = await fixture([ - normalized("invalid", "unsupported"), - normalized("valid"), - ]); - const cli = createCliTest(main); - expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(1); - expect(JSON.parse(cli.stdout.text())).toMatchObject({ - status: "partial", - read: 2, - ready: 1, - counts: { created: 1, error: 0 }, - unacknowledged: 0, - verified: 1, - excluded: [{ source_finding_id: "invalid", position: 1 }], - }); -}); - -test("server item failures identify the vendor record after local exclusions", async () => { - const f = await fixture([ - normalized("invalid", "unsupported"), - normalized("rejected-vendor"), - ]); - f.state.finalError = true; - const cli = createCliTest(main); - expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(1); - expect(JSON.parse(cli.stdout.text())).toMatchObject({ - status: "partial", - verified: 0, - unacknowledged: 0, - failures: [ - { - source_finding_id: "rejected-vendor", - code: "version_conflict", - message: "Reload current evidence", - }, - ], - }); -}); - -test("interrupted CLI output preserves acknowledged batches and resumes only unfinished uploads", async () => { - const f = await fixture( - Array.from({ length: 101 }, (_, i) => normalized(`vendor-${i}`)), - ); - f.state.postBudget = 1; - const first = createCliTest(main); - expect(await first.runCli([...f.command, "--yes"], f.cliDeps)).toBe(2); - const partial = JSON.parse(first.stdout.text()); - expect(partial).toMatchObject({ - status: "interrupted", - ready: 101, - counts: { created: 100 }, - unacknowledged: 1, - verified: 0, - }); - expect(partial.receipts).toHaveLength(1); - expect(partial.savedSubmission).toEndWith(".pending.json"); - expect(partial.error).toContain("Repeat the same command"); - expect(f.reports.size).toBe(100); - f.state.postBudget = Infinity; - const retry = createCliTest(main); - expect(await retry.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); - expect(JSON.parse(retry.stdout.text())).toMatchObject({ - status: "complete", - counts: { created: 101 }, - unacknowledged: 0, - verified: 101, - }); - expect(f.posts).toHaveLength(3); - expect(f.posts[2]).toBe(f.posts[1]); -}); - -test("readback retry preserves acknowledged findings and scanner diagnostics", async () => { - const input = opengrepJson(); - const f = await fixture(input); - f.state.brokenReadback = true; - const settings = { - ...options, - provider: "opengrep" as const, - sourceRepositoryUrl: f.destination().url, - }; - const prepared = await prepareExternalPublication(f.file, settings, f.deps); - try { - await prepared.publish(); - throw new Error("Expected readback interruption"); - } catch (error) { - if (!(error instanceof ExternalPublicationError)) throw error; - expect(error.result).toMatchObject({ - status: "interrupted", - counts: { created: 1 }, - unacknowledged: 0, - verified: 0, - }); - (error.result.report!["errors"] as unknown[]).pop(); - } - const changed = structuredClone(input); - changed.errors[0]!.message = "A different scanner diagnostic"; - await writeFile(f.file, JSON.stringify(changed)); - const different = await prepareExternalPublication(f.file, settings, f.deps); - expect(different.preview.resumed).toBe(false); - expect(different.preview.report?.["errors"]).toEqual(changed.errors); - await writeFile(f.file, JSON.stringify(input)); - const original = await prepareExternalPublication(f.file, settings, f.deps); - expect(original.preview.resumed).toBe(true); - f.state.brokenReadback = false; - const retried = await original.publish(); - expect(retried.report?.["errors"]).toEqual(input.errors); - expect(retried.verified).toBe(1); - expect(f.posts).toHaveLength(1); -}); - -test("each discovery page requests the repository and refreshes its private import capabilities", async () => { - const f = await fixture(blackDuckBomReport()); - const repository = `${f.destination().url}.git`; - const queries: URLSearchParams[] = []; - const prepared = await prepareExternalPublication( - f.file, - { - ...options, - repository, - provider: "blackduck", - sourceRepositoryUrl: f.destination().url, - }, - { - ...f.deps, - fetch: async (input, init) => { - const url = new URL(input); - if (!url.pathname.endsWith("/repositories")) - return f.deps.fetch(input, init); - queries.push(url.searchParams); - const first = queries.length === 1; - return Response.json( - { - object: "page", - data: first ? [] : [f.destination()], - has_more: first, - next: first ? "next-page" : null, - }, - { - headers: first - ? { - "Codex-Security-Private-Import-Job-Version": "3", - "Codex-Security-Private-Import-Providers": "blackduck", - } - : {}, - }, - ); - }, - }, - ); - expect(prepared.preview.privateImport).toBeUndefined(); - expect(prepared.preview.destination.id).toBe(options.repository); - expect(queries.map((query) => query.get("repository"))).toEqual([ - repository, - repository, - ]); - expect(queries.map((query) => query.get("page"))).toEqual([ - null, - "next-page", - ]); - expect(f.posts).toHaveLength(0); -}); - -test("repository URLs and IDs share the same resumable destination", async () => { - const f = await fixture(); - f.state.loseResponse = true; - const first = await prepareExternalPublication( - f.file, - { ...options, repository: `${f.destination().url}.git` }, - f.deps, - ); - expect(first.preview.destination.id).toBe(options.repository); - await expect(first.publish()).rejects.toThrow("resume the saved request"); - const retry = await prepareExternalPublication(f.file, options, f.deps); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).status).toBe("complete"); - expect(f.posts[1]).toBe(f.posts[0]); -}); - -test("Cloud deployment routing covers every request and isolates saved retries", async () => { - const f = await fixture(); - const cloudBase = "https://cloud.example.test/pilot/backend-api/aardvark"; - const requested: string[] = []; - const deps = { - ...f.deps, - environment: { - ...f.deps.environment, - CODEX_SECURITY_CLOUD_BASE_URL: `${cloudBase}///`, - }, - fetch: async (url: string, init: RequestInit) => { - requested.push(url); - expect(url.startsWith(`${cloudBase}/external/`)).toBe(true); - return f.deps.fetch(url, init); - }, - }; - f.state.brokenReadback = true; - const prepared = await prepareExternalPublication(f.file, options, deps); - expect(prepared.preview.cloudApiUrl).toBe(`${cloudBase}/external`); - await expect(prepared.publish()).rejects.toThrow("resume the saved request"); - const otherDeployment = await prepareExternalPublication(f.file, options, { - ...f.deps, - environment: { - ...f.deps.environment, - CODEX_SECURITY_CLOUD_BASE_URL: - "https://other-cloud.example.test/backend-api/aardvark", - }, - }); - expect(otherDeployment.preview.resumed).toBe(false); - const retry = await prepareExternalPublication(f.file, options, deps); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests).toEqual(prepared.preview.requests); - f.state.brokenReadback = false; - const result = await retry.publish(); - expect(result.cloudApiUrl).toBe(`${cloudBase}/external`); - expect(result.cloudUrl).toBeUndefined(); - expect(result.verified).toBe(1); - expect(f.posts).toHaveLength(1); - expect(requested.some((url) => url.includes("/repositories?"))).toBe(true); - expect(requested.some((url) => url.includes("/source_reports?"))).toBe(true); - expect(requested.some((url) => url.endsWith("/finding_imports"))).toBe(true); - expect(requested.some((url) => url.includes("/source_reports/"))).toBe(true); - expect(requested.some((url) => url.includes("/external/v2/"))).toBe(false); -}); - -test.each(["localhost", "127.0.0.1", "127.0.0.2", "[::1]"])( - "loopback Cloud APIs support local HTTP publication: %s", - async (host) => { - const f = await fixture(); - const cloudBase = `http://${host}:4200/backend-api/aardvark`; - const prepared = await prepareExternalPublication(f.file, options, { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: cloudBase, + vulnerabilityFindings: { + nodes: [record], + pageInfo: { hasNextPage: false }, + }, }, - fetch: (url, init) => { - expect(url.startsWith(`${cloudBase}/external/`)).toBe(true); - return f.deps.fetch(url, init); + }); + const parsed = await readVendorFindings(f.file); + expect(parsed.excluded).toEqual([]); + expect(parsed.findings[0]).toMatchObject({ + source_finding_id: record.id, + evidence: { + packages: [ + { name: "example-lib", ecosystem, installed_version: "1.0.0" }, + ], + image_digests: [digest], + source_data: record, }, }); - expect((await prepared.publish()).status).toBe("complete"); - expect(f.posts).toHaveLength(1); }, ); -test("a separate Cloud app URL follows the resolved repository without changing API routing", async () => { - const f = await fixture(); - const requested: string[] = []; - const environment = { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: - "https://api.example.test/backend-api/aardvark", - CODEX_SECURITY_CLOUD_WEB_URL: - "http://localhost:4200/security?preview=true#/old", - }; - const prepared = await prepareExternalPublication( - f.file, - { ...options, repository: f.destination().url }, - { +test.each([false, true])( + "receipt client IDs correlate reordered results (cached: %p)", + async (cached) => { + const f = await fixture([ + normalized("vendor-first", "high"), + normalized("vendor-second", "low"), + ]); + const deps = { ...f.deps, - environment, - fetch: (url, init) => { - requested.push(url); - return f.deps.fetch(url, init); + fetch: async (url: string, init: RequestInit) => { + const response = await f.deps.fetch(url, init); + if (init.method !== "POST" || !response.ok) return response; + const receipt = (await response.json()) as FindingImportReceipt; + receipt.results.reverse(); + return new Response(JSON.stringify(receipt), { + headers: { "Content-Type": "application/json" }, + }); }, - }, - ); - const link = `http://localhost:4200/security?preview=true#/repositories/${options.repository}/findings`; - expect(prepared.preview.cloudUrl).toBe(link); - expect((await prepared.publish()).cloudUrl).toBe(link); - expect( - requested.every((url) => - url.startsWith(environment.CODEX_SECURITY_CLOUD_BASE_URL), - ), - ).toBe(true); - - const cli = createCliTest(main); - expect( - await cli.runCli( - f.command - .filter((value) => !["--format", "json"].includes(value)) - .concat("--yes"), - { ...f.cliDeps, environment }, - ), - ).toBe(0); - expect(cli.stdout.text()).toContain(`View Cloud repository: ${link}`); -}); - -test.each([ - "", - "/relative", - "file:///tmp/cloud", - "https://user:password@example.test", -])( - "invalid Cloud app URL fails before authentication or network: %s", - async (url) => { - const f = await fixture(); - let calls = 0; - await expect( - prepareExternalPublication(f.file, options, { - ...f.deps, - environment: { ...f.environment, CODEX_SECURITY_CLOUD_WEB_URL: url }, - credentials: async () => { - calls++; - throw new Error("Unexpected auth"); - }, - fetch: async () => { - calls++; - throw new Error("Unexpected network"); - }, - }), - ).rejects.toThrow("CODEX_SECURITY_CLOUD_WEB_URL"); - expect(calls).toBe(0); + }; + let prepared = await prepareExternalPublication(f.file, options, deps); + if (cached) { + f.state.brokenReadback = true; + await expect(prepared.publish()).rejects.toThrow("Readback unavailable"); + f.state.brokenReadback = false; + f.state.postBudget = 0; + prepared = await prepareExternalPublication(f.file, options, deps); + expect(prepared.preview.resumed).toBe(true); + } + const result = await prepared.publish(); + expect(result.counts.created).toBe(2); + expect(result.receipts[0]!.results.map((item) => item.client_id)).toEqual([ + "item-2", + "item-1", + ]); + expect(f.posts).toHaveLength(1); + expect(f.receipts.size).toBe(1); }, ); -test.each([ - "", - " ", - "/relative", - "file:///tmp/cloud", - "https://user:password@cloud.example.test", - "https://cloud.example.test?deployment=pilot", - "https://cloud.example.test#pilot", - "http://cloud.example.test/backend-api/aardvark", - "http://127.example.test/backend-api/aardvark", - "http://192.0.2.1/backend-api/aardvark", -])( - "invalid Cloud base URL fails before authentication or network: %s", - async (baseUrl) => { - const f = await fixture(); - let calls = 0; +test.each(["duplicate", "missing", "unexpected"])( + "receipt client IDs reject %s results before checkpointing", + async (invalid) => { + const f = await fixture([ + normalized("vendor-first"), + normalized("vendor-second"), + ]); + const deps = { + ...f.deps, + fetch: async (url: string, init: RequestInit) => { + const response = await f.deps.fetch(url, init); + if (init.method !== "POST" || !response.ok) return response; + const receipt = (await response.json()) as FindingImportReceipt; + if (invalid === "duplicate") + receipt.results[1]!.client_id = receipt.results[0]!.client_id; + if (invalid === "missing") receipt.results.pop(); + if (invalid === "unexpected") + receipt.results[0]!.client_id = "unexpected-client"; + return new Response(JSON.stringify(receipt), { + headers: { "Content-Type": "application/json" }, + }); + }, + }; await expect( - prepareExternalPublication(f.file, options, { - ...f.deps, - environment: { - ...f.deps.environment, - CODEX_SECURITY_CLOUD_BASE_URL: baseUrl, - }, - credentials: async () => { - calls++; - throw new Error("Unexpected credential read"); - }, - fetch: async () => { - calls++; - throw new Error("Unexpected network request"); - }, - }), - ).rejects.toThrow("CODEX_SECURITY_CLOUD_BASE_URL"); - expect(calls).toBe(0); - }, -); - -test("read lookups and verification run concurrently with bounded requests and ordered input", async () => { - const f = await fixture( - Array.from({ length: 9 }, (_, i) => normalized(`vendor-${i}`)), - ); - const phases: ExternalPublicationProgress[] = []; - let active = 0; - let maximum = 0; - const prepared = await prepareExternalPublication(f.file, options, { - ...f.deps, - onProgress: (event) => phases.push(event), - fetch: async (url, init) => { - if (!url.includes("/source_reports")) return f.deps.fetch(url, init); - active++; - maximum = Math.max(maximum, active); - try { - await Promise.resolve(); - return await f.deps.fetch(url, init); - } finally { - active--; - } - }, - }); - expect( - prepared.preview.requests.flatMap((batch) => - batch.items.map((item) => item.source_finding_id), - ), - ).toEqual(Array.from({ length: 9 }, (_, i) => `vendor-${i}`)); - expect((await prepared.publish()).status).toBe("complete"); - expect(maximum).toBeGreaterThan(1); - expect(maximum).toBeLessThanOrEqual(4); - expect(active).toBe(0); - for (const phase of ["preparing", "uploading", "verifying"]) + (await prepareExternalPublication(f.file, options, deps)).publish(), + ).rejects.toThrow("Cloud returned a receipt for a different publication"); + const directory = join(f.root, "state", "external-finding-publications"); + const pending = (await readdir(directory)).find((name) => + name.endsWith(".pending.json"), + )!; expect( - phases.filter((event) => event.phase === phase).at(-1), - ).toMatchObject({ completed: 9, total: 9 }); -}); - -test("progress is optional and stderr progress never corrupts JSON results", async () => { - const f = await fixture(); - const phases = new Set(); - const prepared = await prepareExternalPublication(f.file, options, { - ...f.deps, - onProgress: ({ phase }) => { - phases.add(phase); - if (phase === "reading") - return Promise.reject(new Error("Optional async observer failed")); - throw new Error("Optional observer failed"); - }, - }); - expect((await prepared.publish()).status).toBe("complete"); - expect([...phases]).toEqual([ - "reading", - "discovering", - "preparing", - "waiting", - "uploading", - "verifying", - ]); - const cli = createCliTest(main, { stderr: true }); - expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); - expect(JSON.parse(cli.stdout.text())).toMatchObject({ - status: "complete", - counts: { unchanged: 1 }, - }); - expect(cli.stderr.text()).toContain("Preparing findings"); - expect(cli.stderr.text()).toContain("Verifying source records: 1/1"); -}); + JSON.parse(await readFile(join(directory, pending), "utf8")).receipts ?? + [], + ).toEqual([]); + expect(f.posts).toHaveLength(1); + }, +); -test("human previews stay compact while JSON previews retain complete evidence", async () => { - const records = Array.from({ length: 6 }, (_, i) => ({ - ...normalized(`vendor-${i}`), - evidence: { - ...normalized().evidence, - source_data: { detail: `complete-vendor-evidence-${i}` }, - }, - })); - const f = await fixture(records); - const human = createCliTest(main); - expect( - await human.runCli([...f.command.slice(0, -2), "--dry-run"], f.cliDeps), - ).toBe(0); - expect(human.stderr.text()).toContain("Showing 5 of 6 findings"); - expect(human.stderr.text()).not.toContain("complete-vendor-evidence-"); - expect(human.stdout.text()).not.toContain("request_id"); - const json = createCliTest(main); - expect(await json.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); - expect(JSON.parse(json.stdout.text()).findings).toHaveLength(6); - expect( - JSON.parse(json.stdout.text()).findings[5].evidence.source_data.detail, - ).toBe("complete-vendor-evidence-5"); - expect(f.posts).toHaveLength(0); -}); +test.each(["replacement-environment", null])( + "saved publication preserves its environment when discovery changes to %p", + async (environmentId) => { + const f = await fixture(); + f.state.loseResponse = true; + await expect( + (await prepareExternalPublication(f.file, options, f.deps)).publish(), + ).rejects.toThrow("resume the saved request"); + const original = f.posts[0]!; + f.state.environmentId = environmentId; + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.repository.environment_id).toBe( + "environment-example", + ); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts.at(-1)).toBe(original); + }, +); -test("normalized SDK publication preserves caller evidence and exclusion accounting", async () => { - const f = await fixture(); - const input = { - read: 2, - findings: [normalized()], - excluded: [ - { - position: 2, - source_finding_id: null, - reason: "Unsupported source record.", - }, - ], - }; - const original = structuredClone(input); - const prepared = await prepareNormalizedExternalPublication( - input as Parameters[0], - Object.create(options) as typeof options, - f.deps, +test("mixed environments preserve input order and are shown before confirmation", async () => { + const f = await fixture([normalized("existing-a"), normalized("existing-c")]); + await (await prepareExternalPublication(f.file, options, f.deps)).publish(); + const originalIds = ["existing-a", "existing-c"].map( + (id) => f.reports.get(id)!.canonical_finding_id, ); - expect(input).toEqual(original); - expect(prepared.preview.read).toBe(2); - const result = await prepared.publish(); - expect(result.status).toBe("partial"); - expect(result.counts.created).toBe(1); - expect(result.excluded).toEqual(input.excluded); - expect(result.verified).toBe(1); -}); - -test("GitHub publication previews, imports, re-imports, updates and resumes selected-ref evidence", async () => { - const imported = importedAlert(); - const f = await fixture([imported]); - const githubOptions = { - ...options, - provider: "github" as const, - sourceKey: "github.com/example/project/code-scanning", - }; - const command = f.command.map((arg) => - arg === "wiz" - ? "github" - : arg === options.sourceKey - ? githubOptions.sourceKey - : arg, + f.state.environmentId = "replacement-environment"; + await writeFile( + f.file, + JSON.stringify([ + normalized("existing-a", "critical"), + normalized("new-b"), + normalized("existing-c", "critical"), + ]), ); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.requests.map( + (request) => request.repository.environment_id, + ), + ).toEqual([ + "environment-example", + "replacement-environment", + "environment-example", + ]); + expect( + prepared.preview.requests.flatMap((request) => + request.items.map((item) => [ + item.source_finding_id, + item.expected_version, + ]), + ), + ).toEqual([ + ["existing-a", 1], + ["new-b", 0], + ["existing-c", 1], + ]); + f.state.throttle = true; + await expect(prepared.publish()).rejects.toThrow("Retry-After"); + f.state.throttle = false; + f.state.environmentId = "changed-default"; const cli = createCliTest(main); expect( - await cli.runCli([...command, "--dry-run", "--format", "json"], f.cliDeps), + await cli.runCli(f.command, { + ...f.cliDeps, + externalPublicationPrompt: { + isInteractive: () => true, + confirm: async () => { + expect(cli.stderr.text()).toContain( + "Environment: environment-example, replacement-environment", + ); + expect(cli.stderr.text()).not.toContain("changed-default"); + expect(f.posts).toHaveLength(2); + return true; + }, + }, + }), ).toBe(0); - expect(f.posts).toHaveLength(0); - expect(JSON.parse(cli.stdout.text()).source.provider).toBe("github"); - const first = await ( - await prepareExternalPublication(f.file, githubOptions, f.deps) - ).publish(); - expect(first.counts.created).toBe(1); - expect(first.verified).toBe(1); - const repeated = await ( - await prepareExternalPublication(f.file, githubOptions, f.deps) - ).publish(); - expect(repeated.counts.unchanged).toBe(1); - imported.alert.most_recent_instance.state = "fixed"; - await writeFile(f.file, JSON.stringify([imported])); - f.state.loseResponse = true; - await expect( - (await prepareExternalPublication(f.file, githubOptions, f.deps)).publish(), - ).rejects.toBeInstanceOf(ExternalPublicationError); - const prepared = await prepareExternalPublication( - f.file, - githubOptions, - f.deps, - ); - expect(prepared.preview.resumed).toBe(true); - const resumed = await prepared.publish(); - expect(resumed.counts.updated).toBe(1); - expect(resumed.verified).toBe(1); - expect([...f.reports.values()][0]).toMatchObject({ - source: { provider: "github" }, - evidence: { details: { source_status: "fixed" }, source_data: imported }, - assessment: { state: "not_assessed" }, + expect( + JSON.parse(cli.stdout.text()).receipts.map( + (receipt: FindingImportReceipt) => receipt.id, + ), + ).toEqual(prepared.preview.requests.map((request) => request.request_id)); + expect(f.posts[2]).toBe(f.posts[1]); + expect(JSON.parse(cli.stdout.text()).counts).toEqual({ + created: 1, + updated: 2, + unchanged: 0, + error: 0, }); - await writeFile(f.file, JSON.stringify([importedAlert("refs/heads/review")])); - const differentRef = await ( - await prepareExternalPublication(f.file, githubOptions, f.deps) - ).publish(); - expect(differentRef.counts.created).toBe(1); - expect(f.reports.size).toBe(2); + expect( + ["existing-a", "new-b", "existing-c"].map( + (id) => f.reports.get(id)!.environment_id, + ), + ).toEqual([ + "environment-example", + "replacement-environment", + "environment-example", + ]); + expect( + ["existing-a", "existing-c"].map( + (id) => f.reports.get(id)!.canonical_finding_id, + ), + ).toEqual(originalIds); }); -test.each([ - ["code scanning", () => normalizeGitHubCodeScanningAlerts([importedAlert()])], - [ - "repository advisory", - () => normalizeGitHubAdvisories([importedAdvisory()]), - ], -] as const)( - "the retained GitHub %s export binds its SDK projection to the original repository", - async (_, make) => { - const input = make(); - const evidence = input.findings[0]!.evidence; - const f = await fixture(undefined, otherRepository); - const prepare = () => - prepareNormalizedExternalPublication( - input, - { ...options, provider: "github" }, - f.deps, - ); - evidence.details!.repository!.url = otherRepository; - await expect(prepare()).rejects.toThrow( - "does not match the original GitHub finding", - ); - delete evidence.details; - await expect(prepare()).rejects.toThrow( - "does not match the selected Cloud repository", - ); - expect(f.posts).toHaveLength(0); - - f.state.repositoryUrl = exampleRepository; - await f.loseResponse(await prepare()); - f.state.repositoryUrl = otherRepository; - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).verified).toBe(1); - expect(f.posts[1]).toBe(f.posts[0]); - }, -); - -test("GitHub imports reject a different Cloud destination before uploading", async () => { - const imported = importedAlert("refs/heads/main", "git.example.test"); - const f = await fixture([imported]); - await expect( - prepareExternalPublication( - f.file, - { ...options, provider: "github" }, - f.deps, - ), - ).rejects.toThrow("does not match the selected Cloud repository"); - expect(f.posts).toHaveLength(0); +test("existing source updates do not need a newly discovered default environment", async () => { + const f = await fixture(); + await (await prepareExternalPublication(f.file, options, f.deps)).publish(); + f.state.environmentId = null; + await writeFile(f.file, JSON.stringify([normalized("vendor-1", "critical")])); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect(prepared.preview.requests[0]!.repository.environment_id).toBe( + "environment-example", + ); + expect((await prepared.publish()).counts.updated).toBe(1); + const posts = f.posts.length; + await writeFile(f.file, JSON.stringify([normalized("new-vendor")])); + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("Configure an authorized Cloud environment"); + expect(f.posts).toHaveLength(posts); }); -test("global GHSA publication retains unassessed applicability through repeat import and withdrawal", async () => { - const imported = importedAdvisory("global"); - const f = await fixture([imported]); - const selected = { - ...options, - provider: "github" as const, - sourceKey: "github.com/example/project/security-advisories", - }; - expect( - ( - await ( - await prepareExternalPublication(f.file, selected, f.deps) - ).publish() - ).counts.created, - ).toBe(1); - expect( - ( - await ( - await prepareExternalPublication(f.file, selected, f.deps) - ).publish() - ).counts.unchanged, - ).toBe(1); - const withdrawn = { - ...imported, - advisory: { ...imported.advisory, withdrawn_at: "2026-01-03T00:00:00Z" }, - }; - await writeFile(f.file, JSON.stringify([withdrawn])); - const result = await ( - await prepareExternalPublication(f.file, selected, f.deps) - ).publish(); - expect(result.counts.updated).toBe(1); - expect(result.verified).toBe(1); - const report = [...f.reports.values()][0]!; - expect(report.evidence.details).toMatchObject({ - kind: "advisory", - advisory: { source: "global" }, - source_status: "withdrawn", +test.each([ + ["id", `aif_${"0".repeat(64)}`], + ["environment_id", "unexpected-environment"], +])("source readback must preserve acknowledged %s", async (field, value) => { + const f = await fixture(); + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + fetch: async (url, init) => { + const response = await f.deps.fetch(url, init); + if (new URL(url).pathname.includes("/source_reports/")) { + return Response.json({ + ...(await response.json()), + [field]: value, + }); + } + return response; + }, }); - expect(report.evidence.details).not.toHaveProperty("repository"); - expect(report.assessment?.state).toBe("not_assessed"); - expect(report.evidence.packages![0]!.installed_version).toBeNull(); + await expect(prepared.publish()).rejects.toThrow( + "Cloud readback did not match the saved finding identity", + ); + expect(f.posts).toHaveLength(1); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); }); -test("saved GitHub JSONL and gzip preserve evidence while malformed UTF-8 cannot change identities", async () => { - const imported = [importedAlert(), importedAdvisory("global")]; - const f = await fixture(imported); - const jsonl = imported.map((record) => JSON.stringify(record)).join("\n"); - for (const bytes of [Buffer.from(jsonl), gzipSync(jsonl)]) { - await writeFile(f.file, bytes); - const parsed = await readGitHubFindings(f.file); - expect(parsed.read).toBe(2); - expect(parsed.excluded).toEqual([]); - expect( - parsed.findings.map((finding) => finding.evidence.source_data) as unknown, - ).toEqual(imported); - } - await writeFile(f.file, Buffer.from([0xff])); - await expect(readGitHubFindings(f.file)).rejects.toThrow("UTF-8"); +test("malformed bracketed URLs exclude only their own finding before upload", async () => { + const invalid = normalized("invalid-url"); + const f = await fixture([ + normalized("valid-url"), + { + ...invalid, + evidence: { ...invalid.evidence, url: "https://[127.0.0.1]/finding" }, + }, + ]); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.findings.map((finding) => finding.source_finding_id), + ).toEqual(["valid-url"]); + expect(prepared.preview.excluded).toHaveLength(1); + expect(prepared.preview.excluded[0]!.reason).toContain("HTTP(S)"); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.reports.has("invalid-url")).toBe(false); }); -test("normalized publication omits only new null fields to match canonical Cloud readback", async () => { - const f = await fixture(); - const evidence = { - title: "Synthetic advisory without known package range", - severity: "unknown" as const, - packages: [ - { - name: "example-package", - vulnerable_version_range: null, - installed_version: null, +test.each([false, true])( + "publication waits through the server budget while retaining cancellation: %p", + async (cancel) => { + const f = await fixture(); + const arrived = Promise.withResolvers(); + const release = Promise.withResolvers(); + const controller = new AbortController(); + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + signal: controller.signal, + fetch: async (url, init) => { + if (init.method === "POST") { + arrived.resolve(); + await release.promise; + } + return f.deps.fetch(url, init); }, - ], - details: { - kind: "advisory" as const, - repository: null, - advisory: { source: "global" as const }, + }); + const deadlines: { milliseconds: number; controller: AbortController }[] = + []; + const timeout = spyOn(AbortSignal, "timeout").mockImplementation( + (milliseconds) => { + const controller = new AbortController(); + deadlines.push({ milliseconds, controller }); + return controller.signal; + }, + ); + const completion = prepared.publish(); + try { + await Promise.race([ + arrived.promise, + completion.then(() => { + throw new Error( + "Publication completed before the POST was released.", + ); + }), + ]); + // Advance a controlled clock through the server's publication budget. + for (const deadline of deadlines) { + if (deadline.milliseconds <= 45_000) + deadline.controller.abort( + new DOMException("Publication deadline elapsed", "TimeoutError"), + ); + } + if (cancel) controller.abort(new Error("Synthetic cancellation")); + release.resolve(); + if (cancel) { + await expect(completion).rejects.toThrow("Synthetic cancellation"); + expect(f.posts).toHaveLength(0); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests[0]!.request_id).toBe( + prepared.preview.requests[0]!.request_id, + ); + expect((await retry.publish()).counts.created).toBe(1); + expect(f.posts[0]).toBe(JSON.stringify(prepared.preview.requests[0])); + } else { + expect((await completion).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); + } + } finally { + release.resolve(); + await completion.catch(() => undefined); + timeout.mockRestore(); + } + }, +); + +test("numeric evidence size excludes an oversized item and preserves a valid mixed-format sibling", async () => { + const oversized = { + ...normalized("oversized-numbers"), + evidence: { + ...normalized().evidence, + source_data: { metrics: Array(48_000).fill(1e-7) }, }, }; - const prepared = await prepareNormalizedExternalPublication( - { - read: 1, - findings: [{ source_finding_id: "synthetic-global-advisory", evidence }], - excluded: [], + const accepted = { + ...normalized("accepted-numbers"), + evidence: { + ...normalized().evidence, + source_data: { + metrics: [...Array(10_000).fill(1e-7), ...Array(30_000).fill(1e-6)], + }, }, - options, - f.deps, + }; + expect(Buffer.byteLength(JSON.stringify(accepted.evidence))).toBeGreaterThan( + 256 * 1024, ); - const sent = prepared.preview.findings[0]!.evidence; - expect(sent.packages![0]).not.toHaveProperty("vulnerable_version_range"); - expect(sent.packages![0]!.installed_version).toBeNull(); - expect(sent.details).not.toHaveProperty("repository"); - expect(evidence.details.repository).toBeNull(); - expect((await prepared.publish()).verified).toBe(1); + const f = await fixture([oversized, accepted]); + const prepared = await prepareExternalPublication(f.file, options, f.deps); + expect( + prepared.preview.findings.map((finding) => finding.source_finding_id), + ).toEqual(["accepted-numbers"]); + expect(prepared.preview.excluded).toHaveLength(1); + expect(prepared.preview.excluded[0]!.reason).toContain("256 KiB"); + expect((await prepared.publish()).counts.created).toBe(1); + expect(f.posts).toHaveLength(1); + expect(JSON.parse(f.posts[0]!).items[0].evidence.source_data).toEqual( + accepted.evidence.source_data, + ); + expect(f.reports.has("oversized-numbers")).toBe(false); +}); + +test.each([0xff, 0xfe])( + "invalid UTF-8 byte %p cannot change a source identity", + async (byte) => { + const f = await fixture(); + await writeFile( + f.file, + Buffer.concat([ + Buffer.from('{"source_finding_id":"vendor-'), + Buffer.from([byte]), + Buffer.from('","evidence":{"title":"Example","severity":"high"}}'), + ]), + ); + await expect( + prepareExternalPublication(f.file, options, f.deps), + ).rejects.toThrow("UTF-8"); + expect(f.posts).toHaveLength(0); + await writeFile( + f.file, + `\uFEFF${JSON.stringify(normalized("vendor-\uFFFD"))}`, + ); + expect( + (await readVendorFindings(f.file)).findings[0]!.source_finding_id, + ).toBe("vendor-\uFFFD"); + }, +); +test("unsafe evidence integers are rejected while exact strings and safe integers survive", async () => { + const f = await fixture(); await writeFile( f.file, - JSON.stringify([ - { - source_finding_id: "synthetic-sast", - evidence: { - title: "Synthetic SAST evidence", - severity: "high", - packages: [ - { name: "example-package", vulnerable_version_range: null }, - ], - details: { - kind: "sast", - repository: { url: "https://github.com/example/project" }, - advisory: null, - }, - }, - }, - ]), - ); - const wiz = await prepareExternalPublication(f.file, options, f.deps); - expect(wiz.preview.findings[0]!.evidence.details).not.toHaveProperty( - "advisory", + `[ + {"source_finding_id":"unsafe-a","evidence":{"title":"Example","severity":"high","source_data":{"counter":9007199254740992}}}, + {"source_finding_id":"unsafe-b","evidence":{"title":"Example","severity":"high","source_data":{"counter":9007199254740993}}}, + {"source_finding_id":"unsafe-negative","evidence":{"title":"Example","severity":"high","source_data":{"counter":-9007199254740993}}}, + {"source_finding_id":"exact","evidence":{"title":"Example","severity":"high","source_data":{"counter":"9007199254740993","safe":9007199254740991,"fraction":0.125}}} + ]`, ); - expect(wiz.preview.findings[0]!.evidence.details!.scanner_origin).toBeNull(); - expect((await wiz.publish()).verified).toBe(1); + const parsed = await readVendorFindings(f.file); + expect(parsed.excluded.map((item) => item.source_finding_id)).toEqual([ + "unsafe-a", + "unsafe-b", + "unsafe-negative", + ]); + expect( + parsed.excluded.every((item) => item.reason.includes("safe integer")), + ).toBe(true); + expect(parsed.findings[0]!.evidence.source_data).toEqual({ + counter: "9007199254740993", + safe: Number.MAX_SAFE_INTEGER, + fraction: 0.125, + }); }); -test("normalized SDK publication detaches raw evidence aliases before validating every finding", async () => { - const f = await fixture(); - const sharedPackage = { name: "example-package" }; - const input = { +test("mixed valid and rejected input reports partial completion with exit code 1", async () => { + const f = await fixture([ + normalized("invalid", "unsupported"), + normalized("valid"), + ]); + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(1); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + status: "partial", read: 2, - report: { package: sharedPackage }, - findings: [ - { - source_finding_id: "shared-first", - evidence: { - title: "First report", - severity: "high" as const, - packages: [sharedPackage], - source_data: { package: sharedPackage }, - }, - }, + ready: 1, + counts: { created: 1, error: 0 }, + unacknowledged: 0, + verified: 1, + excluded: [{ source_finding_id: "invalid", position: 1 }], + }); +}); + +test("server item failures identify the vendor record after local exclusions", async () => { + const f = await fixture([ + normalized("invalid", "unsupported"), + normalized("rejected-vendor"), + ]); + f.state.finalError = true; + const cli = createCliTest(main); + expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(1); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + status: "partial", + verified: 0, + unacknowledged: 0, + failures: [ { - source_finding_id: "shared-second", - evidence: { - title: "Second report", - severity: "high" as const, - source_data: { earlier_package: sharedPackage }, - }, + source_finding_id: "rejected-vendor", + code: "version_conflict", + message: "Reload current evidence", }, ], - excluded: [], - }; - const original = JSON.stringify(input); - const prepared = await prepareNormalizedExternalPublication( - input, - options, - f.deps, - ); - expect(JSON.stringify(input)).toBe(original); - expect(prepared.preview.report).toEqual({ - package: { name: "example-package" }, }); - expect(prepared.preview.findings[0]!.evidence.source_data).toEqual({ - package: { name: "example-package" }, +}); + +test("interrupted CLI output preserves acknowledged batches and resumes only unfinished uploads", async () => { + const f = await fixture( + Array.from({ length: 101 }, (_, i) => normalized(`vendor-${i}`)), + ); + f.state.postBudget = 1; + const first = createCliTest(main); + expect(await first.runCli([...f.command, "--yes"], f.cliDeps)).toBe(2); + const partial = JSON.parse(first.stdout.text()); + expect(partial).toMatchObject({ + status: "interrupted", + ready: 101, + counts: { created: 100 }, + unacknowledged: 1, + verified: 0, }); - expect(prepared.preview.findings[1]!.evidence.source_data).toEqual({ - earlier_package: { name: "example-package" }, + expect(partial.receipts).toHaveLength(1); + expect(partial.savedSubmission).toEndWith(".pending.json"); + expect(partial.error).toContain("Repeat the same command"); + expect(f.reports.size).toBe(100); + f.state.postBudget = Infinity; + const retry = createCliTest(main); + expect(await retry.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); + expect(JSON.parse(retry.stdout.text())).toMatchObject({ + status: "complete", + counts: { created: 101 }, + unacknowledged: 0, + verified: 101, }); - await writeFile(f.file, JSON.stringify(input.findings)); - const fromFile = await prepareExternalPublication(f.file, options, f.deps); - expect(prepared.preview.findings).toEqual(fromFile.preview.findings); + expect(f.posts).toHaveLength(3); + expect(f.posts[2]).toBe(f.posts[1]); }); -test.each(["code-scanning", "advisory"])( - "%s imports bind an explicit default port to the matching Cloud repository", - async (family) => { - const alert = importedAlert("refs/heads/main", "github.com:443"); - const advisory = importedAdvisory(); - advisory.githubHost = "github.com:443"; - advisory.url = advisory.advisory.html_url = advisory.url.replace( - "github.com", - advisory.githubHost, - ); - const f = await fixture([family === "code-scanning" ? alert : advisory]); - const selected = { ...options, provider: "github" as const }; - await expect( - prepareExternalPublication(f.file, selected, f.deps), - ).rejects.toThrow("does not match the selected Cloud repository"); - expect(f.posts).toHaveLength(0); - const matching = { - ...f.deps, - fetch: async (url: string, init: RequestInit) => { - const response = await f.deps.fetch(url, init); - if (new URL(url).pathname.endsWith("/repositories")) { - const body = await response.json(); - body.data[0].url = "https://github.com:443/example/project"; - return Response.json(body); - } - return response; - }, - }; - const result = await ( - await prepareExternalPublication(f.file, selected, matching) - ).publish(); - expect(result.counts.created).toBe(1); - expect(result.verified).toBe(1); - }, -); - -function scoredFindings( - score: unknown, - report?: Record, - aliasSelection = false, -) { - const evidence = { - title: "Scanner score", - severity: "high" as const, - source_data: { score }, +test("readback retry preserves acknowledged findings and scanner diagnostics", async () => { + const input = opengrepJson(); + const f = await fixture(input); + f.state.brokenReadback = true; + const settings = { + ...options, + provider: "opengrep" as const, + sourceRepositoryUrl: f.destination().url, }; - return { - read: 1, - excluded: [], - findings: [ - { - source_finding_id: "boxed-score", - selection: aliasSelection ? evidence : "security", // Hashed by older publishers but absent from requests. - evidence, + const prepared = await prepareExternalPublication(f.file, settings, f.deps); + try { + await prepared.publish(); + throw new Error("Expected readback interruption"); + } catch (error) { + if (!(error instanceof ExternalPublicationError)) throw error; + expect(error.result).toMatchObject({ + status: "interrupted", + counts: { created: 1 }, + unacknowledged: 0, + verified: 0, + }); + (error.result.report!["errors"] as unknown[]).pop(); + } + const changed = structuredClone(input); + changed.errors[0]!.message = "A different scanner diagnostic"; + await writeFile(f.file, JSON.stringify(changed)); + const different = await prepareExternalPublication(f.file, settings, f.deps); + expect(different.preview.resumed).toBe(false); + expect(different.preview.report?.["errors"]).toEqual(changed.errors); + await writeFile(f.file, JSON.stringify(input)); + const original = await prepareExternalPublication(f.file, settings, f.deps); + expect(original.preview.resumed).toBe(true); + f.state.brokenReadback = false; + const retried = await original.publish(); + expect(retried.report?.["errors"]).toEqual(input.errors); + expect(retried.verified).toBe(1); + expect(f.posts).toHaveLength(1); +}); + +test("each discovery page requests the repository and refreshes its private import capabilities", async () => { + const f = await fixture(blackDuckBomReport()); + const repository = `${f.destination().url}.git`; + const queries: URLSearchParams[] = []; + const prepared = await prepareExternalPublication( + f.file, + { + ...options, + repository, + provider: "blackduck", + sourceRepositoryUrl: f.destination().url, + }, + { + ...f.deps, + fetch: async (input, init) => { + const url = new URL(input); + if (!url.pathname.endsWith("/repositories")) + return f.deps.fetch(input, init); + queries.push(url.searchParams); + const first = queries.length === 1; + return Response.json( + { + object: "page", + data: first ? [] : [f.destination()], + has_more: first, + next: first ? "next-page" : null, + }, + { + headers: first + ? { + "Codex-Security-Private-Import-Job-Version": "3", + "Codex-Security-Private-Import-Providers": "blackduck", + } + : {}, + }, + ); }, - ], - report, - }; -} + }, + ); + expect(prepared.preview.privateImport).toBeUndefined(); + expect(prepared.preview.destination.id).toBe(options.repository); + expect(queries.map((query) => query.get("repository"))).toEqual([ + repository, + repository, + ]); + expect(queries.map((query) => query.get("page"))).toEqual([ + null, + "next-page", + ]); + expect(f.posts).toHaveLength(0); +}); -type PublicationFixture = Awaited>; +test("repository URLs and IDs share the same resumable destination", async () => { + const f = await fixture(); + f.state.loseResponse = true; + const first = await prepareExternalPublication( + f.file, + { ...options, repository: `${f.destination().url}.git` }, + f.deps, + ); + expect(first.preview.destination.id).toBe(options.repository); + await expect(first.publish()).rejects.toThrow("resume the saved request"); + const retry = await prepareExternalPublication(f.file, options, f.deps); + expect(retry.preview.resumed).toBe(true); + expect((await retry.publish()).status).toBe("complete"); + expect(f.posts[1]).toBe(f.posts[0]); +}); -// Historical filenames include the deployment. Use the one recorded in these fixtures. -function prepareReleased( - f: PublicationFixture, - input: ExternalFindings, - selected: NormalizedExternalPublicationOptions = options, -) { - return prepareNormalizedExternalPublication(input, selected, { +test("Cloud deployment routing covers every request and isolates saved retries", async () => { + const f = await fixture(); + const cloudBase = "https://cloud.example.test/pilot/backend-api/aardvark"; + const requested: string[] = []; + const deps = { + ...f.deps, + environment: { + ...f.deps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: `${cloudBase}///`, + }, + fetch: async (url: string, init: RequestInit) => { + requested.push(url); + expect(url.startsWith(`${cloudBase}/external/`)).toBe(true); + return f.deps.fetch(url, init); + }, + }; + f.state.brokenReadback = true; + const prepared = await prepareExternalPublication(f.file, options, deps); + expect(prepared.preview.cloudApiUrl).toBe(`${cloudBase}/external`); + await expect(prepared.publish()).rejects.toThrow("resume the saved request"); + const otherDeployment = await prepareExternalPublication(f.file, options, { ...f.deps, environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: undefined, - CODEX_SECURITY_CLOUD_WEB_URL: undefined, + ...f.deps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: + "https://other-cloud.example.test/backend-api/aardvark", }, }); -} + expect(otherDeployment.preview.resumed).toBe(false); + const retry = await prepareExternalPublication(f.file, options, deps); + expect(retry.preview.resumed).toBe(true); + expect(retry.preview.requests).toEqual(prepared.preview.requests); + f.state.brokenReadback = false; + const result = await retry.publish(); + expect(result.cloudApiUrl).toBe(`${cloudBase}/external`); + expect(result.cloudUrl).toBeUndefined(); + expect(result.verified).toBe(1); + expect(f.posts).toHaveLength(1); + expect(requested.some((url) => url.includes("/repositories?"))).toBe(true); + expect(requested.some((url) => url.includes("/source_reports?"))).toBe(true); + expect(requested.some((url) => url.endsWith("/finding_imports"))).toBe(true); + expect(requested.some((url) => url.includes("/source_reports/"))).toBe(true); + expect(requested.some((url) => url.includes("/external/v2/"))).toBe(false); +}); -async function stageReleasedCheckpoint( - f: PublicationFixture, - { - filename, - requests, - inputDigest, - savedEvidenceDigest = inputDigest, - accountId = boxedNumberCheckpoints.accountId, - }: { - filename: string; - requests: readonly unknown[]; - inputDigest: string; - savedEvidenceDigest?: string; - accountId?: string; +test.each(["localhost", "127.0.0.1", "127.0.0.2", "[::1]"])( + "loopback Cloud APIs support local HTTP publication: %s", + async (host) => { + const f = await fixture(); + const cloudBase = `http://${host}:4200/backend-api/aardvark`; + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + environment: { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: cloudBase, + }, + fetch: (url, init) => { + expect(url.startsWith(`${cloudBase}/external/`)).toBe(true); + return f.deps.fetch(url, init); + }, + }); + expect((await prepared.publish()).status).toBe("complete"); + expect(f.posts).toHaveLength(1); }, -) { - const directory = join(f.root, "state", "external-finding-publications"); - const path = join(directory, filename); - const checkpoint = { - accountId, - requests, - inputProjection: { inputDigest, savedEvidenceDigest }, - }; - const saved = JSON.stringify(checkpoint); - await mkdir(directory, { recursive: true, mode: 0o700 }); - await writeFile(path, saved, { mode: 0o600 }); - return { checkpoint, directory, path, saved }; -} +); -// These filenames, request IDs and projections were recorded by the old SDK. -async function oldScoreApproval(field: string, stageApproval = true) { +test("a separate Cloud app URL follows the resolved repository without changing API routing", async () => { const f = await fixture(); - const { filename, requestId, inputProjection } = - boxedNumberCheckpoints.cases.find((item) => item.field === field)!; - const prepare = ( - score: unknown, - report?: Record, - flag?: unknown, - ) => { - const input = scoredFindings(score, report, field === "aliased"); - if (flag !== undefined) - Object.assign(input.findings[0]!.evidence.source_data, { flag }); - return prepareReleased(f, input); - }; - const request = { - request_id: requestId, - ...structuredClone(boxedNumberCheckpoints.request), + const requested: string[] = []; + const environment = { + ...f.environment, + CODEX_SECURITY_CLOUD_BASE_URL: + "https://api.example.test/backend-api/aardvark", + CODEX_SECURITY_CLOUD_WEB_URL: + "http://localhost:4200/security?preview=true#/old", }; - if (field === "flag") { - request.items[0]!.expected_version = 0; - Object.assign(request.items[0]!.evidence.source_data, { - score: 12, - flag: {}, - }); - } else { - if (field === "empty-report") - request.items[0]!.evidence.source_data.score = 12; - await (await prepare(0)).publish(); // The old checkpoint approved an update. - f.posts.length = 0; - } - const directory = join(f.root, "state", "external-finding-publications"); - const path = join(directory, filename); - const saved = JSON.stringify({ - accountId: boxedNumberCheckpoints.accountId, - requests: [request], - inputProjection, - }); - if (stageApproval) { - await mkdir(directory, { recursive: true, mode: 0o700 }); - await writeFile(path, saved, { mode: 0o600 }); - } - return { f, prepare, directory, path, filename, request, saved }; -} + const prepared = await prepareExternalPublication( + f.file, + { ...options, repository: f.destination().url }, + { + ...f.deps, + environment, + fetch: (url, init) => { + requested.push(url); + return f.deps.fetch(url, init); + }, + }, + ); + const link = `http://localhost:4200/security?preview=true#/repositories/${options.repository}/findings`; + expect(prepared.preview.cloudUrl).toBe(link); + expect((await prepared.publish()).cloudUrl).toBe(link); + expect( + requested.every((url) => + url.startsWith(environment.CODEX_SECURITY_CLOUD_BASE_URL), + ), + ).toBe(true); -test.each(["invalid title", "unsafe number", "cross-realm serializer error"])( - "a fresh %s can only publish its verified older request", - async (kind) => { - const { f, path, saved, request } = await oldScoreApproval( - "evidence", - false, - ); - const score = new Number(2.5); - const selected = scoredFindings(score); - if (kind === "cross-realm serializer error") { - Object.defineProperty(score, "toJSON", { - value: runInNewContext( - '() => { throw new TypeError("invalid metric") }', - ), - }); - } else if (kind === "unsafe number") { - Object.defineProperty(score, "toJSON", { value: () => 2 ** 53 }); - } else { - Object.setPrototypeOf(selected.findings[0]!.evidence, { - toJSON(this: Record) { - return { ...this, title: "" }; - }, - }); - } - const prepare = () => prepareReleased(f, selected); - await expect(prepare()).rejects.toThrow(); - expect(f.posts).toHaveLength(0); - await writeFile(path, saved, { mode: 0o600 }); - const recovered = await prepare(); - expect(recovered.preview.resumed).toBe(true); - await recovered.publish(); - expect(f.posts).toEqual([JSON.stringify(request)]); - }, -); + const cli = createCliTest(main); + expect( + await cli.runCli( + f.command + .filter((value) => !["--format", "json"].includes(value)) + .concat("--yes"), + { ...f.cliDeps, environment }, + ), + ).toBe(0); + expect(cli.stdout.text()).toContain(`View Cloud repository: ${link}`); +}); -test.each(["deep shared data", "cycle", "row BigInt"])( - "a compact export remains publishable with an unusable native %s", - async (kind) => { +test.each([ + "", + "/relative", + "file:///tmp/cloud", + "https://user:password@example.test", +])( + "invalid Cloud app URL fails before authentication or network: %s", + async (url) => { const f = await fixture(); - const input = scoredFindings(4); - if (kind === "row BigInt") { - Object.assign(input.findings[0]!, { - cursor: Object.assign(Object(17n), { toJSON: () => "17" }), - }); - } else { - class Compact { - constructor(readonly content: unknown) {} - toJSON() { - return 4; - } - } - let content: unknown = {}; - if (kind === "cycle") Object.assign(content as object, { self: content }); - else for (let n = 0; n < 24; n++) content = [content, content]; - input.findings[0]!.evidence.source_data.score = new Compact(content); - } - const preview = await prepareReleased(f, input); - expect(preview.preview.resumed).toBe(false); - await preview.publish(); - expect(JSON.parse(f.posts[0]!).items[0].evidence.source_data.score).toBe(4); + let calls = 0; + await expect( + prepareExternalPublication(f.file, options, { + ...f.deps, + environment: { ...f.environment, CODEX_SECURITY_CLOUD_WEB_URL: url }, + credentials: async () => { + calls++; + throw new Error("Unexpected auth"); + }, + fetch: async () => { + calls++; + throw new Error("Unexpected network"); + }, + }), + ).rejects.toThrow("CODEX_SECURITY_CLOUD_WEB_URL"); + expect(calls).toBe(0); }, ); -test.each(["getter", "data property"])( - "a JSON hook cannot replace a %s score to match a released approval", - async (kind) => { - const { f, path, saved } = await oldScoreApproval("evidence"); - const input = scoredFindings(undefined); - const data = input.findings[0]!.evidence.source_data; - const score = Object.assign(new Number(8), { - toJSON() { - Object.defineProperty(data, "score", { value: new Number(2.5) }); - return 9; - }, - }); - if (kind === "getter") - Object.defineProperty(data, "score", { - configurable: true, - enumerable: true, - get: () => score, - }); - else data.score = score; - - await expect(prepareReleased(f, input)).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toHaveLength(0); +test.each([ + "", + " ", + "/relative", + "file:///tmp/cloud", + "https://user:password@cloud.example.test", + "https://cloud.example.test?deployment=pilot", + "https://cloud.example.test#pilot", + "http://cloud.example.test/backend-api/aardvark", + "http://127.example.test/backend-api/aardvark", + "http://192.0.2.1/backend-api/aardvark", +])( + "invalid Cloud base URL fails before authentication or network: %s", + async (baseUrl) => { + const f = await fixture(); + let calls = 0; + await expect( + prepareExternalPublication(f.file, options, { + ...f.deps, + environment: { + ...f.deps.environment, + CODEX_SECURITY_CLOUD_BASE_URL: baseUrl, + }, + credentials: async () => { + calls++; + throw new Error("Unexpected credential read"); + }, + fetch: async () => { + calls++; + throw new Error("Unexpected network request"); + }, + }), + ).rejects.toThrow("CODEX_SECURITY_CLOUD_BASE_URL"); + expect(calls).toBe(0); }, ); -test("a shared mutation across two GitHub alerts preserves their released approval", async () => { - const f = await fixture(); - const prepare = () => { - const counter = { n: new Number(0) }; - const first = { - get sample() { - counter.n = new Number(Number(counter.n) + 1); - return counter.n; - }, - counter, - }; - class Metrics { - score = Object.assign(new Number(7), { toJSON: () => 9 }); - counter = counter; - } - const rows = [first, new Metrics()].map((extension, i) => { - const row = importedAlert(); - row.number = row.alert.number = 12 + i; - row.url = row.alert.html_url = row.url.replace(/12$/, String(row.number)); - Object.assign(row.alert, { extension }); - return row; - }); - return prepareReleased(f, normalizeGitHubCodeScanningAlerts(rows), { - ...options, - provider: "github", - }); - }; - - const request = JSON.parse( - JSON.stringify((await prepare()).preview.requests[0]), - ); - const first = request.items[0].evidence.source_data.alert.extension; - const second = request.items[1].evidence.source_data.alert.extension; - expect([first.counter.n, second.counter.n, second.score]).toEqual([1, 1, 9]); - // The released publisher captured both alerts after three getter reads. - first.sample = first.counter.n = second.counter.n = 3; - second.score = 7; - request.request_id = "035de7b2-7e71-4166-be9a-99ae1251217b"; - expect(hash("sha256", JSON.stringify(request))).toBe( - "8ce6e5a6d4be2062a39ba7325554f7849096b472a67ae1672fd614c681db9a37", +test("read lookups and verification run concurrently with bounded requests and ordered input", async () => { + const f = await fixture( + Array.from({ length: 9 }, (_, i) => normalized(`vendor-${i}`)), ); - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "a7b6eecd18536c47e5bcd5e1db25a45b3faca1715eab39e2884b06bf67b1e62f.pending.json", - requests: [request], - inputDigest: - "1fa6cbfd77fccdeec029930585f46595118e9f95b8083480239ac50176fb9a52", + const phases: ExternalPublicationProgress[] = []; + let active = 0; + let maximum = 0; + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + onProgress: (event) => phases.push(event), + fetch: async (url, init) => { + if (!url.includes("/source_reports")) return f.deps.fetch(url, init); + active++; + maximum = Math.max(maximum, active); + try { + await Promise.resolve(); + return await f.deps.fetch(url, init); + } finally { + active--; + } + }, }); - - await expect(prepare()).rejects.toThrow("cannot be matched"); - expect(f.posts).toHaveLength(0); - expect(await readFile(path, "utf8")).toBe(saved); + expect( + prepared.preview.requests.flatMap((batch) => + batch.items.map((item) => item.source_finding_id), + ), + ).toEqual(Array.from({ length: 9 }, (_, i) => `vendor-${i}`)); + expect((await prepared.publish()).status).toBe("complete"); + expect(maximum).toBeGreaterThan(1); + expect(maximum).toBeLessThanOrEqual(4); + expect(active).toBe(0); + for (const phase of ["preparing", "uploading", "verifying"]) + expect( + phases.filter((event) => event.phase === phase).at(-1), + ).toMatchObject({ completed: 9, total: 9 }); }); -test("a Sonar report resumes its released request after a size change", async () => { - const { normalizeSonarFindings: normalize } = - await import("../src/sonar-findings.js"); - const repository = "https://github.com/example/project"; - const issue = { - key: "issue-one", - project: "example", - component: "example:one", - rule: "example:S1", - message: "Synthetic finding", - type: "VULNERABILITY", - }; - const boundary = { - ...issue, - key: "issue-two", - score: new Number(0.00001), - pad: "", - }; - const raw = { - paging: { pageIndex: 1, pageSize: 10, total: 2 }, - issues: [issue, boundary], - }; - const base = normalize(raw, repository); - boundary.pad = "p".repeat( - 262_145 - Buffer.byteLength(JSON.stringify(base.findings[1]!.evidence)), - ); - const selection = normalize(raw, repository); +test("progress is optional and stderr progress never corrupts JSON results", async () => { const f = await fixture(); - const selected = { ...options, provider: "sonar" as const }; - const prepare = () => prepareReleased(f, selection, selected); - const fresh = await prepare(); - expect(fresh.preview.findings.map((row) => row.source_finding_id)).toEqual([ - "issue-one", - "issue-two", - ]); - // The earlier SDK approved only the first issue and retained the Sonar report. - const batch = fresh.preview.requests[0]!; - const request = { - ...batch, - request_id: "bd9002e4-5abe-469b-ad2d-a64514f82e1b", - items: [batch.items[0]!], - }; - expect(hash("sha256", JSON.stringify(request))).toBe( - "e6bc41119d363155e2d2388985ce3b476dfd3c6898297b5154a9b09ab192ac95", - ); - await stageReleasedCheckpoint(f, { - filename: - "bb2529dff62d8c9bb1c064d18f2b649c412b3d310914a8d4221c115a00f71e4c.pending.json", - requests: [request], - inputDigest: - "62f0d447578728d3c8afa0ebfa0d1a2d8d7ab5c9d1a96af2855508bc5e6da986", - savedEvidenceDigest: - "25716f8958e0b38376fe5fdaad85b648ca3b0ed23a86c37a246a1537a75c848d", + const phases = new Set(); + const prepared = await prepareExternalPublication(f.file, options, { + ...f.deps, + onProgress: ({ phase }) => { + phases.add(phase); + if (phase === "reading") + return Promise.reject(new Error("Optional async observer failed")); + throw new Error("Optional observer failed"); + }, }); - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.report).toEqual(selection.report); - expect(retry.preview.findings.map((row) => row.source_finding_id)).toEqual([ - "issue-one", + expect((await prepared.publish()).status).toBe("complete"); + expect([...phases]).toEqual([ + "reading", + "discovering", + "preparing", + "waiting", + "uploading", + "verifying", ]); - expect((await retry.publish()).verified).toBe(1); - expect(f.posts).toEqual([JSON.stringify(request)]); -}); - -test("an uncertain newly eligible advisory cannot replace a released subset", async () => { - const f = await fixture(); - const selected = { ...options, provider: "github" as const }; - const prepare = (rows: ReturnType[]) => - prepareReleased(f, normalizeGitHubAdvisories(rows), selected); - const first = importedAdvisory(); - const id = "GHSA-abcd-efgh-5678"; - const url = first.url.replace(first.ghsaId, id); - const second = { - ...first, - ghsaId: id, - url, - advisory: { ...first.advisory, ghsa_id: id, html_url: url }, - }; - const value = Object.assign(new Number(7), { bad: Infinity }); - Object.defineProperty(second.advisory, "score", { - enumerable: true, - get: () => value, - }); - expect(normalizeGitHubAdvisories([first, second]).findings).toHaveLength(2); - - const request = { - ...(await prepare([first])).preview.requests[0]!, - request_id: "58b468b5-45c6-4804-8f5a-5e2919c967c5", - }; - expect(hash("sha256", JSON.stringify(request))).toBe( - "19039224ccc25a9375739c7f16227a26e9fd92cf900de1a32425c20c94697488", - ); - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "6137d904632ecb17ecdf0276e37794699ff19f6e37f6a89b833b56ec0a7c953b.pending.json", - requests: [request], - inputDigest: - "fbba527f5ff2cf86972d97f543de935515cbaa618ab2f1168ddbb3d1ab3ea338", - }); - - await expect(prepare([first, second])).rejects.toThrow("cannot be matched"); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); - expect((await (await prepare([first])).publish()).receipts[0]!.id).toBe( - request.request_id, - ); - expect(f.posts).toEqual([JSON.stringify(request)]); -}); - -test("an ambiguous getter cannot start another import after a released approval was matched", async () => { - const { f, prepare, path, request } = await oldScoreApproval("evidence"); - const score = Object.assign(new Number(2.5), { toJSON: () => 9 }); - f.state.throttle = true; - await expect((await prepare(score)).publish()).rejects.toThrow("Slow down"); - const pending = await readFile(path, "utf8"); - f.state.throttle = false; - - const input = scoredFindings(undefined); - Object.defineProperty(input.findings[0]!.evidence.source_data, "score", { - enumerable: true, - get: () => score, - }); - const publish = async () => (await prepareReleased(f, input)).publish(); - await expect(publish()).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(pending); - expect(f.posts).toEqual([JSON.stringify(request)]); - - const retry = await prepare(score); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request), JSON.stringify(request)]); -}); - -test("a fresh boxed approval cannot claim a different getter-backed JSON import", async () => { - const { f, prepare } = await oldScoreApproval("evidence", false); - f.state.throttle = true; - const first = await prepare(new Number(1)); - await expect(first.publish()).rejects.toThrow("Slow down"); - const input = scoredFindings(undefined); - Object.defineProperty(input.findings[0]!.evidence.source_data, "score", { - enumerable: true, - get: () => 2, - }); - const second = await prepareReleased(f, input); - expect(second.preview.resumed).toBe(false); - expect(second.preview.requests[0]!.request_id).not.toBe( - first.preview.requests[0]!.request_id, - ); - expect(second.preview.requests[0]!.items[0]!.evidence.source_data).toEqual({ - score: 2, - }); - expect(f.posts).toEqual([JSON.stringify(first.preview.requests[0])]); -}); - -test("a root serializer cannot add a report to distinguish a released approval", async () => { - const { f, request } = await oldScoreApproval("unreported", false); - const { source_finding_id, evidence } = scoredFindings(2.5).findings[0]!; - class ScannerBatch { - read = 1; - excluded = []; - findings = [{ source_finding_id, evidence }]; - reportForJson() { - return { version: "1" }; - } - toJSON() { - return { ...this, report: this.reportForJson() }; - } - } - const input = new ScannerBatch(); - const prepare = () => prepareReleased(f, input); - const held = await prepare(); - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "29fc89c807be65a8b3a173feeb64e86595e3debbdb91dc00b1a9cfb6eea869cc.pending.json", - requests: [ - { ...request, request_id: "7e77248b-2906-44a7-b467-5225966ac7a2" }, - ], - inputDigest: - "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc", - }); - await expect(prepare()).rejects.toThrow("cannot be matched"); - await expect(held.publish()).rejects.toThrow("cannot be matched"); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); - const independent = await prepareReleased(f, { - ...input, - report: input.reportForJson(), + const cli = createCliTest(main, { stderr: true }); + expect(await cli.runCli([...f.command, "--yes"], f.cliDeps)).toBe(0); + expect(JSON.parse(cli.stdout.text())).toMatchObject({ + status: "complete", + counts: { unchanged: 1 }, }); - expect(independent.preview.resumed).toBe(false); + expect(cli.stderr.text()).toContain("Preparing findings"); + expect(cli.stderr.text()).toContain("Verifying source records: 1/1"); }); -test("a finding serializer cannot replace the ID of an older approval", async () => { - const { f, path, saved } = await oldScoreApproval("unreported"); - const input = scoredFindings(2.5); - const stored = input.findings[0]!; - const prepare = () => prepareReleased(f, input); - class ScannerFinding { - exportedId() { - return "exported-id"; - } - toJSON() { - return { ...stored, source_finding_id: this.exportedId() }; - } - } - const finding = Object.assign(new ScannerFinding(), stored); - input.findings[0] = finding; - await expect(prepare()).rejects.toThrow("cannot be matched"); - let idReads = 0; - Object.defineProperty(finding, "source_finding_id", { - enumerable: true, - get() { - idReads++; - return stored.source_finding_id; +test("human previews stay compact while JSON previews retain complete evidence", async () => { + const records = Array.from({ length: 6 }, (_, i) => ({ + ...normalized(`vendor-${i}`), + evidence: { + ...normalized().evidence, + source_data: { detail: `complete-vendor-evidence-${i}` }, }, - }); - await expect(prepare()).rejects.toThrow("cannot be matched"); - expect(idReads).toBe(0); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); - - input.findings[0] = { ...stored, source_finding_id: "independent-id" }; - Object.defineProperty(input.findings[0].evidence.source_data, "score", { - enumerable: true, - get: () => 2.5, - }); - expect((await prepare()).preview.resumed).toBe(false); + })); + const f = await fixture(records); + const human = createCliTest(main); + expect( + await human.runCli([...f.command.slice(0, -2), "--dry-run"], f.cliDeps), + ).toBe(0); + expect(human.stderr.text()).toContain("Showing 5 of 6 findings"); + expect(human.stderr.text()).not.toContain("complete-vendor-evidence-"); + expect(human.stdout.text()).not.toContain("request_id"); + const json = createCliTest(main); + expect(await json.runCli([...f.command, "--dry-run"], f.cliDeps)).toBe(0); + expect(JSON.parse(json.stdout.text()).findings).toHaveLength(6); + expect( + JSON.parse(json.stdout.text()).findings[5].evidence.source_data.detail, + ).toBe("complete-vendor-evidence-5"); + expect(f.posts).toHaveLength(0); }); -test("a vendor evidence class resumes its released title and exports its JSON title for a fresh import", async () => { - class ScannerEvidence { - title = "Scanner score"; - severity = "high" as const; - source_data = { score: 2.5 }; - toJSON() { - return { ...this, title: "New scanner score" }; - } - } - const { f, path, request, saved } = await oldScoreApproval("unreported"); - const input = scoredFindings(2.5); - input.findings[0]!.evidence = new ScannerEvidence(); - const prepare = () => prepareReleased(f, input); - // Without an independent native capture, the exported title cannot rule out this approval. - class UncertainEvidence extends ScannerEvidence { - exportedTitle() { - return "Separate scanner title"; - } - override toJSON() { - return { ...this, title: this.exportedTitle() }; - } - } - input.findings[0]!.evidence = new UncertainEvidence(); - await expect(prepare()).rejects.toThrow("cannot be matched"); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); - - input.findings[0]!.evidence = { - ...new ScannerEvidence(), - title: "Separate scanner title", +test("normalized SDK publication preserves caller evidence and exclusion accounting", async () => { + const f = await fixture(); + const input = { + read: 2, + findings: [normalized()], + excluded: [ + { + position: 2, + source_finding_id: null, + reason: "Unsupported source record.", + }, + ], }; - expect((await prepare()).preview.resumed).toBe(false); - input.findings[0]!.evidence = new ScannerEvidence(); - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request)]); - - const fresh = await prepare(); - expect(fresh.preview.resumed).toBe(false); - expect((await fresh.publish()).verified).toBe(1); - expect(JSON.parse(f.posts[1]!).items[0].evidence.title).toBe( - "New scanner score", + const original = structuredClone(input); + const prepared = await prepareNormalizedExternalPublication( + input as Parameters[0], + Object.create(options) as typeof options, + f.deps, ); + expect(input).toEqual(original); + expect(prepared.preview.read).toBe(2); + const result = await prepared.publish(); + expect(result.status).toBe("partial"); + expect(result.counts.created).toBe(1); + expect(result.excluded).toEqual(input.excluded); + expect(result.verified).toBe(1); }); -test("a shared rule in report metadata resumes the released request", async () => { - const f = await fixture(); - const rule = { name: "example-rule" }; - const input = scoredFindings(new Number(2.5), { rule }); - Object.assign(input.findings[0]!.evidence, { - details: { kind: "sast", repository: { url: f.destination().url }, rule }, - }); - const prepare = () => prepareReleased(f, input); - const request = { - ...(await prepare()).preview.requests[0]!, - request_id: "0697e266-e0a5-4aa5-b30b-3b89cbd48ac1", +test("GitHub publication previews, imports, re-imports, updates and resumes selected-ref evidence", async () => { + const imported = importedAlert(); + const f = await fixture([imported]); + const githubOptions = { + ...options, + provider: "github" as const, + sourceKey: "github.com/example/project/code-scanning", }; - expect(hash("sha256", JSON.stringify(request))).toBe( - "8c76780e8aca7582a293dee9483002401d6907e51833fc806cd1a058fe54b855", + const command = f.command.map((arg) => + arg === "wiz" + ? "github" + : arg === options.sourceKey + ? githubOptions.sourceKey + : arg, ); - await stageReleasedCheckpoint(f, { - filename: - "749e46bc07036abe74f047011264147993bd96f384099bcf1b729c8c2634a1b0.pending.json", - requests: [request], - inputDigest: - "49cf7aa8362d8967c5638238b719d925e7e1814d48a4a7a0844e69ef9260b869", - savedEvidenceDigest: - "a2d3adb20cf97c597828fd54ec361490a8a34e479a42d01d3b85ff245b0b384b", - }); - - const resumed = await prepare(); - expect(resumed.preview.resumed).toBe(true); - expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request)]); -}); - -test("a boxed JSON hook keeps the released request and applies to fresh normalized publication", async () => { - const { f, prepare, path, request, saved } = - await oldScoreApproval("evidence"); - const score = (native: number) => - Object.assign(new Number(native), { toJSON: () => 9 }); - - await expect(prepare(score(8))).rejects.toThrow(); - for (const plain of [2.5, JSON.parse(JSON.stringify(score(2.5))), "7.00"]) - await expect(prepare(plain)).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(saved); + const cli = createCliTest(main); + expect( + await cli.runCli([...command, "--dry-run", "--format", "json"], f.cliDeps), + ).toBe(0); expect(f.posts).toHaveLength(0); - - f.state.brokenReadback = true; - const migrated = await prepare(new Number(2.5)); - await expect(migrated.publish()).rejects.toThrow("Readback unavailable"); - f.state.brokenReadback = false; - const resumed = await prepare(score(2.5)); - expect(resumed.preview.resumed).toBe(true); - expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request)]); - - const ordinary = await prepare(new Number(2.5)); - f.state.brokenReadback = true; - await expect(ordinary.publish()).rejects.toThrow("Readback unavailable"); - const pending = await readFile(path, "utf8"); - await expect(prepare(score(2.5))).rejects.toThrow("Finish the other pending"); - expect(await readFile(path, "utf8")).toBe(pending); - f.state.brokenReadback = false; - expect((await (await prepare(2.5)).publish()).receipts[0]!.id).toBe( - ordinary.preview.requests[0]!.request_id, + expect(JSON.parse(cli.stdout.text()).source.provider).toBe("github"); + const first = await ( + await prepareExternalPublication(f.file, githubOptions, f.deps) + ).publish(); + expect(first.counts.created).toBe(1); + expect(first.verified).toBe(1); + const repeated = await ( + await prepareExternalPublication(f.file, githubOptions, f.deps) + ).publish(); + expect(repeated.counts.unchanged).toBe(1); + imported.alert.most_recent_instance.state = "fixed"; + await writeFile(f.file, JSON.stringify([imported])); + f.state.loseResponse = true; + await expect( + (await prepareExternalPublication(f.file, githubOptions, f.deps)).publish(), + ).rejects.toBeInstanceOf(ExternalPublicationError); + const prepared = await prepareExternalPublication( + f.file, + githubOptions, + f.deps, ); - expect(f.posts).toEqual([ - JSON.stringify(request), - JSON.stringify(ordinary.preview.requests[0]), - ]); - const fresh = await prepare(score(2.5)); - expect(fresh.preview.resumed).toBe(false); - expect((await fresh.publish()).verified).toBe(1); - expect(JSON.parse(f.posts[2]!).items[0].evidence.source_data).toEqual({ - score: 9, + expect(prepared.preview.resumed).toBe(true); + const resumed = await prepared.publish(); + expect(resumed.counts.updated).toBe(1); + expect(resumed.verified).toBe(1); + expect([...f.reports.values()][0]).toMatchObject({ + source: { provider: "github" }, + evidence: { details: { source_status: "fixed" }, source_data: imported }, + assessment: { state: "not_assessed" }, }); + await writeFile(f.file, JSON.stringify([importedAlert("refs/heads/review")])); + const differentRef = await ( + await prepareExternalPublication(f.file, githubOptions, f.deps) + ).publish(); + expect(differentRef.counts.created).toBe(1); + expect(f.reports.size).toBe(2); }); test.each([ - { field: "evidence", native: 2.5, report: undefined }, - { field: "empty-report", native: 12, report: {} }, -])( - "a released $field approval stays on its original deployment", - async ({ field, native, report }) => { - const { f, path, saved, request } = await oldScoreApproval(field); - const input = scoredFindings( - Object.assign(new Number(native), { toJSON: () => 9 }), - report, - ); - if (report) { - class SourceData { - score = new Number(native); - toJSON() { - return { score: 9 }; - } - } - input.findings[0]!.evidence.source_data = new SourceData(); - } - const other = await prepareNormalizedExternalPublication(input, options, { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: "https://other.example.test/backend-api", - CODEX_SECURITY_CLOUD_WEB_URL: "https://other.example.test", - }, - }); - expect(other.preview.resumed).toBe(false); - expect(other.preview.requests[0]!.request_id).not.toBe(request.request_id); - expect(other.preview.requests[0]!.items[0]!.evidence.source_data).toEqual({ - score: 9, - }); - expect(f.posts).toHaveLength(0); - expect(await readFile(path, "utf8")).toBe(saved); - }, -); - -test("an opaque retry cannot choose between independent old and new approvals", async () => { - const { - f, - path: oldPath, - saved, - directory, - } = await oldScoreApproval("unreported", false); - const exported = scoredFindings(9); - exported.findings[0]!.source_finding_id = "exported-row"; - const opaque = scoredFindings(2.5); - const nativeRows = opaque.findings; - Object.defineProperty(opaque, "findings", { get: () => nativeRows }); - Object.defineProperty(opaque, "toJSON", { value: () => exported }); - const prepare = (input: typeof opaque) => prepareReleased(f, input); - - const newer = await prepare(exported); - const request = newer.preview.requests[0]!; - const body = JSON.stringify(request); - f.state.throttle = true; - await expect(newer.publish()).rejects.toThrow("Slow down"); - const filename = (await readdir(directory)).find((name) => - name.endsWith(".pending.json"), - )!; - const newPath = join(directory, filename); - const newerBytes = await readFile(newPath, "utf8"); - const held = await prepare(opaque); - expect(held.preview.requests[0]!.request_id).toBe(request.request_id); - f.state.throttle = false; - await writeFile(oldPath, saved, { mode: 0o600 }); - - await expect(prepare(opaque)).rejects.toThrow(); - await expect(held.publish()).rejects.toThrow(); - expect(f.posts).toEqual([body]); - expect(await readFile(oldPath, "utf8")).toBe(saved); - expect(await readFile(newPath, "utf8")).toBe(newerBytes); - const approved = await prepare(exported); - expect((await approved.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([body, body]); - expect(await readFile(oldPath, "utf8")).toBe(saved); -}); - -test.each([false, true])( - "a getter-backed approval stays scoped to its deployment (with report: %s)", - async (reported) => { - const f = await fixture(); - const finding = { - source_finding_id: "synthetic-score", - evidence: { - title: "Score", - severity: "high" as const, - source_data: { - get score() { - return 2.5; - }, - }, - }, - }; - const prepare = (host: string) => + ["code scanning", () => normalizeGitHubCodeScanningAlerts([importedAlert()])], + [ + "repository advisory", + () => normalizeGitHubAdvisories([importedAdvisory()]), + ], +] as const)( + "the retained GitHub %s export binds its SDK projection to the original repository", + async (_, make) => { + const input = make(); + const evidence = input.findings[0]!.evidence; + const f = await fixture(undefined, otherRepository); + const prepare = () => prepareNormalizedExternalPublication( - { - read: 1, - excluded: [], - findings: [finding], - report: reported ? { version: "1" } : undefined, - }, - options, - { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: `https://${host}.example.test/backend-api/aardvark`, - CODEX_SECURITY_CLOUD_WEB_URL: `https://${host}.example.test`, - }, - }, + input, + { ...options, provider: "github" }, + f.deps, ); - // Recorded with the released SDK on deployment one. - const request = { - ...(await prepare("one")).preview.requests[0]!, - request_id: reported - ? "afcc3987-d289-4b85-9e7c-8ff755ebb64b" - : "bf6f1063-9647-43d8-82b9-982961522482", - }; - const digest = - "01ad76cac2893a22136a81e7e61aeef35edb7a51e97f2944713e744756362721"; - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: reported - ? "66fdd41e29ebfe07ca8fe9e54689df0027301c08a19b8b17b82254aadb536b44.pending.json" - : "b4eed57e616f2289bf467f18fba85e2e9ba305a360a798d9e9d8125b06e148fe.pending.json", - requests: [request], - inputDigest: reported - ? "af3bca9d5d1b53284f6da3d32b4de04d196cf14f96a0b7a00711dcd6e90e896d" - : digest, - savedEvidenceDigest: digest, - }); - expect((await prepare("one")).preview.requests).toEqual([request]); - const independent = await prepare("two"); - expect(independent.preview.resumed).toBe(false); - expect(independent.preview.requests[0]!.request_id).not.toBe( - request.request_id, - ); - expect((await independent.publish()).verified).toBe(1); - expect(f.posts).toEqual([JSON.stringify(independent.preview.requests[0])]); - expect(await readFile(path, "utf8")).toBe(saved); - }, -); - -test.each([ - { first: "reported", reportedScore: 2.5 }, - { first: "report-free", reportedScore: 2.5 }, - { first: "reported", reportedScore: 7 }, -])( - "distinct released boxed approvals preserve both selections when $first resumes first (reported score $reportedScore)", - async ({ first, reportedScore }) => { - const { f, prepare, directory, path, request, saved } = - await oldScoreApproval("evidence"); - // The released SDK used this report-bound digest for either boxed score. - const reportedPath = join( - directory, - "ad28704f08feeeaf1adb30d1010204b8ac942fa0c215c5b4eabb4990d338f23a.pending.json", + evidence.details!.repository!.url = otherRepository; + await expect(prepare()).rejects.toThrow( + "does not match the original GitHub finding", ); - const recorded = JSON.parse(saved); - const reportedRequest = structuredClone(request); - reportedRequest.request_id = - reportedScore === 7 - ? "fdfa27bc-13f6-4c83-8abc-fc3619e3b512" - : "ec3a5d47-b29e-493a-90ea-effa70148ef4"; - reportedRequest.items[0]!.evidence.source_data.score = reportedScore; - recorded.requests = [reportedRequest]; - recorded.inputProjection.inputDigest = - "53cc56f227da631bd73c1301c623b9d8c55aca65e03feb324808ae9c886ed1da"; - await writeFile(reportedPath, JSON.stringify(recorded), { mode: 0o600 }); - - const reported = { - report: { version: "1" }, - score: reportedScore, - path: reportedPath, - request: reportedRequest, - }; - const plain = { report: undefined, score: 2.5, path, request }; - const order = first === "reported" ? [reported, plain] : [plain, reported]; - f.state.throttle = true; - for (const selection of order) { - const sibling = selection === reported ? plain : reported; - const siblingBytes = await readFile(sibling.path, "utf8"); - const prepared = await prepare( - new Number(selection.score), - selection.report, - ); - expect(prepared.preview.resumed).toBe(true); - expect(prepared.preview.requests[0]!.request_id).toBe( - selection.request.request_id, - ); - await expect(prepared.publish()).rejects.toThrow("Slow down"); - expect(await readFile(sibling.path, "utf8")).toBe(siblingBytes); - } - expect(f.posts).toEqual( - order.map((selection) => JSON.stringify(selection.request)), + delete evidence.details; + await expect(prepare()).rejects.toThrow( + "does not match the selected Cloud repository", ); - }, -); - -function failCheckpointReads(unreadable: (path: string) => boolean) { - const read = fs.readFile; - return spyOn(fs, "readFile").mockImplementation((async ( - ...args: Parameters - ) => { - if (unreadable(String(args[0]))) - throw Object.assign(new Error("Unreadable checkpoint"), { - code: "EACCES", - }); - return read(...args); - }) as typeof read); -} - -test.each(["released", "fresh"])( - "a primitive retry uses the lookup for a %s boxed approval without reading unrelated checkpoints", - async (kind) => { - const { f, prepare, directory, filename, path, request, saved } = - await oldScoreApproval("evidence", kind === "released"); - const boxed = await prepare(new Number(2.5)); - const approved = boxed.preview.requests[0]!; - if (kind === "released") - expect(approved.request_id).toBe(request.request_id); - f.state.brokenReadback = true; - await expect(boxed.publish()).rejects.toThrow("Readback unavailable"); - const files = await readdir(directory); - expect(files.filter((file) => file.endsWith(".pending.json"))).toEqual([ - filename, - ]); - const checkpoint = JSON.parse(await readFile(path, "utf8")); + expect(f.posts).toHaveLength(0); - const unrelated = join( - directory, - `${hash("sha256", "another-account")}.pending.json`, - ); - await writeFile( - unrelated, - JSON.stringify({ ...JSON.parse(saved), accountId: "another-account" }), - { mode: 0o600 }, - ); - using reads = failCheckpointReads((path) => path === unrelated); - f.state.brokenReadback = false; - const retry = await prepare(2.5); + f.state.repositoryUrl = exampleRepository; + await f.loseResponse(await prepare()); + f.state.repositoryUrl = otherRepository; + const retry = await prepare(); expect(retry.preview.resumed).toBe(true); - expect(JSON.stringify(retry.preview.requests)).toBe( - JSON.stringify(boxed.preview.requests), - ); - expect((await retry.publish()).receipts[0]!.id).toBe(approved.request_id); - expect(f.posts).toEqual([JSON.stringify(approved)]); - expect(files).toContain(`${checkpoint.wireKey}.lookup.json`); - expect(reads.mock.calls.some(([file]) => String(file) === unrelated)).toBe( - false, - ); + expect((await retry.publish()).verified).toBe(1); + expect(f.posts[1]).toBe(f.posts[0]); }, ); -test("the original boxed approval keeps its filename, request and receipts on a primitive retry", async () => { - const { f, prepare, directory, path, filename, request, saved } = - await oldScoreApproval("evidence"); - for (const score of [new Number(9.5), {}]) { - await expect(prepare(score)).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(saved); +test("GitHub imports reject a different Cloud destination before uploading", async () => { + const imported = importedAlert("refs/heads/main", "git.example.test"); + const f = await fixture([imported]); + await expect( + prepareExternalPublication( + f.file, + { ...options, provider: "github" }, + f.deps, + ), + ).rejects.toThrow("does not match the selected Cloud repository"); + expect(f.posts).toHaveLength(0); +}); + +test("global GHSA publication retains unassessed applicability through repeat import and withdrawal", async () => { + const imported = importedAdvisory("global"); + const f = await fixture([imported]); + const selected = { + ...options, + provider: "github" as const, + sourceKey: "github.com/example/project/security-advisories", + }; + expect( + ( + await ( + await prepareExternalPublication(f.file, selected, f.deps) + ).publish() + ).counts.created, + ).toBe(1); + expect( + ( + await ( + await prepareExternalPublication(f.file, selected, f.deps) + ).publish() + ).counts.unchanged, + ).toBe(1); + const withdrawn = { + ...imported, + advisory: { ...imported.advisory, withdrawn_at: "2026-01-03T00:00:00Z" }, + }; + await writeFile(f.file, JSON.stringify([withdrawn])); + const result = await ( + await prepareExternalPublication(f.file, selected, f.deps) + ).publish(); + expect(result.counts.updated).toBe(1); + expect(result.verified).toBe(1); + const report = [...f.reports.values()][0]!; + expect(report.evidence.details).toMatchObject({ + kind: "advisory", + advisory: { source: "global" }, + source_status: "withdrawn", + }); + expect(report.evidence.details).not.toHaveProperty("repository"); + expect(report.assessment?.state).toBe("not_assessed"); + expect(report.evidence.packages![0]!.installed_version).toBeNull(); +}); + +test("saved GitHub JSONL and gzip preserve evidence while malformed UTF-8 cannot change identities", async () => { + const imported = [importedAlert(), importedAdvisory("global")]; + const f = await fixture(imported); + const jsonl = imported.map((record) => JSON.stringify(record)).join("\n"); + for (const bytes of [Buffer.from(jsonl), gzipSync(jsonl)]) { + await writeFile(f.file, bytes); + const parsed = await readGitHubFindings(f.file); + expect(parsed.read).toBe(2); + expect(parsed.excluded).toEqual([]); + expect( + parsed.findings.map((finding) => finding.evidence.source_data) as unknown, + ).toEqual(imported); } - expect(f.posts).toEqual([]); - const recovered = await prepare(new Number(2.5)); - expect(recovered.preview.resumed).toBe(true); - expect(JSON.stringify(recovered.preview.requests)).toBe( - JSON.stringify([request]), - ); - f.state.brokenReadback = true; - const interrupted = await recovered - .publish() - .catch((error: unknown) => error); - if (!(interrupted instanceof ExternalPublicationError)) throw interrupted; - expect(interrupted.result.savedSubmission).toBe(path); - const pending = (await readdir(directory)).filter((name) => - name.endsWith(".pending.json"), - ); - expect(pending).toEqual([filename]); - expect(JSON.parse(await readFile(path, "utf8")).receipts[0].id).toBe( - request.request_id, - ); - f.state.brokenReadback = false; - const retry = await prepare(2.5); - const sibling = await prepare(2.5); - const stale = await prepare(2.5); - expect(retry.preview.resumed).toBe(true); - expect(JSON.stringify(retry.preview.requests)).toBe( - JSON.stringify([request]), + await writeFile(f.file, Buffer.from([0xff])); + await expect(readGitHubFindings(f.file)).rejects.toThrow("UTF-8"); +}); + +test("normalized publication omits only new null fields to match canonical Cloud readback", async () => { + const f = await fixture(); + const evidence = { + title: "Synthetic advisory without known package range", + severity: "unknown" as const, + packages: [ + { + name: "example-package", + vulnerable_version_range: null, + installed_version: null, + }, + ], + details: { + kind: "advisory" as const, + repository: null, + advisory: { source: "global" as const }, + }, + }; + const prepared = await prepareNormalizedExternalPublication( + { + read: 1, + findings: [{ source_finding_id: "synthetic-global-advisory", evidence }], + excluded: [], + }, + options, + f.deps, ); - // An older checkpoint at an unknown filename can appear after both retries were prepared. - const unknown = join( - directory, - `${hash("sha256", "other-deployment")}.pending.json`, + const sent = prepared.preview.findings[0]!.evidence; + expect(sent.packages![0]).not.toHaveProperty("vulnerable_version_range"); + expect(sent.packages![0]!.installed_version).toBeNull(); + expect(sent.details).not.toHaveProperty("repository"); + expect(evidence.details.repository).toBeNull(); + expect((await prepared.publish()).verified).toBe(1); + + await writeFile( + f.file, + JSON.stringify([ + { + source_finding_id: "synthetic-sast", + evidence: { + title: "Synthetic SAST evidence", + severity: "high", + packages: [ + { name: "example-package", vulnerable_version_range: null }, + ], + details: { + kind: "sast", + repository: { url: "https://github.com/example/project" }, + advisory: null, + }, + }, + }, + ]), ); - const projectionless = JSON.stringify({ - accountId: boxedNumberCheckpoints.accountId, - requests: [request], - }); - await writeFile(unknown, projectionless, { mode: 0o600 }); - const result = await retry.publish(); - expect(result.verified).toBe(1); - expect(result.receipts.map(({ id }) => id)).toEqual([request.request_id]); - { - using reads = spyOn(fs, "readFile"); - expect(await sibling.publish()).toEqual(result); - expect(reads.mock.calls.map(([path]) => String(path))).not.toContain( - unknown, - ); - } - expect(f.posts).toEqual([JSON.stringify(request)]); - expect(await Bun.file(path).exists()).toBe(false); - expect(await readFile(unknown, "utf8")).toBe(projectionless); - await rm(unknown); - - // A later approval of the same input replaced the completed result. - const fresh = await prepare(2.5); - const replacement = await fresh.publish(); - expect(replacement.receipts[0]!.id).not.toBe(request.request_id); - await expect(stale.publish()).rejects.toThrow(/another publication|review/i); - expect(f.posts).toEqual([ - JSON.stringify(request), - JSON.stringify(fresh.preview.requests[0]), - ]); - expect(await Bun.file(path).exists()).toBe(false); + const wiz = await prepareExternalPublication(f.file, options, f.deps); + expect(wiz.preview.findings[0]!.evidence.details).not.toHaveProperty( + "advisory", + ); + expect(wiz.preview.findings[0]!.evidence.details!.scanner_origin).toBeNull(); + expect((await wiz.publish()).verified).toBe(1); }); -test.each(["released", "current"])( - "a known %s retry preserves its approval when unrelated checkpoints are unreadable", - async (format) => { - const released = format === "released"; - const { f, prepare, directory, request, path, saved } = - await oldScoreApproval("unreported", released); - f.state.throttle = true; - for (const score of [1, 2, 3]) { - await expect((await prepare(score)).publish()).rejects.toThrow( - "Slow down", - ); - } - const unrelated = new Set( - (await readdir(directory)) - .filter((name) => name.endsWith(".pending.json")) - .map((name) => join(directory, name)) - .filter((candidate) => candidate !== path), - ); - expect(unrelated.size).toBe(3); - let requestId = request.request_id; - if (released) { - expect(await readFile(path, "utf8")).toBe(saved); - } else { - const selected = await prepare(2.5); - requestId = selected.preview.requests[0]!.request_id; - await expect(selected.publish()).rejects.toThrow("Slow down"); - } - - let unreadable = false; - const statePaths = new Set([directory, await fs.realpath(directory)]); - using listings = spyOn(fs, "readdir"); - using reads = failCheckpointReads( - (path) => unreadable && unrelated.has(path), - ); - const operations = () => ({ - listed: listings.mock.calls.some(([path]) => - statePaths.has(String(path)), - ), - readUnrelated: reads.mock.calls.some(([path]) => - unrelated.has(String(path)), - ), - }); - if (released) { - const original = await prepare(2.5); - reads.mockClear(); - listings.mockClear(); - await expect(original.publish()).rejects.toThrow("Slow down"); - expect(operations()).toEqual({ listed: true, readUnrelated: true }); - expect(f.posts.at(-1)).toBe(JSON.stringify(request)); - } - unreadable = true; - f.state.throttle = false; - reads.mockClear(); - listings.mockClear(); - const retry = await prepare(2.5); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests[0]!.request_id).toBe(requestId); - expect(operations()).toEqual({ listed: false, readUnrelated: false }); - f.state.brokenReadback = true; - await expect(retry.publish()).rejects.toThrow("Readback unavailable"); - const stored = JSON.parse(await readFile(path, "utf8")); - expect(stored.receipts[0].id).toBe(requestId); - if (released) - expect(f.posts.slice(-2)).toEqual([ - JSON.stringify(request), - JSON.stringify(request), - ]); - const posted = [...f.posts]; - f.state.brokenReadback = false; - expect((await (await prepare(2.5)).publish()).receipts).toEqual( - stored.receipts, - ); - expect(f.posts).toEqual(posted); - expect(operations()).toEqual({ listed: false, readUnrelated: false }); - listings.mockClear(); - reads.mockClear(); - // A fresh selection must discover possible historical approvals. - await expect(prepare(4)).rejects.toThrow("Unreadable checkpoint"); - expect(operations()).toEqual({ listed: true, readUnrelated: true }); - }, -); - -test("a released receipt can finish readback when caching the checkpoint scan fails", async () => { - const { f, prepare, path, saved, request } = - await oldScoreApproval("unreported"); - f.state.brokenReadback = true; - await expect((await prepare(2.5)).publish()).rejects.toThrow( - "Readback unavailable", +test("normalized SDK publication detaches raw evidence aliases before validating every finding", async () => { + const f = await fixture(); + const sharedPackage = { name: "example-package" }; + const input = { + read: 2, + report: { package: sharedPackage }, + findings: [ + { + source_finding_id: "shared-first", + evidence: { + title: "First report", + severity: "high" as const, + packages: [sharedPackage], + source_data: { package: sharedPackage }, + }, + }, + { + source_finding_id: "shared-second", + evidence: { + title: "Second report", + severity: "high" as const, + source_data: { earlier_package: sharedPackage }, + }, + }, + ], + excluded: [], + }; + const original = JSON.stringify(input); + const prepared = await prepareNormalizedExternalPublication( + input, + options, + f.deps, ); - const receipts = [f.receipts.get(request.request_id)!]; - const released = { ...JSON.parse(saved), receipts }; - await writeFile(path, JSON.stringify(released)); - - const rename = fs.rename; - using _writes = spyOn(fs, "rename").mockImplementation(async (from, to) => { - if (String(to) === path) { - const { canonicalChecked, ...pending } = JSON.parse( - await readFile(from, "utf8"), - ); - if (canonicalChecked && Bun.deepEquals(pending, released)) - throw Object.assign(new Error("No space for checkpoint metadata"), { - code: "ENOSPC", - }); - } - return rename(from, to); + expect(JSON.stringify(input)).toBe(original); + expect(prepared.preview.report).toEqual({ + package: { name: "example-package" }, }); - const retry = await prepare(2.5); - expect(retry.preview.resumed).toBe(true); - const interrupted = await retry.publish().catch((error: unknown) => error); - if (!(interrupted instanceof ExternalPublicationError)) throw interrupted; - expect(interrupted.result).toMatchObject({ - status: "interrupted", - counts: { updated: 1 }, - receipts, - unacknowledged: 0, - savedSubmission: path, + expect(prepared.preview.findings[0]!.evidence.source_data).toEqual({ + package: { name: "example-package" }, }); - expect(await readFile(path, "utf8")).toBe(JSON.stringify(released)); - f.state.brokenReadback = false; - const completed = await (await prepare(2.5)).publish(); - expect(completed).toMatchObject({ - status: "complete", - verified: 1, - receipts, + expect(prepared.preview.findings[1]!.evidence.source_data).toEqual({ + earlier_package: { name: "example-package" }, }); - expect(f.posts).toEqual([JSON.stringify(request)]); + await writeFile(f.file, JSON.stringify(input.findings)); + const fromFile = await prepareExternalPublication(f.file, options, f.deps); + expect(prepared.preview.findings).toEqual(fromFile.preview.findings); }); -test("checking a released canonical approval preserves ambiguous boxed report metadata", async () => { - const { f, prepare } = await oldScoreApproval("report"); - f.state.throttle = true; - const older = await prepare(2.5, { elapsed: {} }); - expect(older.preview.resumed).toBe(true); - await expect(older.publish()).rejects.toThrow("Slow down"); - const numeric = await prepare(2.5, { elapsed: 4.5 }); - expect(numeric.preview.resumed).toBe(false); - await expect(numeric.publish()).rejects.toThrow("Slow down"); - await expect(prepare(2.5, { elapsed: new Number(4.5) })).rejects.toThrow( - "older saved import cannot be matched", - ); - expect(f.posts).toEqual([ - JSON.stringify(older.preview.requests[0]), - JSON.stringify(numeric.preview.requests[0]), - ]); -}); +test.each(["code-scanning", "advisory"])( + "%s imports bind an explicit default port to the matching Cloud repository", + async (family) => { + const alert = importedAlert("refs/heads/main", "github.com:443"); + const advisory = importedAdvisory(); + advisory.githubHost = "github.com:443"; + advisory.url = advisory.advisory.html_url = advisory.url.replace( + "github.com", + advisory.githubHost, + ); + const f = await fixture([family === "code-scanning" ? alert : advisory]); + const selected = { ...options, provider: "github" as const }; + await expect( + prepareExternalPublication(f.file, selected, f.deps), + ).rejects.toThrow("does not match the selected Cloud repository"); + expect(f.posts).toHaveLength(0); + const matching = { + ...f.deps, + fetch: async (url: string, init: RequestInit) => { + const response = await f.deps.fetch(url, init); + if (new URL(url).pathname.endsWith("/repositories")) { + const body = await response.json(); + body.data[0].url = "https://github.com:443/example/project"; + return Response.json(body); + } + return response; + }, + }; + const result = await ( + await prepareExternalPublication(f.file, selected, matching) + ).publish(); + expect(result.counts.created).toBe(1); + expect(result.verified).toBe(1); + }, +); -test("a second matching checkpoint preserves the receipts shown at preview", async () => { - const prior = await oldScoreApproval("unreported"); - prior.f.state.brokenReadback = true; - await expect((await prior.prepare(2.5)).publish()).rejects.toThrow( - "Readback unavailable", - ); - const receipts = [prior.f.receipts.get(prior.request.request_id)!]; - const saved = JSON.stringify({ ...JSON.parse(prior.saved), receipts }); - await writeFile(prior.path, saved); - const held = await prior.prepare(2.5); - const competing = await oldScoreApproval("evidence"); - await competing.prepare(new Number(2.5)); - const tagged = await readFile(competing.path, "utf8"); - const otherPath = join(prior.directory, competing.filename); - await writeFile(otherPath, tagged); - - const error = await held.publish().catch((cause: unknown) => cause); - expect(error).toBeInstanceOf(ExternalPublicationError); - if (!(error instanceof ExternalPublicationError)) throw error; - expect(error.message).toContain("Two saved imports match"); - expect(error.result).toMatchObject({ - status: "interrupted", - receipts, - counts: { updated: 1 }, - unacknowledged: 0, - }); - expect(prior.f.posts).toEqual([JSON.stringify(prior.request)]); - expect(await readFile(prior.path, "utf8")).toBe(saved); - expect(await readFile(otherPath, "utf8")).toBe(tagged); -}); +function scoredFindings(score: unknown, report?: Record) { + const evidence = { + title: "Scanner score", + severity: "high" as const, + source_data: { score }, + }; + return { + read: 1, + excluded: [], + findings: [ + { + source_finding_id: "boxed-score", + evidence, + }, + ], + report, + }; +} -test("matching released primitive and boxed approvals preserve both requests", async () => { - const primitive = await oldScoreApproval("unreported"); - const boxed = await oldScoreApproval("evidence", false); - const held = await primitive.prepare(2.5); - const boxedPath = join(primitive.directory, boxed.filename); - await writeFile(boxedPath, boxed.saved, { mode: 0o600 }); - await expect(primitive.prepare(new Number(2.5))).rejects.toThrow( - "Two saved imports match", - ); - expect(primitive.f.posts).toEqual([]); - expect(await readFile(primitive.path, "utf8")).toBe(primitive.saved); - expect(await readFile(boxedPath, "utf8")).toBe(boxed.saved); - - // A later independent canonical approval must still be checked alongside a valid lookup. - await rm(primitive.path); - await primitive.prepare(new Number(2.5)); - const tagged = await readFile(boxedPath, "utf8"); - const lookup = `${JSON.parse(tagged).wireKey}.lookup.json`; - expect(await readdir(primitive.directory)).toContain(lookup); - await writeFile(primitive.path, primitive.saved, { mode: 0o600 }); - for (const hintAvailable of [true, false]) { - // A process can exit after writing the tagged approval and before saving its lookup. - if (!hintAvailable) await rm(join(primitive.directory, lookup)); - for (const score of [2.5, new Number(2.5)]) - await expect(primitive.prepare(score)).rejects.toThrow( - "Two saved imports match", - ); - await expect(held.publish()).rejects.toThrow("Two saved imports match"); - expect(await readFile(primitive.path, "utf8")).toBe(primitive.saved); - expect(await readFile(boxedPath, "utf8")).toBe(tagged); - expect(primitive.f.posts).toEqual([]); - } +type PublicationFixture = Awaited>; - // A partial check must see the second approval once it becomes readable. - { - using _reads = failCheckpointReads((path) => path === boxedPath); - primitive.f.state.throttle = true; - await expect(held.publish()).rejects.toThrow("Slow down"); - } - await expect(primitive.prepare(2.5)).rejects.toThrow( - "Two saved imports match", - ); - expect(await readFile(primitive.path, "utf8")).toBe(primitive.saved); - expect(await readFile(boxedPath, "utf8")).toBe(tagged); - expect(primitive.f.posts).toEqual([JSON.stringify(primitive.request)]); -}); +function prepareNormalizedFixture( + f: PublicationFixture, + input: ExternalFindings, + selected: NormalizedExternalPublicationOptions = options, +) { + return prepareNormalizedExternalPublication(input, selected, f.deps); +} -test.each(["report", "mixed", "aliased"])( - "ambiguous historical %s data preserves the pending approval", - async (field) => { - const { f, prepare, path, saved } = await oldScoreApproval(field); - const retry = - field === "mixed" - ? prepare(new Number(2.5), { elapsed: {} }) - : prepare(2.5, { elapsed: new Number(4.5) }); - await expect(retry).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); +test.each(["deep shared data", "cycle", "row BigInt"])( + "a compact export remains publishable with an unusable native %s", + async (kind) => { + const f = await fixture(); + const input = scoredFindings(4); + if (kind === "row BigInt") { + Object.assign(input.findings[0]!, { + cursor: Object.assign(Object(17n), { toJSON: () => "17" }), + }); + } else { + class Compact { + constructor(readonly content: unknown) {} + toJSON() { + return 4; + } + } + let content: unknown = {}; + if (kind === "cycle") Object.assign(content as object, { self: content }); + else for (let n = 0; n < 24; n++) content = [content, content]; + input.findings[0]!.evidence.source_data.score = new Compact(content); + } + const preview = await prepareNormalizedFixture(f, input); + expect(preview.preview.resumed).toBe(false); + await preview.publish(); + expect(JSON.parse(f.posts[0]!).items[0].evidence.source_data.score).toBe(4); }, ); -test("a new boxed report does not claim an unrelated older approval", async () => { - const { f, prepare, path, saved, request } = - await oldScoreApproval("unreported"); - const fresh = await prepare(2.5, { elapsed: new Number(4.5) }); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.requests[0]!.request_id).not.toBe(request.request_id); - expect(JSON.stringify((await prepare(2.5)).preview.requests)).toBe( - JSON.stringify([request]), - ); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); -}); - -test("an older boxed score with an empty report resumes its original request", async () => { - const { f, prepare, request } = await oldScoreApproval("empty-report"); - const boxed = await prepare(new Number(12), {}); - expect(boxed.preview.resumed).toBe(true); - f.state.throttle = true; - await expect(boxed.publish()).rejects.toThrow("Slow down"); - f.state.throttle = false; - const primitive = await prepare(12, {}); - expect(primitive.preview.resumed).toBe(true); - expect((await primitive.publish()).verified).toBe(1); - const approved = JSON.stringify(request); - expect(f.posts).toEqual([approved, approved]); -}); - test("a boxed false flag remains independent of an approved empty object", async () => { const f = await fixture(); const prepare = (score: unknown, flag: unknown) => { @@ -9358,61 +6248,6 @@ test("a boxed false flag remains independent of an approved empty object", async expect(JSON.stringify(empty.preview.requests[0])).toBe(f.posts[0]!); }); -test("an older empty flag cannot authorize a boxed Boolean", async () => { - const { f, prepare, path, saved, request } = await oldScoreApproval("flag"); - await expect( - prepare(new Number(12), undefined, new Boolean(true)), - ).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); - const original = await prepare(new Number(12), undefined, {}); - expect(original.preview.resumed).toBe(true); - expect((await original.publish()).verified).toBe(1); - expect(f.posts).toEqual([JSON.stringify(request)]); -}); - -test.each(["evidence", "report"])( - "a fresh boxed %s reuses its historical filename until the numeric retry completes", - async (field) => { - const { f, prepare, path } = await oldScoreApproval(field, false); - const select = (value: unknown) => - field === "report" ? prepare(2.5, { elapsed: value }) : prepare(value); - const value = field === "report" ? 4.5 : 2.5; - const ordinary = await select({}); - const competing = await select(new Number(value + 1)); - const boxed = await select(new Number(value)); - f.state.throttle = true; - await expect(boxed.publish()).rejects.toThrow("Slow down"); - const saved = await readFile(path, "utf8"); - const body = f.posts[0]!; - expect(JSON.parse(saved).requests).toEqual([JSON.parse(body)]); - for (const stale of [ordinary, competing]) { - await expect(stale.publish()).rejects.toThrow(/pending|review/i); - } - await expect(select({})).rejects.toThrow(/finish the other pending/i); - expect(f.posts).toEqual([body]); - expect(await readFile(path, "utf8")).toBe(saved); - - const numeric = await select(value); - expect(numeric.preview.resumed).toBe(true); - expect(numeric.preview.requests).toEqual([JSON.parse(body)]); - f.state.throttle = false; - expect((await numeric.publish()).verified).toBe(1); - expect(await Bun.file(path).exists()).toBe(false); - const fresh = await select({}); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.requests[0]!.request_id).not.toBe( - numeric.preview.requests[0]!.request_id, - ); - expect((await fresh.publish()).verified).toBe(1); - expect(f.posts).toEqual([ - body, - body, - JSON.stringify(fresh.preview.requests[0]), - ]); - }, -); - test("normalized SDK publication rejects nonfinite raw evidence without JSON coercion", async () => { const f = await fixture(); const input = { @@ -9458,130 +6293,16 @@ test("normalized SDK publication rejects unserializable report metadata before u options, f.deps, ), - ).rejects.toThrow("finite JSON numbers"); - await expect( - prepareNormalizedExternalPublication( - { ...input, report: { toJSON: () => Infinity } }, - options, - f.deps, - ), - ).rejects.toThrow("finite JSON numbers"); - expect(f.posts).toHaveLength(0); -}); - -test.each([Error, TypeError])( - "a native %p resumes its released message-only approval", - async (Diagnostic) => { - const { f, directory, prepare, request } = await oldScoreApproval( - "evidence", - false, - ); - request.request_id = "15ab6427-635d-4537-9975-f7f536155b49"; - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "19d4eb5e5738ac54ae3d847f50534f09ac60882b0145fa5abc4260299e7f02b6.pending.json", - requests: [request], - inputDigest: - "603ea14278affa7a9a27fc2a624f5a2ce8fd85460f99175bef69a0572430b59b", - savedEvidenceDigest: - "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc", - }); - const diagnostic = () => - Object.assign( - new Diagnostic("scanner message", { cause: "scanner failure" }), - { code: "E_TEST" }, - ); - const exportedDiagnostic = Object.assign(diagnostic(), { - toJSON: () => ({ code: "TIMEOUT" }), - }); - const prepareExported = () => - prepare(2.5, { errors: [exportedDiagnostic] }); - await expect(prepareExported()).rejects.toThrow("cannot be matched"); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toEqual([]); - - const uncertain = diagnostic(); - let stackReads = 0; - Object.defineProperty(uncertain, "stack", { - get() { - stackReads++; - return "custom stack"; - }, - }); - await expect(prepare(2.5, { errors: [uncertain] })).rejects.toThrow( - "cannot be matched", - ); - expect(stackReads).toBe(0); - expect(f.posts).toEqual([]); - expect(await readFile(path, "utf8")).toBe(saved); - - const prepareUnsafeExport = (error: Error, beforeExport?: () => void) => { - const input = scoredFindings(2.5, { errors: [error] }); - Object.setPrototypeOf(input.findings[0]!.evidence.source_data, { - toJSON() { - beforeExport?.(); - return { score: 2 ** 53 }; - }, - }); - return prepareReleased(f, input); - }; - const changed = diagnostic(); - await expect( - prepareUnsafeExport(changed, () => - Object.defineProperty(changed, "stack", { - get() { - stackReads++; - return "custom stack"; - }, - }), - ), - ).rejects.toThrow("safe integer"); - expect(stackReads).toBe(0); - expect(f.posts).toEqual([]); - expect(await readFile(path, "utf8")).toBe(saved); - - const report = { errors: [diagnostic()] }; - expect((await prepare(2.5, report)).preview.resumed).toBe(true); - const retry = await prepareUnsafeExport(report.errors[0]!); - expect(retry.preview.resumed).toBe(true); - const completed = await retry.publish(); - expect(completed.receipts[0]!.id).toBe(request.request_id); - const expected = structuredClone(report.errors[0]!); - for (const value of [retry.preview.report, completed.report]) { - const error = (value?.["errors"] as Error[])[0]!; - expect(error).toBeInstanceOf(Diagnostic); - expect(error.message).toBe(expected.message); - expect(error.stack).toBe(expected.stack); - expect(error.cause).toBe(expected.cause); - expect(error).not.toHaveProperty("code"); - } - const results = await Promise.all( - (await readdir(directory)) - .filter((name) => name.endsWith(".result.json")) - .map( - async (name) => - JSON.parse( - await readFile(join(directory, name), "utf8"), - ) as ExternalPublicationResult, - ), - ); - const receipt = results.find( - (result) => result.receipts[0]?.id === request.request_id, - ); - expect(receipt?.report).toEqual({ - errors: [{ message: "scanner message" }], - }); - expect(f.posts).toEqual([JSON.stringify(request)]); - expect((await prepareExported()).preview.report?.["errors"]).toEqual([ - { code: "TIMEOUT" }, - ]); - const fresh = await prepare(2.5, report); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.report?.["errors"]).toEqual([ - expect.objectContaining({ message: "scanner message", code: "E_TEST" }), - ]); - }, -); + ).rejects.toThrow("finite JSON numbers"); + await expect( + prepareNormalizedExternalPublication( + { ...input, report: { toJSON: () => Infinity } }, + options, + f.deps, + ), + ).rejects.toThrow("finite JSON numbers"); + expect(f.posts).toHaveLength(0); +}); test("publication JSON retains native Error messages and scanner diagnostics", async () => { const scannerErrors = opengrepJson().errors; @@ -10739,37 +7460,6 @@ test("Semgrep publication preserves identity across JSON and SARIF with explicit }); }); -test.each([ - ["semgrep", semgrepJson], - ["opengrep", opengrepJson], -] as const)( - "%s file retries a released request with its original source URL after a rename", - async (provider, scannerOutput) => { - const f = await fixture(scannerOutput()); - const originalUrl = f.destination().url; - const prepare = (sourceRepositoryUrl: string) => - prepareExternalPublication( - f.file, - { ...options, provider, sourceRepositoryUrl }, - f.deps, - ); - const original = await prepare(originalUrl); - const pending = await f.loseResponse(original); - await useReleasedCheckpoint(pending); - f.state.repositoryUrl = "https://github.com/example/renamed"; - - await expect(prepare("https://github.com/other/project")).rejects.toThrow( - /does not match/, - ); - const retry = await prepare(originalUrl); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests).toEqual(original.preview.requests); - await retry.publish(); - expect(f.posts).toEqual([f.posts[0]!, f.posts[0]!]); - expect(f.receipts.size).toBe(1); - }, -); - test("OpenGrep publication retains grouped evidence through Cloud readback and replay", async () => { const input = opengrepTraceJson(); const { results, ...report } = input; @@ -11152,36 +7842,6 @@ test.each(["file", "selected rows"] as const)( }, ); -test("unverified Trivy checkpoints retain their requests and receipts for the original importer", async () => { - const cloud = await trivyCloud(); - const prepare = cloud.fileImport; - cloud.state.brokenReadback = true; - const failure = await ( - await prepare() - ) - .publish() - .catch((error: unknown) => error); - if (!(failure instanceof ExternalPublicationError)) throw failure; - const filename = failure.result.savedSubmission!; - cloud.state.brokenReadback = false; - const retry = await prepare(); - const { accountId, requests, receipts, inputProjection } = JSON.parse( - await readFile(filename, "utf8"), - ); - expect(receipts).toHaveLength(1); - const original = JSON.stringify({ - accountId, - requests, - receipts, - inputProjection, - }); - await writeFile(filename, original); - await expect(prepare()).rejects.toThrow("the build that created it"); - await expect(retry.publish()).rejects.toThrow(); - expect(await readFile(filename, "utf8")).toBe(original); - expect(cloud.posts).toHaveLength(1); -}); - test("Endor source repository binding preserves immutable requests across a repository rename", async () => { const input: any = endorSarif(); input.runs[0].versionControlProvenance = [ @@ -11211,715 +7871,109 @@ test("Endor source repository binding preserves immutable requests across a repo const retry = await prepareExternalPublication( f.file, selected, - changedDestination, - ); - expect(retry.preview.resumed).toBe(true); - expect((await retry.publish()).status).toBe("complete"); - expect(f.posts[1]).toBe(f.posts[0]); - expect([...f.reports.values()][0]!.evidence.source_data).toMatchObject({ - runContext: { - versionControlProvenance: original.runs[0].versionControlProvenance, - }, - }); - const posted = f.posts.length; - await expect( - prepareExternalPublication(f.file, selected, changedDestination), - ).rejects.toThrow("does not match the selected Cloud repository"); - expect(f.posts).toHaveLength(posted); - expect(input).toEqual(original); -}); - -test("publication rejects malformed options without writing findings", async () => { - const f = await fixture(); - const missingSourceKey = [ - ...f.command.slice(1, f.command.indexOf("--source-key") + 1), - "--dry-run", - "--yes", - ]; - for (const flags of [ - ["--json"], - ["--json", "--filter-output", "--token-limit"], - ["--filter-output", "--to"], - ]) { - const cli = createCliTest(main); - expect( - await cli.runCli(["publish", ...flags, ...missingSourceKey], f.cliDeps), - ).toBe(2); - expect(cli.stderr.text()).toContain("Missing value for flag"); - } - const extraArguments = createCliTest(main); - expect( - await extraArguments.runCli( - [ - "--filter-output", - "--token-limit", - ...f.command, - "--yes", - "extra-1", - "extra-2", - ], - f.cliDeps, - ), - ).toBe(2); - expect(extraArguments.stderr.text()).toContain( - "Unexpected positional argument", - ); - const preview = createCliTest(main); - expect( - await preview.runCli( - ["--json", ...f.command, "--dry-run", "--yes"], - f.cliDeps, - ), - ).toBe(0); - expect(JSON.parse(preview.stdout.text())).toMatchObject({ dryRun: true }); - expect(f.posts).toHaveLength(0); -}); - -test("a held primitive retry cannot republish after a newer boxed approval", async () => { - const f = await fixture(); - const prepare = (elapsed: unknown) => - prepareNormalizedExternalPublication( - scoredFindings(12, { elapsed }), - options, - f.deps, - ); - const original = await prepare(2.5); - f.state.loseResponse = true; - await expect(original.publish()).rejects.toThrow(/resume the saved request/); - const held = await prepare(2.5); - expect(held.preview.resumed).toBe(true); - const completed = await (await prepare(2.5)).publish(); - - const newer = await prepare(new Number(2.5)); - const newerId = newer.preview.requests[0]!.request_id; - expect(newerId).not.toBe(original.preview.requests[0]!.request_id); - f.state.loseResponse = true; - await expect(newer.publish()).rejects.toThrow(/resume the saved request/); - const before = [...f.posts]; - expect(await held.publish()).toEqual(completed); - expect(f.posts).toEqual(before); - - const remaining = await prepare(2.5); - expect(remaining.preview.requests).toEqual(newer.preview.requests); - expect((await remaining.publish()).receipts[0]!.id).toBe(newerId); - const directory = join(f.root, "state", "external-finding-publications"); - const filename = (await readdir(directory)).find((name) => - name.endsWith(".result.json"), - )!; - const resultPath = join(directory, filename); - const saved = await readFile(resultPath, "utf8"); - await expect(held.publish()).rejects.toThrow(/changed after preview/); - expect(await readFile(resultPath, "utf8")).toBe(saved); - expect(f.posts).toEqual([ - ...before, - JSON.stringify(newer.preview.requests[0]), - ]); -}); - -const concurrentBoxedApproval = - "concurrent prepares reuse only the matching completed boxed approval"; -test(concurrentBoxedApproval, async () => { - const { runTestInSubprocess } = await import("./support/test-subprocess.js"); - if (runTestInSubprocess(import.meta.filename, concurrentBoxedApproval)) - return; - - for (const state of [ - "original result", - "newer pending", - "newer complete", - "released result", - "released different", - "released malformed", - ]) { - const { f, prepare, directory, path, request } = - await oldScoreApproval("evidence"); - const atLock = Promise.withResolvers(); - const release = Promise.withResolvers(); - const lstat = fs.lstat; - let holdFirst = true; - const metadata = spyOn(fs, "lstat").mockImplementation((async ( - ...args: Parameters - ) => { - const file = String(args[0]); - if ( - holdFirst && - file.startsWith(directory) && - file.endsWith(".lock.sqlite") - ) { - holdFirst = false; - atLock.resolve(); - await release.promise; - } - return lstat(...args); - }) as typeof fs.lstat); - - const waiting = prepare(new Number(2.5)); - try { - await Promise.race([atLock.promise, waiting]); - const leader = await prepare(new Number(2.5)); - const { wireKey } = JSON.parse(await readFile(path, "utf8")); - const completed = await leader.publish(); - expect(completed.receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request)]); - let pending: { path: string; bytes: string } | undefined; - if (state.startsWith("released")) { - // The released publisher stores this receipt under its original checkpoint filename. - const oldResult = path.replace(/\.pending\.json$/, ".result.json"); - const currentName = `${wireKey}.result.json`; - const seed = (await readdir(directory)).find( - (name) => name.endsWith(".result.json") && name !== currentName, - )!; - const currentResult = join(directory, currentName); - await fs.rename(currentResult, oldResult); - if (state === "released malformed") { - const result = JSON.parse(await readFile(oldResult, "utf8")); - result.receipts[0].results = []; - await writeFile(oldResult, JSON.stringify(result)); - } else if (state === "released different") { - await fs.copyFile(join(directory, seed), oldResult); - } else { - // A different completed request can still occupy the current result filename. - await fs.copyFile(join(directory, seed), currentResult); - } - } else if (state !== "original result") { - const newer = await prepare(2.5); - const newerId = newer.preview.requests[0]!.request_id; - expect(newerId).not.toBe(request.request_id); - if (state === "newer pending") { - f.state.throttle = true; - await expect(newer.publish()).rejects.toThrow("Slow down"); - const [name] = (await readdir(directory)).filter((name) => - name.endsWith(".pending.json"), - ); - const path = join(directory, name!); - pending = { path, bytes: await readFile(path, "utf8") }; - expect(JSON.parse(pending.bytes).requests[0].request_id).toBe( - newerId, - ); - f.state.throttle = false; - } else { - expect((await newer.publish()).receipts[0]!.id).toBe(newerId); - } - } - const before = [...f.posts]; - release.resolve(); - if (state === "released malformed") { - await expect(waiting).rejects.toThrow(); - } else if (state === "newer complete" || state === "released different") { - await expect(waiting).rejects.toThrow(/changed while preparing/); - } else { - const trailing = await waiting; - expect(trailing.preview.resumed).toBe(true); - expect(trailing.preview.requests[0]!.request_id).toBe( - request.request_id, - ); - expect(await trailing.publish()).toEqual(completed); - } - if (pending) - expect(await readFile(pending.path, "utf8")).toBe(pending.bytes); - expect(f.posts).toEqual(before); - } finally { - release.resolve(); - metadata.mockRestore(); - await Promise.allSettled([waiting]); - } - } -}); - -test("a lookup hint cannot substitute for the tagged approval", async () => { - const { f, prepare, directory, path, filename, request } = - await oldScoreApproval("evidence"); - f.state.throttle = true; - await expect((await prepare(new Number(2.5))).publish()).rejects.toThrow( - "Slow down", - ); - const tag = JSON.parse(await readFile(path, "utf8")); - const lookup = join(directory, `${tag.wireKey}.lookup.json`); - const suffix = ".pending.json"; - expect(JSON.parse(await readFile(lookup, "utf8"))).toBe( - filename.slice(0, -suffix.length), - ); - - const otherReport = { units: "seconds" }; - const other = await prepare(new Number(3.5), otherReport); - await expect(other.publish()).rejects.toThrow("Slow down"); - const [otherName] = (await readdir(directory)).filter( - (name) => name.endsWith(suffix) && name !== filename, - ); - const otherPath = join(directory, otherName!); - const otherSaved = await readFile(otherPath, "utf8"); - const otherTag = JSON.parse(otherSaved); - expect(otherTag.wireKey).not.toBe(tag.wireKey); - const otherKey = otherName!.slice(0, -suffix.length); - expect( - JSON.parse( - await readFile( - join(directory, `${otherTag.wireKey}.lookup.json`), - "utf8", - ), - ), - ).toBe(otherKey); - f.posts.length = 0; - f.state.throttle = false; - - const outside = join(directory, "..", "foreign.pending.json"); - await writeFile(outside, otherSaved, { mode: 0o600 }); - const reads = spyOn(fs, "readFile"); - try { - for (const hint of [ - undefined, - "{", - JSON.stringify("../foreign"), - JSON.stringify(otherKey), - ]) { - if (hint === undefined) await rm(lookup); - else await writeFile(lookup, hint); - const actual = await prepare(2.5); - expect(actual.preview.resumed).toBe(true); - expect(actual.preview.requests[0]!.request_id).toBe(request.request_id); - } - expect(reads.mock.calls.map(([path]) => String(path))).not.toContain( - outside, - ); - } finally { - reads.mockRestore(); - } - const completed = await (await prepare(2.5)).publish(); - expect(completed.receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([JSON.stringify(request)]); - expect(await readFile(otherPath, "utf8")).toBe(otherSaved); - - const held = await prepare(3.5, otherReport); - expect(held.preview.requests[0]!.request_id).toBe( - other.preview.requests[0]!.request_id, - ); - await rm(otherPath); - await expect(held.publish()).rejects.toThrow(/changed after preview/); - const fresh = await prepare(3.5, otherReport); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.requests[0]!.request_id).not.toBe( - other.preview.requests[0]!.request_id, - ); - expect(f.posts).toEqual([JSON.stringify(request)]); -}); - -test("a lookup write or cleanup failure leaves the approved request usable", async () => { - const { f, prepare, path, directory, request } = - await oldScoreApproval("evidence"); - const rename = fs.rename; - const write = spyOn(fs, "rename").mockImplementation(async (from, to) => { - if (String(to).endsWith(".lookup.json")) - throw new Error("Lookup write failed"); - return rename(from, to); - }); - f.state.throttle = true; - try { - const boxed = await prepare(new Number(2.5)); - await expect(boxed.publish()).rejects.toThrow("Slow down"); - expect( - write.mock.calls.some(([, to]) => String(to).endsWith(".lookup.json")), - ).toBe(true); - } finally { - write.mockRestore(); - } - const tag = JSON.parse(await readFile(path, "utf8")); - const lookup = join(directory, `${tag.wireKey}.lookup.json`); - expect(await Bun.file(lookup).exists()).toBe(false); - f.state.throttle = false; - const remove = fs.rm; - const cleanup = spyOn(fs, "rm").mockImplementation(async (file, options) => { - if (String(file) === lookup) throw new Error("Lookup cleanup failed"); - return remove(file, options); - }); - try { - const retry = await prepare(2.5); - expect(retry.preview.resumed).toBe(true); - const result = await retry.publish(); - expect(result.status).toBe("complete"); - expect(result.verified).toBe(1); - expect(result.receipts[0]!.id).toBe(request.request_id); - expect(cleanup.mock.calls.some(([file]) => String(file) === lookup)).toBe( - true, - ); - } finally { - cleanup.mockRestore(); - } - expect(await Bun.file(path).exists()).toBe(false); - expect(await Bun.file(lookup).exists()).toBe(true); - expect((await prepare(2.5)).preview.resumed).toBe(false); - expect(f.posts).toEqual([JSON.stringify(request), JSON.stringify(request)]); -}); - -test("partially unboxing an approved input resumes its verified wire request", async () => { - const { f, prepare, directory, request } = await oldScoreApproval( - "evidence", - false, - ); - // This filename, projection and request ID came from the released SDK with b boxed. - request.request_id = "dd905f37-1437-44a3-b559-8c88fe873e52"; - Object.assign(request.items[0]!.evidence.source_data, { - score: { a: 7, b: 99 }, - }); - const { path: oldPath, saved } = await stageReleasedCheckpoint(f, { - filename: - "ad0ce218fdfcfa75455f2fbda30dc6e248a79419190e5fa77a7c12005ead7926.pending.json", - requests: [request], - inputDigest: - "232fbafeda24e0939a033a0d59479908389a8e167612feabc174e36f716303a5", - savedEvidenceDigest: - "3a5021a32dc41a1fe4b2d8e2f74abfd1238ef63cf5bfb4cf44cb28bb09e5bfad", - }); - const partial = () => prepare({ a: 7, b: new Number(2) }); - await expect(partial()).rejects.toThrow("cannot be matched"); - - const approved = await prepare({ a: new Number(7), b: new Number(2) }); - f.state.throttle = true; - await expect(approved.publish()).rejects.toThrow("Slow down"); - f.state.throttle = false; - const body = JSON.stringify(approved.preview.requests[0]); - const hinted = await partial(); - expect(hinted.preview.resumed).toBe(true); - expect(JSON.stringify(hinted.preview.requests[0])).toBe(body); - const lookup = (await readdir(directory)).find((name) => - name.endsWith(".lookup.json"), - )!; - await rm(join(directory, lookup)); - const retry = await partial(); - expect(retry.preview.resumed).toBe(true); - expect(JSON.stringify(retry.preview.requests[0])).toBe(body); - expect((await retry.publish()).receipts[0]!.id).toBe( - approved.preview.requests[0]!.request_id, - ); - expect(f.posts).toEqual([body, body]); - expect(await readFile(oldPath, "utf8")).toBe(saved); -}); - -test("preparing across a checkpoint handoff returns the observed approval", async () => { - const { - f, - prepare, - path: boxedPath, - } = await oldScoreApproval("evidence", false); - const first = await prepare(2.5); - f.state.loseResponse = true; - const interrupted = await first.publish().catch((error: unknown) => error); - if (!(interrupted instanceof ExternalPublicationError)) throw interrupted; - const canonicalPath = interrupted.result.savedSubmission!; - const leading = await prepare(2.5); - - const captured = Promise.withResolvers(); - const release = Promise.withResolvers(); - let hold = true; - const read = fs.readFile; - const reads = spyOn(fs, "readFile").mockImplementation((async ( - ...args: Parameters - ) => { - const bytes = await read(...args); - if (hold && String(args[0]) === canonicalPath) { - hold = false; - captured.resolve(); - await release.promise; - } - return bytes; - }) as typeof read); - const waiting = prepare(2.5); - try { - await Promise.race([captured.promise, waiting]); - expect(hold).toBe(false); - const completed = await leading.publish(); - expect(await Bun.file(canonicalPath).exists()).toBe(false); - const next = await prepare(new Number(2.5)); - expect(next.preview.requests[0]!.request_id).not.toBe( - first.preview.requests[0]!.request_id, - ); - f.state.throttle = true; - await expect(next.publish()).rejects.toThrow("Slow down"); - const newer = await readFile(boxedPath, "utf8"); - const before = [...f.posts]; - release.resolve(); - const trailing = await waiting; - expect(trailing.preview.resumed).toBe(true); - expect(trailing.preview.requests).toEqual(first.preview.requests); - expect(await trailing.publish()).toEqual(completed); - expect(await readFile(boxedPath, "utf8")).toBe(newer); - expect(f.posts).toEqual(before); - } finally { - release.resolve(); - reads.mockRestore(); - await Promise.allSettled([waiting]); - } -}); - -test("a changing GitHub ref cannot expand an older pending publication", async () => { - const f = await fixture(); - const prepare = ( - selected: ReturnType, - ) => - prepareNormalizedExternalPublication( - selected, - { ...options, provider: "github" }, - f.deps, - ); - const first = importedAlert(); - const preview = ( - await prepare(normalizeGitHubCodeScanningAlerts([first], "cloud")) - ).preview; - // The released reader excluded the second alert before its rule serializer ran. - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "0b43d13c77ddf5fa43ce0bb536402595aea270191adde5974f2206753fa607b1.pending.json", - accountId: preview.accountId, - requests: [ - { - ...preview.requests[0]!, - request_id: "f49f8193-10b0-4c6c-a20d-390304a5f8d0", - }, - ], - inputDigest: - "af139e1baa9e9e70a47d4ef0f811cb3960e9c280252affab51c250e7e6f9aa88", - }); - expect(hash("sha256", saved)).toBe( - "5f17c21d058c685c128d488b5a2a1e466fa5d18e37cc5284268b95ef25307293", - ); - - const second = importedAlert(); - second.number = second.alert.number = 13; - second.url = second.alert.html_url = first.url.replace(/12$/, "13"); - let ref = ""; - Object.defineProperty(second.alert.most_recent_instance, "ref", { - enumerable: true, - get: () => ref, - }); - Object.defineProperty(second.alert.rule, "toJSON", { - value() { - ref = "refs/heads/changed"; - return { ...this }; - }, - }); - const selected = normalizeGitHubCodeScanningAlerts([first, second], "cloud"); - expect(selected.findings).toHaveLength(2); - await expect(prepare(selected)).rejects.toThrow("cannot be matched"); - expect(f.posts).toHaveLength(0); - expect(await readFile(path, "utf8")).toBe(saved); -}); - -test("a normalized Buffer at the Cloud depth limit resumes only its released request", async () => { - const f = await fixture(); - const prepare = (leaf: unknown) => { - let tree = leaf; - for (let i = 0; i < 17; i++) tree = { nested: tree }; - const selected = scoredFindings(new Number(2.5)); - Object.assign(selected.findings[0]!.evidence.source_data, { tree }); - return prepareReleased(f, selected); - }; - await expect(prepare(Buffer.from([1]))).rejects.toThrow( - "Cloud JSON depth limit of 20", - ); - - // The released SDK cloned this Buffer into a Uint8Array with one fewer JSON level. - const request = { - ...(await prepare({ 0: 1 })).preview.requests[0]!, - request_id: "54bf2d2b-9380-415e-a364-963088768421", - }; - const serialized = JSON.stringify(request); - expect(hash("sha256", serialized)).toBe( - "5d105b251af91c09d0b206404bd9332031da8e3cb4bb02a9b8add4e4c21814c4", + changedDestination, ); - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "8376bdd00875ba8349ea341fab1c1d7c91eface0f2346d5eacf28e5c534a6d6a.pending.json", - requests: [request], - inputDigest: - "4eace8a1f10136be4258341e48f747f3609919238018ad5f8d84ceebfe469bb2", - savedEvidenceDigest: - "68702d81bc70bb23815207a069b0780b2b5b1f304250ca4696ecfe129c48e497", + expect(retry.preview.resumed).toBe(true); + expect((await retry.publish()).status).toBe("complete"); + expect(f.posts[1]).toBe(f.posts[0]); + expect([...f.reports.values()][0]!.evidence.source_data).toMatchObject({ + runContext: { + versionControlProvenance: original.runs[0].versionControlProvenance, + }, }); - await expect(prepare(Buffer.from([2]))).rejects.toThrow(); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toHaveLength(0); - - const recovered = await prepare(Buffer.from([1])); - expect(recovered.preview.resumed).toBe(true); - expect((await recovered.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([serialized]); + const posted = f.posts.length; + await expect( + prepareExternalPublication(f.file, selected, changedDestination), + ).rejects.toThrow("does not match the selected Cloud repository"); + expect(f.posts).toHaveLength(posted); + expect(input).toEqual(original); }); -test("the local GitHub reader recovers its released Buffer at the Cloud depth limit", async () => { +test("publication rejects malformed options without writing findings", async () => { const f = await fixture(); - const selected = (leaf: unknown) => { - const row = importedAlert(); - let payload = leaf; - for (let i = 0; i < 16; i++) payload = { nested: payload }; - Object.assign(row.alert, { payload }); - return normalizeGitHubCodeScanningAlerts([row], "local"); - }; - const prepare = (input: ReturnType) => - prepareReleased(f, input, { ...options, provider: "github" }); - await expect(prepare(selected(Buffer.from([1])))).rejects.toThrow( - "Cloud JSON depth limit of 20", - ); - - // The released publisher sent the native byte without the Buffer's JSON envelope. - const request = { - ...(await prepare(selected({ 0: 1 }))).preview.requests[0]!, - request_id: "3681fc33-491a-4ef4-877c-4cdd6f7dac6c", - }; - const wire = JSON.stringify(request); - expect(hash("sha256", wire)).toBe( - "0603739963094b95e0f454f2cfd70ec9ee9f7abf891d43d536a67ad27dcf7b95", + const missingSourceKey = [ + ...f.command.slice(1, f.command.indexOf("--source-key") + 1), + "--dry-run", + "--yes", + ]; + for (const flags of [ + ["--json"], + ["--json", "--filter-output", "--token-limit"], + ["--filter-output", "--to"], + ]) { + const cli = createCliTest(main); + expect( + await cli.runCli(["publish", ...flags, ...missingSourceKey], f.cliDeps), + ).toBe(2); + expect(cli.stderr.text()).toContain("Missing value for flag"); + } + const extraArguments = createCliTest(main); + expect( + await extraArguments.runCli( + [ + "--filter-output", + "--token-limit", + ...f.command, + "--yes", + "extra-1", + "extra-2", + ], + f.cliDeps, + ), + ).toBe(2); + expect(extraArguments.stderr.text()).toContain( + "Unexpected positional argument", ); - const { path, saved } = await stageReleasedCheckpoint(f, { - filename: - "0f045c3b26dc68db90c2d4e8462be50efa7a758d256398e22f13d1fe070e552a.pending.json", - requests: [request], - inputDigest: - "85488c56d0f48bb7dff1f931e69603aa4aa75c1ef287bf612fd84dc7df369aa2", - }); - - const edited = selected(Buffer.from([1])); - edited.findings[0]!.evidence.title = "Edited after the local read"; - await expect(prepare(edited)).rejects.toThrow("Cloud JSON depth limit of 20"); - expect(await readFile(path, "utf8")).toBe(saved); + const preview = createCliTest(main); + expect( + await preview.runCli( + ["--json", ...f.command, "--dry-run", "--yes"], + f.cliDeps, + ), + ).toBe(0); + expect(JSON.parse(preview.stdout.text())).toMatchObject({ dryRun: true }); expect(f.posts).toHaveLength(0); - - const resumed = await prepare(selected(Buffer.from([1]))); - expect(resumed.preview.resumed).toBe(true); - expect((await resumed.publish()).receipts[0]!.id).toBe(request.request_id); - expect(f.posts).toEqual([wire]); }); -test.each([ - [ - "cloned", - ["main"], - "0e174a52fb4c08c30d5b452239859f60186b2e5f3ea11476b7c174712b5f9dca", - "f97a11da51b2461c09e003c6167239198cde1e28a0b0ec97d8e40a512f396c99", - ], - [ - "reversed", - ["a", "b"], - "a64d3d2ca1432fae7c53f2880f7e6719f4e4f673ce61e85f8084980d95bc55ae", - "28c71167a82f47a1f038e6469f36a8f740fe3584ba67f8d4a52d5dbcbdff44d5", - ], -] as const)( - "preserves a released GitHub Buffer checkpoint with %s reader output", - async (kind, refs, key, digest) => { - const f = await fixture(); - const prepare = (bytes: unknown) => { - const rows = refs.map((ref) => { - const row = importedAlert(`refs/heads/${ref}`); - Object.assign(row.alert, { bytes }); - return row; - }); - const result = normalizeGitHubCodeScanningAlerts(rows); - if (kind === "reversed") result.findings.reverse(); - return prepareReleased( - f, - kind === "cloned" ? structuredClone(result) : result, - { ...options, provider: "github" }, - ); - }; - // The released reader cloned each Buffer into a Uint8Array before publication. - const requests = (await prepare({ 0: 1 })).preview.requests; - const { checkpoint, directory, path, saved } = - await stageReleasedCheckpoint(f, { - filename: `${key}.pending.json`, - requests, - inputDigest: digest, - }); - await expect(prepare(Buffer.from([1]))).rejects.toThrow( - "checkpoint was preserved", - ); - expect(await readFile(path, "utf8")).toBe(saved); - expect(f.posts).toHaveLength(0); - if (kind === "cloned") { - expect((await prepare(Buffer.from([2]))).preview.resumed).toBe(false); - - // A migrated approval can be linked to its scoped wire key from an older filename. - const linkedPath = join(directory, `${"a".repeat(64)}.pending.json`); - await fs.rename(path, linkedPath); - const linked = { - ...checkpoint, - wireKey: key, - inputProjection: { ...checkpoint.inputProjection, identityFormat: 2 }, - }; - await writeFile( - linkedPath, - JSON.stringify({ ...linked, legacyApproval: true }), - ); - await expect(prepare(Buffer.from([1]))).rejects.toThrow( - "checkpoint was preserved", - ); - // A format-2 request for the plain indexed object is an independent selection. - await writeFile(linkedPath, JSON.stringify(linked)); - expect((await prepare(Buffer.from([1]))).preview.resumed).toBe(false); - } - }, -); - -test("the GitHub reader resumes a released boxed score whose Buffer fit the Cloud request", async () => { +test("a held primitive retry cannot republish after a newer boxed approval", async () => { const f = await fixture(); - const row = importedAlert(); - const extension = { - score: new Number(1e-7), - bytes: Buffer.from([1]), - pad: "", - }; - Object.assign(row.alert, { extension }); - const normalize = () => normalizeGitHubCodeScanningAlerts([row], "cloud"); - const prepare = (input = normalize()) => + const prepare = (elapsed: unknown) => prepareNormalizedExternalPublication( - input, - { ...options, provider: "github" }, + scoredFindings(12, { elapsed }), + options, f.deps, ); + const original = await prepare(2.5); + f.state.loseResponse = true; + await expect(original.publish()).rejects.toThrow(/resume the saved request/); + const held = await prepare(2.5); + expect(held.preview.resumed).toBe(true); + const completed = await (await prepare(2.5)).publish(); - // Rebuild the released request without storing the 256 KiB row in the fixture. - const priorReader = structuredClone(normalize()); - priorReader.findings[0]!.evidence.source_data = { ...row }; - const template = await prepare(); - const request = JSON.parse(JSON.stringify(template.preview.requests[0])); - request.request_id = "5e640fea-52f8-43c2-ba4c-a0822c040458"; - extension.pad = "x".repeat(260303); - const wire = request.items[0].evidence; - wire.source_data.alert.extension.bytes = { "0": 1 }; - wire.source_data.alert.extension.pad = extension.pad; - expect( - Buffer.byteLength(JSON.stringify(priorReader.findings[0]!.evidence)), - ).toBe(256 * 1024); - const bodyHash = - "69a99d16710c9dc0710268e00b91ecdbb45df094d2918ec9797163332bcd37c5"; - expect(Buffer.byteLength(JSON.stringify(wire))).toBe(262123); - expect(hash("sha256", JSON.stringify(request))).toBe(bodyHash); - await expect(prepare()).rejects.toThrow("Cloud limit of 256 KiB"); - await expect(prepare(priorReader)).rejects.toThrow("Cloud limit of 256 KiB"); - expect(f.posts).toHaveLength(0); - - await stageReleasedCheckpoint(f, { - filename: - "c03e748fff72692564d445fc232317c164f16cf16fc5b1969bbee80d98bdb97e.pending.json", - requests: [request], - inputDigest: - "6d0cd82bed91273c26d2d9cd6eea9814f7ce3ac33af58a266f6554617d560362", - }); + const newer = await prepare(new Number(2.5)); + const newerId = newer.preview.requests[0]!.request_id; + expect(newerId).not.toBe(original.preview.requests[0]!.request_id); + f.state.loseResponse = true; + await expect(newer.publish()).rejects.toThrow(/resume the saved request/); + const before = [...f.posts]; + expect(await held.publish()).toEqual(completed); + expect(f.posts).toEqual(before); - const resumed = await prepare(priorReader); - expect(resumed.preview.resumed).toBe(true); - expect(resumed.preview.excluded).toEqual([]); - f.state.brokenReadback = true; - await expect(resumed.publish()).rejects.toThrow("Readback unavailable"); - f.state.brokenReadback = false; - const completed = await (await prepare()).publish(); - expect(completed).toMatchObject({ - status: "complete", - ready: 1, - excluded: [], - unacknowledged: 0, - counts: { created: 1 }, - verified: 1, - }); - expect(completed.receipts.map(({ id }) => id)).toEqual([request.request_id]); - expect(f.posts.map((body) => hash("sha256", body))).toEqual([bodyHash]); + const remaining = await prepare(2.5); + expect(remaining.preview.requests).toEqual(newer.preview.requests); + expect((await remaining.publish()).receipts[0]!.id).toBe(newerId); + const directory = join(f.root, "state", "external-finding-publications"); + const filename = (await readdir(directory)).find((name) => + name.endsWith(".result.json"), + )!; + const resultPath = join(directory, filename); + const saved = await readFile(resultPath, "utf8"); + await expect(held.publish()).rejects.toThrow(/changed after preview/); + expect(await readFile(resultPath, "utf8")).toBe(saved); + expect(f.posts).toEqual([ + ...before, + JSON.stringify(newer.preview.requests[0]), + ]); }); const sonarReport = sonarCapture(exampleRepository); @@ -11986,91 +8040,6 @@ test.each(["package", "malware aliases"] as const)( }, ); -test("an interrupted native Endor checkpoint retains its original bytes and IDs until its inbox changes", async () => { - const raw = { - ...endorSource, - extra: "x".repeat(190 * 1024), - spec: { - ...endorSource.spec, - target_dependency_name: null, - target_dependency_package_name: "x".repeat(495), - dependency_file_paths: Array.from( - { length: 100 }, - (_, i) => `modules/${i}/yarn.lock`, - ), - }, - }; - const { f, inbox, uploads, prepareFile, jobs, fileId, settings } = - await privateUploadFixture(raw, "endor"); - f.state.catalogHeaders = {}; - const repositoryPreview = await prepareFile(); - f.state.catalogHeaders["Codex-Security-Private-Import-Providers"] = "endor"; - const fresh = await prepareFile(); - expect(fresh.preview.privateImport?.format).toBe("security.findings.v1"); - expect(fresh.preview.findings[0]!.evidence).toMatchObject({ - packages: [], - source_data: raw, - }); - inbox.formats = ["endor.findings_json.v1"]; - await expect(prepareFile()).rejects.toThrow("security.findings.v1"); - // This released checkpoint predates structured-only uploads for reduced Endor fields. - const key = - "52629feec0c5f197e2bce1779b908306b6b01d232112b359451da0da111c6ccd"; - const checkpoint = { - format: "endor.findings_json.v1", - apiBaseUrl: "https://chatgpt.com/backend-api/aardvark", - accountId: "synthetic-account", - contextVersion: inbox.version, - source: { provider: "endor", source_key: settings.sourceKey }, - repository: { id: options.repository, repo_connector_id: "github" }, - sha256: "2aba52fd27bf9b6b2fdf6cb84b701a61690266a29924fb6d6a25e9a4c1574341", - uploadRequestId: "200ce23c-1576-4525-8b58-50c43812bce8", - jobRequestId: "9bceaf8a-8ef4-4bbc-99a7-78fc0d6dcbcf", - }; - await f.writeCheckpoint(key + ".private-import.json", checkpoint); - const retry = await prepareFile(); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.privateImport).toMatchObject({ - format: checkpoint.format, - sha256: checkpoint.sha256, - }); - expect(checkpoint.sha256).not.toBe(fresh.preview.privateImport?.sha256); - await expect(retry.publish()).rejects.toThrow("Interrupted private upload"); - expect(uploads).toEqual([ - { id: checkpoint.uploadRequestId, bytes: JSON.stringify(raw) }, - ]); - await expect(repositoryPreview.publish()).rejects.toThrow( - "A private import already owns this selection", - ); - inbox.recoverUpload = true; - expect((await (await prepareFile()).publish()).privateJob?.status).toBe( - "completed", - ); - expect(jobs).toEqual([ - { - request_id: checkpoint.jobRequestId, - context_version: inbox.version, - ...privateJobFields("endor", fileId, checkpoint.format), - }, - ]); - inbox.version = "new-inbox"; - inbox.formats.push("security.findings.v1"); - const next = await prepareFile(); - expect(next.preview.resumed).toBe(false); - expect(next.preview.privateImport?.sha256).toBe( - fresh.preview.privateImport?.sha256, - ); - inbox.recoverUpload = false; - await expect(next.publish()).rejects.toThrow("Interrupted private upload"); - const uploaded = JSON.parse(uploads.at(-1)!.bytes); - expect(uploaded.format).toBe("security.findings.v1"); - expect(uploaded.findings).toHaveLength(1); - expect(uploaded.findings[0].evidence).toMatchObject({ - packages: [], - source_data: raw, - }); -}); - const versionThreeReports = [ ["blackduck", blackDuckBomReport(), { row: blackDuckBomRow() }], [ @@ -12097,11 +8066,20 @@ test.each([ format: "security.findings.v1", raw: [importedAlert()], }, - ...awsReaderUpgrades.map((upgrade) => ({ - ...upgrade, - format: "security.findings.v1" as const, - jobVersion: 2 as const, - })), + { + provider: "inspector", + raw: { findings: [inspectorCodeFinding()] }, + sourceRepositoryUrl: inspectorRepositoryUrl, + format: "security.findings.v1", + jobVersion: 2, + }, + { + provider: "continuum", + raw: continuumDetail(), + sourceRepositoryUrl: continuumRepository, + format: "security.findings.v1", + jobVersion: 2, + }, { provider: "snyk", format: "security.findings.v1", raw: snykSarif() }, { provider: "endor", @@ -12117,10 +8095,6 @@ test.each([ }, }, ], - normalizePrevious: (report: unknown) => - normalizePreviousEndorFindings(report), - previousSha256: - "a6a869d6ac1ac02acf44849bbd55d571e350ff9ef3c8ca8562891eb8cdd64708", }, { provider: "endor", @@ -12163,11 +8137,7 @@ test.each([ ...(provider === "trivy" && { trivyScanRoot: "." }), }; let contextVersion = "generation-a"; - let formats = [ - "security.findings.v1", - "endor.findings_json.v1", - "wiz.vulnerability_json.v1", - ]; + let formats: string[] = [format]; let expectedFormat: typeof format = format; let loseResponses = true; let unavailable = false; @@ -12178,18 +8148,13 @@ test.each([ "jobVersion" in scenario ? scenario.jobVersion : undefined; const requestedVersion = String(jobVersion ?? 2); const versionedJobs = jobVersion ? `/v${jobVersion}${v1Jobs}` : v1Jobs; - const upgrade = provider === "inspector" || provider === "checkmarx"; - // An older acknowledgement also keeps the provider on its unversioned route. - let acknowledgedVersion = upgrade ? undefined : requestedVersion; - if (provider === "checkmarx") acknowledgedVersion = "2"; - let expectedJobsPath = upgrade ? v1Jobs : versionedJobs; + let acknowledgedVersion: string | undefined = requestedVersion; + const expectedJobsPath = versionedJobs; let canCreate = true; let reportCount = 1; - let unorderedUpdates = 0; const uploadedReports: { id: string; bytes: string }[] = []; let jobRequest: string | undefined; let loseNextJobResponse = true; - const jobBodies: string[] = []; const fileId = "aifile_" + "a".repeat(32); let jobId = "import_" + "b".repeat(32); const calls: string[] = []; @@ -12202,7 +8167,6 @@ test.each([ jobId, status, reportCount, - unorderedUpdates, ); const deps = { ...f.deps, @@ -12249,9 +8213,8 @@ test.each([ } if (provider === "inspector" || provider === "continuum") { const native = provider === "inspector" ? raw.findings : [raw]; - const expected = loseResponses ? native.slice(0, 1) : native; - expect(doc.findings).toHaveLength(expected.length); - for (const row of expected) { + expect(doc.findings).toHaveLength(native.length); + for (const row of native) { const finding = doc.findings.find( (item: { source_finding_id: string }) => item.source_finding_id === row.findingArn, @@ -12260,7 +8223,7 @@ test.each([ } } } else { - expect(bytes).toBe(originalBytes); + expect(JSON.parse(bytes)).toEqual(JSON.parse(originalBytes)); } uploadedReports.push({ id, bytes }); if (loseResponses && uploadedReports.length === 1) @@ -12269,12 +8232,9 @@ test.each([ } const fileRequest = uploadedReports.at(-1)?.id; if (path === `/finding-imports/file-requests/${fileRequest}`) { - if ("previousSha256" in scenario && uploadedReports.length === 1) - return new Response(null, { status: 404 }); return Response.json(fileMetadata()); } if (path === expectedJobsPath) { - jobBodies.push(String(init.body)); const body = JSON.parse(String(init.body)); expect(body.request_id).not.toBe(fileRequest); expect(body).toEqual({ @@ -12328,31 +8288,17 @@ test.each([ expect(cli.stderr.text()).toContain( `Report: ${format} (${original.preview.privateImport?.bytes} bytes)`, ); - if (format === "endor.findings_json.v1") { - const supported = formats; - formats = [format]; - expect((await prepare()).preview.privateImport?.format).toBe(format); - formats = supported; - } expect(calls.filter((call) => call.startsWith("POST"))).toEqual([]); contextVersion = "generation-b"; await expect(original.publish()).rejects.toThrow("private inbox changed"); canCreate = false; await expect(prepare()).rejects.toThrow("unavailable"); canCreate = true; - const staged = - "normalizePrevious" in scenario && scenario.normalizePrevious - ? await prepareNormalizedExternalPublication( - JSON.parse( - JSON.stringify( - scenario.normalizePrevious(raw, sourceRepositoryUrl ?? ""), - ), - ), - settings, - deps, - ) - : await prepare(); - const interrupted = await staged.publish().catch((error: unknown) => error); + const interrupted = await ( + await prepare() + ) + .publish() + .catch((error: unknown) => error); if (!(interrupted instanceof ExternalPublicationError)) throw new Error("Expected an interrupted private upload", { cause: interrupted, @@ -12365,31 +8311,22 @@ test.each([ sha256: fileMetadata().sha256, uploadRequestId: uploadedReports.at(-1)!.id, }); - expect(saved.jobApiVersion).toBe(upgrade ? undefined : jobVersion); - if ("previousSha256" in scenario) { - expect(saved.sha256).toBe(scenario.previousSha256); - expect(saved.sha256).not.toBe(original.preview.privateImport?.sha256); - } + expect(saved.jobApiVersion).toBe(jobVersion); await expect(repositoryPreview.publish()).rejects.toThrow( - "previousSha256" in scenario - ? "Another publication prepared this input" - : "A private import already owns this selection", + "A private import already owns this selection", ); expect(f.posts).toHaveLength(0); if ("rename" in scenario) { - f.state.repositoryUrl = "https://github.com/example/renamed"; - expect(saved.approvedRepositoryUrl).toBe(exampleRepository); - // Earlier checkpoints bound the immutable report and connector ID only. - const { approvedRepositoryUrl, ...legacy } = saved; const path = interrupted.result.savedSubmission!; const pending = await prepare(); await rm(path); await expect(pending.publish()).rejects.toThrow( "saved private import changed", ); - await writeFile(path, JSON.stringify(legacy), { mode: 0o600 }); - expect((await prepare()).preview.resumed).toBe(true); - await writeFile(path, JSON.stringify(saved)); + expect(uploadedReports).toHaveLength(1); + await writeFile(path, JSON.stringify(saved), { mode: 0o600 }); + f.state.repositoryUrl = "https://github.com/example/renamed"; + expect(saved.approvedRepositoryUrl).toBe(exampleRepository); } advertised = false; const resumed = await prepare(); @@ -12398,28 +8335,6 @@ test.each([ await expect(resumed.publish()).rejects.toThrow("lost job response"); expect(jobRequest).toBe(saved.jobRequestId); expect(calls.at(-1)).toBe(`POST ${expectedJobsPath}`); - if (upgrade) { - // The upgraded server accepts the same approved request on its new route. - acknowledgedVersion = requestedVersion; - expectedJobsPath = versionedJobs; - loseNextJobResponse = true; - await expect((await prepare()).publish()).rejects.toThrow( - "lost job response", - ); - expect(calls.at(-1)).toBe(`POST ${versionedJobs}`); - expect(jobBodies).toHaveLength(2); - expect(jobBodies[1]).toBe(jobBodies[0]); - const promoted = JSON.parse( - await readFile(interrupted.result.savedSubmission!, "utf8"), - ); - expect(promoted).toMatchObject({ - jobApiVersion: jobVersion, - jobRequestId: saved.jobRequestId, - uploadRequestId: saved.uploadRequestId, - sha256: saved.sha256, - fileId, - }); - } // Saved jobs retain their route if the catalogue acknowledgement disappears. acknowledgedVersion = undefined; const completed = await (await prepare()).publish(); @@ -12427,32 +8342,7 @@ test.each([ expect(calls.at(-1)).toBe(`GET ${expectedJobsPath}/${jobId}`); expect(completed.receipts).toEqual([]); expect(completed.verified).toBe(0); - if ("previousSha256" in scenario) { - expect(uploadedReports).toHaveLength(2); - expect(uploadedReports[1]).toEqual(uploadedReports[0]); - } else { - expect(uploadedReports).toHaveLength(1); - } - if (provider === "inspector") { - // A completed old job cannot account for the newly admitted package finding. - advertised = true; - acknowledgedVersion = requestedVersion; - loseResponses = false; - const next = await prepare(); - expect(next.preview.resumed).toBe(false); - expect(next.preview.privateImport?.sha256).toBe( - original.preview.privateImport?.sha256, - ); - // The code finding's changed evidence has no baseline. Only the new ARN - // can be created in this workspace. - unorderedUpdates = 1; - const nextResult = await next.publish(); - expect(nextResult.privateJob?.id).not.toBe(completed.privateJob?.id); - expect(nextResult.privateJob?.counts).toMatchObject({ - created: 1, - error: 1, - }); - } + expect(uploadedReports).toHaveLength(1); canCreate = false; advertised = false; acknowledgedVersion = requestedVersion; @@ -12462,7 +8352,7 @@ test.each([ const acknowledged = { result: { status: "interrupted", - counts: { created: reportCount - unorderedUpdates }, + counts: { created: reportCount }, unacknowledged: 0, privateJob: { id: jobId }, }, @@ -12478,8 +8368,6 @@ test.each([ expectedFormat, ); contextVersion = "generation-c"; - expectedJobsPath = versionedJobs; - unorderedUpdates = 0; advertised = true; canCreate = true; loseResponses = false; @@ -12493,10 +8381,6 @@ test.each([ const fresh = await prepare(); expect(fresh.preview.resumed).toBe(false); expect(fresh.preview.privateImport?.format).toBe(expectedFormat); - if ("previousSha256" in scenario) - expect(fresh.preview.privateImport?.sha256).toBe( - original.preview.privateImport?.sha256, - ); expect((await fresh.publish()).privateJob?.status).toBe(terminal); expect(fresh.preview.privateImport?.sha256).toBe(fileMetadata().sha256); const failedJob = jobId; @@ -12635,113 +8519,6 @@ function privateInboxContext( }); } -test.each([ - { - data: checkmarxFormat, - report: { sourceRepositoryUrl: exampleRepository, scanId: "123" }, - key: "0b6db48e67edf53d39403c9fd1897c34a179edf5b6cce9bffabc9f52f8757f33", - sha256: "8b0e177506af75c363184d3627fed74325acfcada7e3ee8e9d81041c5db5b2d1", - }, - { - data: { vulnerableAsset: wizBranchAsset }, - report: checkmarxFormat, - key: "7400c077334e75f2b1ca49f05eb5cc4f431b62a9f2063734f42a03d60500ea5e", - sha256: "bf944f1fec0d2f97675f35dc8db1bee753807879fa6077e2bd8c5169889ce7ae", - }, - { - data: { sourceRepositoryUrl: exampleRepository }, - report: checkmarxFormat, - key: "0c9c5eef708bb27f19dcaebb67832cc6dcd43bfbf9dee0f0d4c4d2da9145ffa8", - sha256: "2ae1422cd86875a7be2d73f2dc8f6833e494668f6e4be9982c403d336557ef9f", - }, -])( - "only a verified private Wiz checkpoint replays pre-Checkmarx evidence ($data)", - async ({ data, report, key, sha256 }) => { - const { f, deps, inbox, jobs } = await privateUploadFixture([], "wiz"); - f.state.catalogHeaders = {}; // Recovery must also work after discovery stops advertising Wiz. - const selection = sdkSelection( - [sdkRow(normalized().source_finding_id, data)], - report, - ); - const source = { - provider: options.provider, - source_key: options.sourceKey, - }; - const repository = { id: options.repository, repo_connector_id: "github" }; - const checkpoint = { - format: "security.findings.v1", - apiBaseUrl: "https://chatgpt.com/backend-api/aardvark", - accountId: "synthetic-account", - contextVersion: "generation-a", - source, - repository, - sha256, - uploadRequestId: "ff52f1ac-ee15-4126-a3f4-bc9984fa0c51", - jobRequestId: "a942a3d4-ad70-4b4d-aaca-4e1e2b70cb20", - fileId: "aifile_" + "a".repeat(32), - }; - const path = await f.writeCheckpoint( - key + ".private-import.json", - checkpoint, - ); - const prepare = () => - prepareNormalizedExternalPublication(selection, options, { - ...deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: undefined, - CODEX_SECURITY_CLOUD_WEB_URL: undefined, - }, - }); - inbox.version = "different-inbox"; - await expect(prepare()).rejects.toThrow("requires the checkmarx provider"); - inbox.version = checkpoint.contextVersion; - f.state.repositoryUrl = otherRepository; - await expect(prepare()).rejects.toThrow(/belongs to|does not match/); - f.state.repositoryUrl = exampleRepository; - const resumed = await prepare(); - expect(resumed.preview.resumed).toBe(true); - expect(resumed.preview.privateImport?.sha256).toBe(checkpoint.sha256); - expect((await resumed.publish()).counts.created).toBe(1); - expect(jobs).toEqual([ - { - request_id: checkpoint.jobRequestId, - context_version: inbox.version, - ...privateJobFields("wiz", checkpoint.fileId, checkpoint.format), - }, - ]); - await rm(path); - await expect(resumed.publish()).rejects.toThrow( - "requires the checkmarx provider", - ); - await expect(prepare()).rejects.toThrow("requires the checkmarx provider"); - expect(jobs).toHaveLength(1); - }, -); - -test("a private preview recognizes a later repository checkpoint under its historical filename", async () => { - const { f, deps, uploads, jobs } = await privateUploadFixture([], "wiz"); - const privatePreview = await prepareNormalizedExternalPublication( - scoredFindings(2.5), - options, - deps, - ); - f.state.catalogHeaders = {}; - const repository = await prepareNormalizedExternalPublication( - scoredFindings(new Number(2.5)), - options, - f.deps, - ); - f.state.throttle = true; - await expect(repository.publish()).rejects.toThrow("Slow down"); - await expect(privatePreview.publish()).rejects.toThrow( - "A repository submission is pending", - ); - expect(uploads).toHaveLength(0); - expect(jobs).toHaveLength(0); - expect(f.posts).toHaveLength(1); -}); - test("a private structured report retains occurrence IDs, unknown severity and source lines", () => { const input = [ { @@ -12773,54 +8550,3 @@ test("a private structured report retains occurrence IDs, unknown severity and s privateReport(input, { ...limits, max_record_bytes: 32 }), ).toThrow("record limit"); }); - -test("filtered, unsupported and changed files never upload an unreviewed native record", async () => { - const raw = endorFinding(); - const f = await fixture(raw); - const selection = normalizeEndorFindings(raw); - const context = { - object: "security.finding_import_context" as const, - context_version: "v1", - can_upload: true, - can_create_import_jobs: true, - can_read: true, - formats: ["security.findings.v1", "endor.findings_json.v1"], - limits: { - max_direct_file_bytes: 32 * 1024 * 1024, - max_record_bytes: 1024 * 1024, - }, - }; - const select = (findings = selection, formats = context.formats) => - selectPrivateReport( - findings, - "endor", - f.file, - { ...context, formats }, - undefined, - ); - expect((await select()).format).toBe("endor.findings_json.v1"); - expect((await select(selection, ["security.findings.v1"])).format).toBe( - "security.findings.v1", - ); - expect( - ( - await select({ - ...selection, - read: 2, - excluded: [ - { position: 2, source_finding_id: null, reason: "excluded" }, - ], - }) - ).format, - ).toBe("security.findings.v1"); - await writeFile( - f.file, - JSON.stringify({ list: { objects: [raw] }, unreviewed: "omitted" }), - ); - expect((await select()).bytes).not.toContain("unreviewed"); - await writeFile( - f.file, - JSON.stringify({ ...raw, meta: { ...raw.meta, name: "unreviewed" } }), - ); - expect((await select()).bytes).not.toContain("unreviewed"); -}); diff --git a/sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json b/sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json deleted file mode 100644 index 78a99106ab..0000000000 --- a/sdk/typescript/tests-ts/fixtures/boxed-number-checkpoints.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "accountId": "synthetic-account", - "request": { - "repository": { - "id": "repo-example", - "repo_connector_id": "github", - "environment_id": "environment-example", - "reset_marker": "generation-1" - }, - "source": { - "provider": "wiz", - "source_key": "tenant-example/vulnerability-finding" - }, - "items": [ - { - "client_id": "item-1", - "source_finding_id": "boxed-score", - "expected_version": 1, - "evidence": { - "title": "Scanner score", - "severity": "high", - "source_data": { - "score": 2.5 - }, - "description": null, - "url": null, - "advisory_ids": [], - "packages": [], - "locations": [], - "branch": null, - "code_revision": null, - "source_scan_id": null, - "image_digests": [], - "source_updated_at": null - } - } - ] - }, - "cases": [ - { - "field": "evidence", - "filename": "1b86d2d5fc316cb278171f1a8eabbb4c2662236348fe12d7064a4c2b0b8d76d7.pending.json", - "requestId": "e68c368b-9650-4c7a-9560-fab0c24b5a95", - "inputProjection": { - "inputDigest": "c7582873f1bfaab3cb3a8f5f0b6bdea63e622dfdf3896e4d4c34e3510125082f", - "savedEvidenceDigest": "425dc35452a9246322e7751766727cbfb373d5dedce85f143e6c8f1210ce5013" - } - }, - { - "field": "report", - "filename": "7281a4be61bfaf2614d8399d91dbc1a8fd34a2bac0f3e8ae1dfc0fa143fcb9ba.pending.json", - "requestId": "5b646989-937e-4497-83ee-6694b967597c", - "inputProjection": { - "inputDigest": "790edb4daf5954c0d992847a2aa38c665a4e36fe94a2e46cddc62aef4cc83fde", - "savedEvidenceDigest": "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc" - } - }, - { - "field": "aliased", - "filename": "294b9a84b47d7b08595ae253884111b17b21557f48e90897231da61ab4971035.pending.json", - "requestId": "9462e9b9-b78f-49a9-95bd-59bc3d842304", - "inputProjection": { - "inputDigest": "8f63530e51452c8bb96583dd2bb85705196a209c41258d45be4280e2b5ce424a", - "savedEvidenceDigest": "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc" - } - }, - { - "field": "mixed", - "filename": "2bdb5e0976da513a32b080995421a304bb498f2ecd8cf1dca323e50e4684cf2c.pending.json", - "requestId": "eb070433-a248-4313-a692-21c53effa137", - "inputProjection": { - "inputDigest": "e0d874045a94c17362500a970e819c07a10d4a27103acabf4f3fd50156050a4d", - "savedEvidenceDigest": "425dc35452a9246322e7751766727cbfb373d5dedce85f143e6c8f1210ce5013" - } - }, - { - "field": "unreported", - "filename": "a01f282b3e06ba2cf0ffb43749e47d15620e6f7b057bfb16a4baa3044e90bd6b.pending.json", - "requestId": "9ac96055-5fa4-4786-ad5b-a044fb59ce8f", - "inputProjection": { - "inputDigest": "7fddcc63cf22a90e05f4fb5dc5ef1107661d0a0147ae3eb7e610e840cf663fba", - "savedEvidenceDigest": "875a8aec2f1b7c573a6d05d983f6afa0f964a6e1c96d22c8c8691cc2de824ccc" - } - }, - { - "field": "flag", - "filename": "2302ce8a88755173629443d05cb1b296e77e62c99b2c2334116b04677202d96b.pending.json", - "requestId": "a47dc563-e876-46dc-b7a9-469dc4c32583", - "inputProjection": { - "inputDigest": "28f91d7339da91f5497a628f8e084ee938b363d7ce9bd18a0be95115f9769ecf", - "savedEvidenceDigest": "c731c2bd0466093f147300606be75bad7b61c6f308fc9fafa8e55c5e2f93049c" - } - }, - { - "field": "empty-report", - "filename": "d14283c9bb988385486811b4a41dcdbf4cf5b6474c5654e087ae0d60c4cd64ad.pending.json", - "requestId": "e1c3295f-41a3-4bc7-b6fd-58fab4338933", - "inputProjection": { - "inputDigest": "ac73456afe6f082161af44bc2134d65c1f75c1dc3ba191ab8a9b7ddf3fd05c73", - "savedEvidenceDigest": "425dc35452a9246322e7751766727cbfb373d5dedce85f143e6c8f1210ce5013" - } - } - ] -} diff --git a/sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json b/sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json deleted file mode 100644 index 7bf69b8c46..0000000000 --- a/sdk/typescript/tests-ts/fixtures/boxed-sonar-checkpoint.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "filename": "64800b63401460aadf4376cef6cf4d8aa6c1684d66c0541aa11aa3ce84306799.pending.json", - "checkpoint": { - "accountId": "synthetic-account", - "requests": [ - { - "request_id": "1b018e6f-7f68-40f0-90cc-e855e383f63c", - "repository": { - "id": "repo-example", - "repo_connector_id": "github", - "environment_id": "environment-example", - "reset_marker": "generation-1" - }, - "source": { - "provider": "sonar", - "source_key": "tenant-example/vulnerability-finding" - }, - "items": [ - { - "client_id": "item-1", - "source_finding_id": "issue-1", - "expected_version": 1, - "evidence": { - "title": "Review input handling", - "severity": "high", - "source_data": { - "format": "sonar-issues/v1", - "repositoryUrl": "https://github.com/example/project", - "issue": { - "key": "issue-1" - }, - "effort": 2.5 - }, - "description": null, - "url": null, - "advisory_ids": [], - "packages": [], - "locations": [], - "branch": null, - "code_revision": null, - "source_scan_id": null, - "image_digests": [], - "source_updated_at": null - } - }, - { - "client_id": "item-2", - "source_finding_id": "issue-2", - "expected_version": 1, - "evidence": { - "title": "Review input handling", - "severity": "high", - "source_data": { - "format": "sonar-issues/v1", - "repositoryUrl": "https://github.com/example/project", - "branch": "main", - "issue": { - "key": "issue-2" - } - }, - "branch": "main", - "description": null, - "url": null, - "advisory_ids": [], - "packages": [], - "locations": [], - "code_revision": null, - "source_scan_id": null, - "image_digests": [], - "source_updated_at": null - } - } - ] - } - ], - "inputProjection": { - "inputDigest": "35c7201ca98d5808994aece0d04b7f330ae32f9836d443f9676c329f808dbb58", - "savedEvidenceDigest": "5443dab95a2cd679f58b9fcfefc2ccacdc74dad2e6b2697f318883013ab5740a" - } - } -} diff --git a/sdk/typescript/tests-ts/fixtures/snyk-released-checkpoints.json b/sdk/typescript/tests-ts/fixtures/snyk-released-checkpoints.json deleted file mode 100644 index 770e05f4d2..0000000000 --- a/sdk/typescript/tests-ts/fixtures/snyk-released-checkpoints.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "code": { - "file": "9f5e2365b32a81e81b8257099217472aa8f10616073188538ac0bd995591eb4c.pending.json", - "checkpoint": { - "accountId": "synthetic-account", - "requests": [ - { - "request_id": "e13cc517-c37c-4ba0-ac03-866808c9a613", - "repository": { - "id": "repo-example", - "repo_connector_id": "github", - "environment_id": "environment-example", - "reset_marker": "generation-1" - }, - "source": { - "provider": "snyk", - "source_key": "tenant-example/vulnerability-finding" - }, - "items": [ - { - "client_id": "item-1", - "source_finding_id": "snyk-rest:96875c30e9652a9e0b475e75cd19e0fb32bcba6dc0b8bfe3ef0169ee7408e95f", - "expected_version": 0, - "evidence": { - "title": "Synthetic finding", - "description": null, - "severity": "high", - "advisory_ids": [], - "locations": [ - { - "path": "src/app.ts", - "line": 3 - } - ], - "code_revision": "abc123", - "source_updated_at": 1767225600, - "details": { - "kind": "sast", - "repository": { - "url": "https://github.com/example/project", - "id": null, - "name": null - }, - "scanner_origin": "Snyk Code", - "source_status": "open", - "rule": { - "id": "sample-rule", - "short_id": null, - "name": null - }, - "weakness_ids": [], - "scanner_verdict": null, - "remediation_instructions": null, - "code": null, - "secret": null, - "configuration": null, - "file_url": null - }, - "url": null, - "packages": [], - "branch": null, - "source_scan_id": null, - "image_digests": [], - "source_data": {} - } - } - ] - } - ], - "inputProjection": { - "inputDigest": "ab915d32fcdeb8b03c684d77c20c7371c188b9e4468ebd2fd4090d15a43d40a5", - "savedEvidenceDigest": "ab915d32fcdeb8b03c684d77c20c7371c188b9e4468ebd2fd4090d15a43d40a5" - } - } - }, - "open-source": { - "file": "fff528eb227b82ed41cb3390cfd12473d1c1b756bdd4f399d8bc0117b67d9498.pending.json", - "checkpoint": { - "accountId": "synthetic-account", - "requests": [ - { - "request_id": "dac96f20-5073-4ad4-89d8-3361cb30a072", - "repository": { - "id": "repo-example", - "repo_connector_id": "github", - "environment_id": "environment-example", - "reset_marker": "generation-1" - }, - "source": { - "provider": "snyk", - "source_key": "tenant-example/vulnerability-finding" - }, - "items": [ - { - "client_id": "item-1", - "source_finding_id": "snyk-rest:1c938a65924060f2e39c4135d140fa882f2e6b0c4875202feee308e7e44ea815", - "expected_version": 0, - "evidence": { - "title": "Synthetic finding", - "description": null, - "severity": "high", - "advisory_ids": ["sample-rule"], - "packages": [ - { - "name": "example-lib", - "installed_version": "1.0.0", - "ecosystem": "npm", - "manifest_path": "app/package.json", - "fixed_versions": [] - } - ], - "source_updated_at": 1767225600, - "url": null, - "locations": [], - "branch": null, - "code_revision": null, - "source_scan_id": null, - "image_digests": [], - "source_data": {} - } - } - ] - } - ], - "inputProjection": { - "inputDigest": "9facde5e89e65c58515c4b84f445b64adf796228af93a4fba49b894b16d807a9", - "savedEvidenceDigest": "9facde5e89e65c58515c4b84f445b64adf796228af93a4fba49b894b16d807a9" - } - } - } -} diff --git a/sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json b/sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json deleted file mode 100644 index 74eb8553b0..0000000000 --- a/sdk/typescript/tests-ts/fixtures/wiz-preview-checkpoint.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "accountId": "synthetic-account", - "requests": [ - { - "request_id": "4d815058-28ce-4c37-9c59-36188c3372be", - "repository": { - "id": "cloud-repository", - "repo_connector_id": "github", - "environment_id": "environment-1", - "reset_marker": "generation-1" - }, - "source": { - "provider": "wiz", - "source_key": "synthetic-tenant/wiz" - }, - "items": [ - { - "client_id": "item-1", - "source_finding_id": "unicode-evidence", - "expected_version": 0, - "evidence": { - "title": "Unicode vendor metadata", - "severity": "high", - "source_data": { - "é": "precomposed", - "é": "combining", - "Z": 1, - "a": 2, - "ä": 3 - }, - "description": null, - "url": null, - "advisory_ids": [], - "packages": [], - "locations": [], - "branch": null, - "code_revision": null, - "source_scan_id": null, - "image_digests": [], - "source_updated_at": null - } - } - ] - } - ], - "receipts": [ - { - "id": "4d815058-28ce-4c37-9c59-36188c3372be", - "repository": { - "id": "cloud-repository", - "repo_connector_id": "github", - "environment_id": "environment-1", - "reset_marker": "generation-1" - }, - "source": { - "provider": "wiz", - "source_key": "synthetic-tenant/wiz" - }, - "actor": "synthetic-user", - "created_at": 1, - "item_count": 1, - "counts": { - "created": 1, - "updated": 0, - "unchanged": 0, - "error": 0 - }, - "results": [ - { - "client_id": "item-1", - "outcome": "created", - "source_report_id": "aif_4fb0fb6973a0ba27504f384bf21a7408bfca3a4b281cc7d11012e8cac270d591", - "observation_id": "aio_85b5b4200540ca44d31f8b8ca9ee8314461d55b8d94f5e97b35a832da08d9b81", - "canonical_finding_id": "acf_4fb0fb6973a0ba27504f384bf21a7408", - "version": 1, - "error": null - } - ], - "object": "security.finding_import" - } - ] -} diff --git a/sdk/typescript/tests-ts/github-findings.test.ts b/sdk/typescript/tests-ts/github-findings.test.ts index 47cb0098cb..860c640fef 100644 --- a/sdk/typescript/tests-ts/github-findings.test.ts +++ b/sdk/typescript/tests-ts/github-findings.test.ts @@ -1,11 +1,10 @@ -import { expect, spyOn, test } from "bun:test"; +import { expect, test } from "bun:test"; import { Octokit } from "@octokit/core"; import { runInNewContext } from "node:vm"; import { importGitHubCodeScanningAlerts } from "../src/github.js"; import { importGitHubAdvisories } from "../src/github-advisories.js"; import { validateExternalEvidence } from "../src/external-import-contract.js"; import type { ExternalFindings } from "../src/external-findings.js"; -import { previousReaderSelection } from "../src/reader-selection.js"; import { normalizeGitHubCodeScanningAlerts, normalizeGitHubAdvisories, @@ -17,10 +16,6 @@ import { importedAdvisory, } from "./support/github-import-fixtures.js"; -function priorSelection(result: ExternalFindings) { - return previousReaderSelection(result, structuredClone(result)); -} - function alertWith(fields: Record, ref?: string) { const row = importedAlert(ref); Object.assign(row.alert, fields); @@ -51,277 +46,6 @@ test("captures a record-root serializer with its selection index once", () => { const output = normalizeGitHubCodeScanningAlerts([row]); expect(output.findings[0]?.evidence.branch).toBe("refs/heads/0"); expect(keys).toEqual(["0"]); - const old = priorSelection(output); - expect(old?.sourceFindingIds).toHaveLength(1); - expect(old?.snapshot).toBeUndefined(); -}); - -test.each(["object", "class"])( - "flags hidden %s rule fields during replay", - (kind) => { - class Rule {} - const row = importedAlert(); - const rule = row.alert["rule"] as Record; - if (kind === "class") Object.setPrototypeOf(rule, Rule.prototype); - Object.defineProperty(rule, "name", { - enumerable: false, - value: "Hidden title", - }); - Object.defineProperty(rule, "security_severity_level", { - enumerable: false, - value: "critical", - }); - const current = normalizeGitHubCodeScanningAlerts([row]); - const id = current.findings[0]!.source_finding_id; - expect(current.findings[0]?.evidence.title).not.toBe("Hidden title"); - expect(current.findings[0]?.evidence.severity).toBe("unknown"); - const prior = priorSelection(current); - expect(prior?.snapshot).toBeUndefined(); - expect(prior?.uncertainFindingIds).toEqual([id]); - }, -); - -test("advisory membership with a hidden numeric getter is uncertain without reading it", () => { - const first = importedAdvisory("repository"), - second = importedAdvisory("global"); - let reads = 0; - const score = Object.defineProperty(new Number(7), "bad", { - enumerable: true, - get: () => { - reads++; - return Infinity; - }, - }); - Object.assign(second.advisory, { score }); - const current = normalizeGitHubAdvisories([first, second]); - expect(reads).toBe(0); - expect(current.findings).toHaveLength(2); - const prior = priorSelection(current); - expect(prior?.snapshot).toBeUndefined(); - expect(prior?.uncertainFindingIds).toEqual([ - current.findings[1]!.source_finding_id, - ]); - expect(prior?.knownFindings?.map((f) => f.source_finding_id)).toEqual([ - current.findings[0]!.source_finding_id, - ]); -}); - -test("later occurrence IDs can depend on the key passed to an earlier GitHub serializer", () => { - function alert(number: number) { - const row = importedAlert(); - row.number = number; - row.url = row.url.replace(/12$/, String(number)); - Object.assign(row.alert, { number, html_url: row.url }); - return row; - } - const stable = alert(11), - first = alert(12), - later = alert(13); - const instance = later.alert["most_recent_instance"] as Record< - string, - unknown - >; - Object.defineProperty(first, "toJSON", { - value(key: string) { - instance["ref"] = "refs/heads/" + key; - return { ...first }; - }, - }); - const current = normalizeGitHubCodeScanningAlerts([stable, first, later]); - expect(current.findings[2]?.evidence.branch).toBe("refs/heads/1"); - const prior = priorSelection(current); - expect(prior?.knownFindings?.map((row) => row.source_finding_id)).toEqual([ - current.findings[0]!.source_finding_id, - ]); - expect(prior?.uncertainFindingIds).toContain( - current.findings[2]!.source_finding_id, - ); - expect(prior?.unprovenOccurrenceCount).toBe(1); -}); - -test("a getter can mutate a later GitHub row before its historical evidence is observed", () => { - const first = importedAlert("refs/heads/first"); - const middle = importedAlert("refs/heads/middle"); - const last = importedAlert("refs/heads/last"); - const counter = { count: 0 }; - Object.assign(middle.alert, { - extension: { - get sample() { - return ++counter.count; - }, - }, - }); - Object.assign(last.alert, { extension: counter }); - const fresh = normalizeGitHubCodeScanningAlerts([first, middle, last]); - const prior = priorSelection(fresh); - expect(counter.count).toBe(1); - expect(prior?.sourceFindingIds).toEqual( - fresh.findings.map((row) => row.source_finding_id), - ); - expect(prior?.uncertainFindingIds).toEqual([ - fresh.findings[1]!.source_finding_id, - ]); - expect(prior?.knownFindings?.map((row) => row.source_finding_id)).toEqual([ - fresh.findings[0]!.source_finding_id, - ]); -}); - -test.each(["malformed details", "hidden ID", "unobservable ID"] as const)( - "retains the old scope when an advisory has %s", - (kind) => { - const first = importedAdvisory("repository"), - second = importedAdvisory("global"); - const oldId = normalizeGitHubAdvisories([second]).findings[0]! - .source_finding_id; - if (kind === "hidden ID") - Object.defineProperty(second, "ghsaId", { - value: second.ghsaId, - enumerable: false, - }); - else { - const old = second.ghsaId; - if (kind === "unobservable ID") - Object.defineProperty(second, "ghsaId", { - get: () => old, - enumerable: true, - }); - Object.defineProperty(second.advisory, "score", { - enumerable: true, - get: () => 7, - }); - Object.defineProperty(second.advisory, "toJSON", { - value: () => ({ ...second.advisory, vulnerabilities: "invalid" }), - }); - } - const fresh = normalizeGitHubAdvisories([first, second]); - expect(fresh.findings).toHaveLength(1); - const hints = priorSelection(fresh); - expect(hints?.snapshot).toBeUndefined(); - if (kind === "unobservable ID") - expect(hints?.unprovenOccurrenceCount).toBe(1); - else expect(hints?.uncertainFindingIds).toContain(oldId); - }, -); - -test("an inert toJSON field retains ordinary advisory selection semantics", () => { - const item = Object.assign(importedAdvisory(), { toJSON: null }); - const result = normalizeGitHubAdvisories([item]); - expect(result.findings).toHaveLength(1); - expect(priorSelection(result)).toBeUndefined(); -}); - -test("a caller-added report accompanies the earlier selection without changing its row binding", () => { - const row = alertWith({ bytes: Buffer.from([5, 17, 129]) }); - const result = normalizeGitHubCodeScanningAlerts([row]); - const input = { ...result, report: { run: "1" } }; - const exported = structuredClone(input); - const prior = previousReaderSelection(input, exported)?.snapshot; - expect(prior?.report).toEqual({ run: "1" }); - expect(sourceAlert(prior)).toMatchObject({ - bytes: Uint8Array.from([5, 17, 129]), - }); - input.report.run = "2"; - expect(prior?.report).toEqual({ run: "1" }); - expect(priorSelection(input)?.snapshot?.report).toEqual({ run: "2" }); - expect(priorSelection(result)?.snapshot?.report).toBeUndefined(); - expect( - previousReaderSelection(input, exported, null)?.snapshot, - ).toBeUndefined(); - expect( - previousReaderSelection(input, { ...exported, read: 2 })?.snapshot, - ).toBeUndefined(); -}); - -test("reader provenance distinguishes native mutations hidden by unchanged JSON", () => { - const item = alertWith({ - score: Object.assign(new Number(7), { toJSON: () => 9 }), - }); - const result = normalizeGitHubCodeScanningAlerts([item]); - const exported = structuredClone(result); - expect( - previousReaderSelection({ ...result }, exported)?.snapshot?.findings, - ).toHaveLength(1); - const alert = sourceAlert(result)!; - alert["score"] = Object.assign(new Number(8), { toJSON: () => 9 }); - expect(JSON.parse(JSON.stringify(result))).toEqual(exported); - expect(previousReaderSelection(result, exported)?.snapshot).toBeUndefined(); -}); - -test("locally normalized GitHub output retains the original Buffer for Cloud checkpoint verification", () => { - const row = alertWith({ payload: Buffer.from([5, 17, 129]) }); - const current = normalizeGitHubCodeScanningAlerts([row], "local"); - expect(sourceAlert(current)).toMatchObject({ - payload: { type: "Buffer", data: [5, 17, 129] }, - }); - const prior = priorSelection(current); - expect(sourceAlert(prior?.snapshot)).toMatchObject({ - payload: Uint8Array.from([5, 17, 129]), - }); -}); - -test("an oversized local export retains the small native GitHub selection", () => { - class Metric { - value = 7; - toJSON() { - return "x".repeat(300 * 1024); - } - } - const row = alertWith({ metric: new Metric() }); - const current = normalizeGitHubCodeScanningAlerts([row], "local"); - const evidence = current.findings[0]!.evidence; - expect(() => validateExternalEvidence(evidence, "cloud")).toThrow("256 KiB"); - const prior = priorSelection(current); - expect(sourceAlert(prior?.snapshot)).toMatchObject({ - metric: { value: 7 }, - }); -}); - -test("local reader provenance serializes selections one finding at a time", () => { - const first = alertWith({ payload: Buffer.from([5, 17]) }); - const stringify = spyOn(JSON, "stringify"); - try { - const current = normalizeGitHubCodeScanningAlerts( - [first, importedAlert("refs/heads/second")], - "local", - ); - const lookup = structuredClone(current); - expect( - previousReaderSelection(current, lookup)?.snapshot?.findings, - ).toHaveLength(2); - const batches = [current.findings, lookup.findings]; - const serializedBatch = stringify.mock.calls.some( - ([value]) => batches.includes(value) || batches.includes(value?.findings), - ); - expect(serializedBatch).toBe(false); - } finally { - stringify.mockRestore(); - } -}); - -test("a deeply shared graph excluded from historical evidence is not copied again for comparison", () => { - let graph: unknown = {}; - for (let level = 0; level < 21; level++) - graph = { left: graph, right: graph }; - class Extension { - graph = graph; - toJSON() { - return "summary"; - } - } - const row = alertWith({ extension: new Extension() }); - const clone = spyOn(globalThis, "structuredClone"); - try { - const output = normalizeGitHubCodeScanningAlerts([row]); - expect(sourceAlert(output)).toMatchObject({ - extension: "summary", - }); - expect(priorSelection(output)?.snapshot?.excluded[0]?.reason).toContain( - "depth", - ); - expect(clone.mock.calls.filter(([value]) => value === row)).toHaveLength(1); - } finally { - clone.mockRestore(); - } }); test.each([7, 2 ** 53])( @@ -393,13 +117,6 @@ test.each(["local", "cloud"] as const)( expect(() => normalizeGitHubCodeScanningAlerts(rows, target)).toThrow( "Duplicate source finding id", ); - if (target === "cloud") { - const old = priorSelection(read)?.snapshot; - expect(old?.findings[0]!.evidence.branch).toBe("refs/heads/main"); - expect(old?.findings[0]!.source_finding_id).not.toBe( - read.findings[0]!.source_finding_id, - ); - } }, ); @@ -462,7 +179,7 @@ test.each(["local", "cloud"] as const)( ); test.each(["local", "cloud"] as const)( - "checks emitted GitHub numbers and retains previous numeric exclusions on %s", + "checks numeric fields in the emitted GitHub JSON on %s", (target) => { function reason(value: unknown) { const row = alertWith({ metric: value }); @@ -476,22 +193,15 @@ test.each(["local", "cloud"] as const)( ]) expect(reason(value)).toContain("safe integer"); expect(reason({ toJSON: () => NaN })).toContain("finite JSON"); - for (const [metric, exported, previousReason] of [ - [Object.assign(new Number(7), { hidden: Infinity }), 7, "finite JSON"], - [{ x: 2 ** 53, toJSON: () => "summary" }, "summary", "safe integer"], + for (const [metric, exported] of [ + [Object.assign(new Number(7), { hidden: Infinity }), 7], + [{ x: 2 ** 53, toJSON: () => "summary" }, "summary"], ] as const) { const allowed = importedAlert("refs/heads/first"); const row = alertWith({ metric }); const result = normalizeGitHubCodeScanningAlerts([allowed, row], target); expect(result.excluded).toEqual([]); expect(sourceAlert(result, 1)).toMatchObject({ metric: exported }); - if (target === "cloud") { - const previous = priorSelection(result)?.snapshot; - expect(previous?.findings.map((f) => f.source_finding_id)).toEqual([ - result.findings[0]!.source_finding_id, - ]); - expect(previous?.excluded[0]?.reason).toContain(previousReason); - } } }, ); @@ -515,26 +225,7 @@ test.each(["cloud", "local"] as const)( }, ); -test("a data-field number's original native value survives only for verifying an earlier approval", () => { - const score = Object.assign(new Number(7), { toJSON: () => 9 }); - const row = alertWith({ score }); - const result = normalizeGitHubCodeScanningAlerts([row]); - const previous = priorSelection(result); - const saved = sourceAlert(previous?.snapshot)!; - expect( - previousReaderSelection({ ...result }, structuredClone(result))?.snapshot, - ).toBe(previous?.snapshot); - expect( - previousReaderSelection({ ...result }, { ...result, read: result.read + 1 }) - ?.snapshot, - ).toBeUndefined(); - expect(Number.prototype.valueOf.call(saved["score"])).toBe(7); - expect(sourceAlert(result)).toMatchObject({ - score: 9, - }); -}); - -test("uses the actual first wire and preserves only an occurrence hint for getter-dependent approvals", () => { +test("snapshots each GitHub row once when a shared getter changes", () => { const score = Object.assign(new Number(7), { toJSON: () => 9 }); let reads = 0; const shared = { @@ -564,27 +255,6 @@ test("uses the actual first wire and preserves only an occurrence hint for gette { score: 9, sample: 2 }, ]); expect(reads).toBe(3); - const prior = priorSelection(output); - expect(prior?.snapshot).toBeUndefined(); - expect(prior?.unprovenIdPrefixes).toEqual([ - "github.com/example/project/code-scanning/12/", - ]); -}); - -test("an exported row over the Cloud byte limit can have an exact historical selection", () => { - const raw = { score: new Number(1e-7), bytes: Buffer.from([1]), pad: "" }; - const row = alertWith({ raw }); - const read = () => normalizeGitHubCodeScanningAlerts([row]); - const size = Buffer.byteLength(JSON.stringify(read().findings[0]!.evidence)); - raw.pad = "x".repeat(256 * 1024 - size - 1); - expect(read().findings).toHaveLength(1); - raw.pad += "x"; - const current = read(); - expect(current.findings).toHaveLength(0); - expect(current.excluded[0]?.reason).toContain("256 KiB"); - const old = priorSelection(current)?.snapshot; - expect(old?.findings).toHaveLength(1); - expect(old?.excluded).toEqual([]); }); test("applies the Cloud byte limit to numbers returned by a vendor serializer", () => { @@ -961,58 +631,7 @@ test("Cloud rejects over-depth exported GitHub rows before reading shared leaves expect(reads).toBe(128); }); -test("an over-depth exported GitHub Buffer can retain its valid native selection", () => { - let graph: unknown = Buffer.from([7]); - for (let i = 0; i < 16; i++) graph = { next: graph }; - const row = alertWith({ extension: graph }); - - const previous = normalizeGitHubCodeScanningAlerts([structuredClone(row)]); - const current = normalizeGitHubCodeScanningAlerts([row]); - expect(current.excluded[0]?.reason).toBe( - "Evidence exceeds the Cloud JSON depth limit of 20.", - ); - expect(previous.findings).toHaveLength(1); - expect( - JSON.stringify(previousReaderSelection(current, current)?.snapshot), - ).toBe(JSON.stringify(previous)); -}); - -test("an over-depth GitHub export cannot prove a custom serializer's old byte limit", () => { - for (const oldPayload of [{ tiny: true }, { pad: "x".repeat(256 * 1024) }]) { - const row = importedAlert(); - let graph: unknown = 1; - for (let i = 0; i < 24; i++) graph = { next: graph }; - const keys: string[] = []; - Object.defineProperty(row, "toJSON", { - value(key: string) { - keys.push(key); - return key === "source_data" ? oldPayload : { graph }; - }, - }); - const current = normalizeGitHubCodeScanningAlerts([row]); - expect(keys).toEqual(["0"]); - expect(current.excluded[0]?.reason).toContain("depth limit"); - expect(previousReaderSelection(current, current)?.snapshot).toBeUndefined(); - expect( - previousReaderSelection(current, current)?.sourceFindingIds, - ).toHaveLength(1); - } -}); - -test("discarded native Buffer fields cannot establish an old GitHub selection", () => { - const buffer = Buffer.from([7]); - let extra: unknown = 7, - graph: unknown = buffer; - for (let i = 0; i < 24; i++) extra = { next: extra }; - Object.assign(buffer, { extra }); - for (let i = 0; i < 16; i++) graph = { next: graph }; - const row = alertWith({ extension: graph }); - const current = normalizeGitHubCodeScanningAlerts([row]); - expect(current.excluded[0]?.reason).toContain("depth limit"); - expect(previousReaderSelection(current, current)?.snapshot).toBeUndefined(); -}); - -test("historical GitHub mapping cannot use a native clone with a hidden array index", () => { +test("reads non-enumerable entries in a GitHub rule tags array", () => { const row = importedAlert(); const tags = Object.defineProperty([], "0", { value: "external/cwe/cwe-089", @@ -1022,11 +641,6 @@ test("historical GitHub mapping cannot use a native clone with a hidden array in expect(result.findings[0]?.evidence.details?.weakness_ids).toEqual([ "CWE-89", ]); - const previous = priorSelection(result); - expect(previous?.snapshot).toBeUndefined(); - expect(previous?.uncertainFindingIds).toEqual([ - result.findings[0]!.source_finding_id, - ]); }); test("a failed GitHub serializer retains an own URL without reading a URL getter", () => { diff --git a/sdk/typescript/tests-ts/inspector-findings.test.ts b/sdk/typescript/tests-ts/inspector-findings.test.ts index 863553e651..4669eae6e0 100644 --- a/sdk/typescript/tests-ts/inspector-findings.test.ts +++ b/sdk/typescript/tests-ts/inspector-findings.test.ts @@ -1,4 +1,3 @@ -import { previousReaderSelection } from "../src/reader-selection.js"; import { expect, test } from "bun:test"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -8,7 +7,6 @@ import type { ExternalImportTarget } from "../src/external-import-contract.js"; import { inspectorReportRepository, normalizeInspectorFindings, - normalizePreviousInspectorFindings, readInspectorFindings, } from "../src/inspector-findings.js"; import { @@ -107,12 +105,6 @@ test("related vulnerabilities stay in native evidence even when they exceed the native.packageVulnerabilityDetails.vulnerabilityId, ]); expect(findings[0]!.evidence.source_data?.["finding"]).toEqual(native); - const previous = normalizePreviousInspectorFindings( - report, - inspectorRepositoryUrl, - ); - expect(previous.findings).toHaveLength(0); - expect(previous.excluded[0]?.reason).toContain("advisory_ids"); }); test.each([ @@ -235,12 +227,6 @@ test("projects only repository-relative code locations with a matching filename" evidence(codeFinding("src/example.ts", "example.ts", 13)).details?.code ?.end_line ?? null, ).toBeNull(); - expect( - normalizePreviousInspectorFindings( - { findings: [code] }, - inspectorRepositoryUrl, - ).findings[0]!.evidence.details, - ).toBeUndefined(); expect(evidence(code).locations).toEqual([ { path: "src/example.ts", line: 8 }, ]); @@ -279,12 +265,9 @@ test("oversized optional SAST fields preserve the complete Inspector finding", ( test("SAST details do not displace native evidence at the Cloud byte limit", () => { const native = { ...codeFinding(), padding: "" }; - const previous = normalizePreviousInspectorFindings( - { findings: [native] }, - inspectorRepositoryUrl, - ); + const { details: _, ...base } = normalize([native]).findings[0]!.evidence; native.padding = " ".repeat( - 262_144 - Buffer.byteLength(JSON.stringify(previous.findings[0]!.evidence)), + 262_144 - Buffer.byteLength(JSON.stringify(base)), ); expect( normalize([native]).findings[0]!.evidence.source_data?.["finding"], @@ -676,64 +659,6 @@ test("only surviving rows share project proof, independently of input order", () } }); -function scopedProjectRows() { - const witness = projectRow("a", enterprise, "GITHUB"); - const dependent = projectRow("b"); - const independent = projectRow("c", enterprise, "GITHUB"); - Object.assign(independent.resources[0]!.details.codeRepository, { - integrationArn: undefined, - }); - const rows = [witness, dependent, independent]; - for (const row of rows) row.resources[0]!.id = "example/project"; - return { rows, witness, dependent, independent }; -} - -test.each([ - [0.00001, 262_145, false], - [1e-7, 262_144, true], -] as const)( - "a size change rechecks provider proof for each selected subset (%d)", - (score, bytes, previouslyAccepted) => { - const { rows, witness, independent } = scopedProjectRows(); - Object.assign(witness, { score: new Number(score), pad: "" }); - const evidence = normalize(rows, enterprise).findings[0]!.evidence; - Object.assign(witness, { - pad: "p".repeat(bytes - Buffer.byteLength(JSON.stringify(evidence))), - }); - const current = normalize(rows, enterprise); - const previous = previousReaderSelection( - current, - JSON.parse(JSON.stringify(current)), - ); - const all = rows.map((row) => row.findingArn); - const onlyIndependent = [independent.findingArn]; - expect(current.findings.map((row) => row.source_finding_id)).toEqual( - previouslyAccepted ? onlyIndependent : all, - ); - expect( - previous?.snapshot?.findings.map((row) => row.source_finding_id), - ).toEqual(previouslyAccepted ? all : onlyIndependent); - }, -); - -test("an uncertain Inspector row can obscure the membership of its dependent", () => { - const { rows, witness, dependent, independent } = scopedProjectRows(); - Object.assign(witness, { score: new Number(NaN) }); - const current = normalize(rows, enterprise); - const prior = previousReaderSelection( - current, - JSON.parse(JSON.stringify(current)), - ); - expect(prior?.uncertainFindingIds).toEqual([ - witness.findingArn, - dependent.findingArn, - ]); - expect(prior?.knownFindings?.map((row) => row.source_finding_id)).toEqual([ - independent.findingArn, - ]); - expect(prior?.snapshot).toBeUndefined(); -}); - test("after the only provider hint is excluded, valid local rows still share their native URL", () => { const source = "https://scm.example.test/Owner/Repo"; const duplicate = projectRow("a", "Owner/Repo", "GITHUB"); diff --git a/sdk/typescript/tests-ts/opengrep-findings.test.ts b/sdk/typescript/tests-ts/opengrep-findings.test.ts index 5429f75ea0..dd94e24e44 100644 --- a/sdk/typescript/tests-ts/opengrep-findings.test.ts +++ b/sdk/typescript/tests-ts/opengrep-findings.test.ts @@ -7,7 +7,6 @@ import { normalizeOpenGrepFindings as normalize, readOpenGrepFindings, } from "../src/opengrep-findings.js"; -import { previousReaderSelection } from "../src/reader-selection.js"; import { opengrepJson, opengrepTraceJson, @@ -175,80 +174,6 @@ test("grouped observations still honor the existing Cloud evidence size contract ); }); -test.each(["group", "trace"] as const)( - "a size change affecting a %s keeps the unrelated occurrence proven", - (changed) => { - const input = opengrepTraceJson(); - const second = input.results[1]!; - if (changed === "trace") second.extra.message = "x".repeat(140_000); - // The released reader counted a boxed decimal by its JSON spelling. - Object.assign(second.extra.metadata, { - score: new Number(0.00001), - pad: "", - }); - const group = normalize(input, options).findings[0]!; - const bytes = Buffer.byteLength(JSON.stringify(group.evidence)); - Object.assign(second.extra.metadata, { pad: "p".repeat(262_145 - bytes) }); - const unrelated = structuredClone(input.results[0]!); - unrelated.check_id = "example.unrelated"; - input.results.push(unrelated); - const current = normalize(input, options); - expect(current.excluded).toEqual([]); - expect(current.findings).toHaveLength(2); - const known = current.findings.filter( - (row) => row.source_finding_id !== group.source_finding_id, - ); - expect(known).toHaveLength(1); - const previous = previousReaderSelection( - current, - JSON.parse(JSON.stringify(current)), - ); - expect(previous?.uncertainFindingIds).toEqual([group.source_finding_id]); - expect(previous?.knownFindings).toEqual(known); - const local = normalize(input, options, "local"); - expect( - previousReaderSelection(local, JSON.parse(JSON.stringify(local))), - ).toBeUndefined(); - }, -); - -test.each([ - { score: 0.00001, bytes: 262_145, freshCount: 2, priorCount: 1 }, - { score: 1e-7, bytes: 262_144, freshCount: 1, priorCount: 2 }, -])( - "retains exact singleton membership at $bytes bytes", - ({ score, bytes, freshCount, priorCount }) => { - const input = opengrepJson(); - const stable = normalize(input, options).findings[0]!; - const candidate = { - ...input.results[0]!, - check_id: "example.boundary", - score: new Number(score), - padding: "", - }; - const selected = normalize({ ...input, results: [candidate] }, options) - .findings[0]!; - candidate.padding = "x".repeat( - bytes - Buffer.byteLength(JSON.stringify(selected.evidence)), - ); - input.results.push(candidate); - const current = normalize(input, options); - expect(current.findings).toHaveLength(freshCount); - const prior = previousReaderSelection( - current, - JSON.parse(JSON.stringify(current)), - ); - const expectedIds = - priorCount === 1 - ? [stable.source_finding_id] - : [stable.source_finding_id, selected.source_finding_id].sort(); - expect( - prior?.snapshot?.findings.map((row) => row.source_finding_id), - ).toEqual(expectedIds); - expect(prior?.uncertainFindingIds).toBeUndefined(); - }, -); - test("an oversized trace leaves the publishable traces at its source occurrence", () => { const input = opengrepJson(); const valid = input.results[0]!; diff --git a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts index f14166358b..c29322d7f5 100644 --- a/sdk/typescript/tests-ts/semgrep-findings-local.test.ts +++ b/sdk/typescript/tests-ts/semgrep-findings-local.test.ts @@ -89,7 +89,7 @@ test("a marked Platform report accepts the Semgrep and OpenGrep readers' ordinar const input = { ...ordinary, ...options, - sourceFormat: "semgrep-platform-findings/v1", + sourceFormat: "semgrep-platform-findings/v2", semgrepPlatformScope: { deployment: "example", projectId: "14", diff --git a/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts b/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts index 54d0fbdfd3..bcf9d4c1d2 100644 --- a/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts +++ b/sdk/typescript/tests-ts/semgrep-platform-collector.test.ts @@ -108,14 +108,6 @@ test.each(["%23", "%3F"])( }); expect(semgrepPlatformFindingRepository(row, "semgrep")).toBe(native.url); } - const legacy = normalizeSemgrepPlatformFindings({ - ...report, - format: "semgrep-platform-findings/v1", - project: native, - findings: rows, - }); - expect(legacy.excluded.map((item) => item.position)).toEqual([2]); - expect(legacy.findings).toEqual([batch.findings[0]!]); }, ); @@ -148,14 +140,9 @@ test("a supplied empty repository URL remains bulk-only on capture and replay", format: "semgrep-platform-selection/v1", }), ).toThrow(/repository/); - for (const input of [ - bulk, - { ...bulk, format: "semgrep-platform-findings/v1", findings: [row] }, - ]) { - const [finding] = normalizeSemgrepPlatformFindings(input).findings; - expect(finding!.evidence.source_data?.["format"]).toBe(bulk.format); - expect(semgrepPlatformFindingRepository(finding!, "semgrep")).toBe(url); - } + const [finding] = normalizeSemgrepPlatformFindings(bulk).findings; + expect(finding!.evidence.source_data?.["format"]).toBe(bulk.format); + expect(semgrepPlatformFindingRepository(finding!, "semgrep")).toBe(url); }); test("an opaque bulk finding ID still requires the original native occurrence ID", async () => { @@ -485,7 +472,7 @@ test("a normalized Platform report binds native rows even when both projects hav expect((await prepare(batch)).preview.findings).toHaveLength(2); }); -test("a marked Platform report requires scope for caller-managed partial v1 findings", async () => { +test("a marked Platform report requires scope for caller-managed partial native findings", async () => { const native = { id: 9, rule_name: "example.rule", @@ -495,11 +482,12 @@ test("a marked Platform report requires scope for caller-managed partial v1 find }; const report = normalizeSemgrepPlatformFindings({ ...options, - project, + project: { ...project, id: String(project.id) }, product: "sast", - format: "semgrep-platform-findings/v1", + format: "semgrep-platform-findings/v2", + repositoryAliases: [], findingCount: 1, - findings: [native], + findings: [JSON.stringify(native)], }); const partial = structuredClone(report.findings[0]!); partial.source_finding_id = "caller-managed"; @@ -571,77 +559,6 @@ test("a marked Platform report requires scope for caller-managed partial v1 find } }); -test("a v1 normalized report binds its object rows and empty results to the declared checkout", async () => { - const source = { provider: "semgrep" as const, sourceKey: "example/topic" }; - await using fixture = await localReportFixture("v1.json", source, { - remote: url, - }); - const prepare = ( - input: Parameters[0], - provider: "semgrep" | "wiz" = "semgrep", - ) => - prepareNormalizedLocalImport( - input, - { ...source, provider, repository: fixture.repository }, - { environment: fixture.environment }, - ); - - const native = { id: 9, rule_name: "example.rule" }; - const scope = { - deployment: options.deployment, - ref: options.ref, - project, - product: "sast", - format: "semgrep-platform-findings/v1", - }; - const projected = normalizeSemgrepPlatformFindings({ - ...scope, - findingCount: 1, - findings: [native], - }).findings[0]!; - projected.evidence.source_data = { ...scope, finding: native }; - // This is the envelope returned by the v1 reader, including when it accepted no rows. - const legacy = { - sourceRepositoryUrl: url, - read: 1, - findings: [projected], - excluded: [], - }; - const other = "https://github.com/example/fork"; - expect((await prepare(legacy)).preview.findings).toHaveLength(1); - const rowOnly = { ...legacy, sourceRepositoryUrl: undefined }; - expect((await prepare(rowOnly)).preview.findings).toHaveLength(1); - for (const findings of [[projected], []]) { - const wrong = { - ...legacy, - read: findings.length, - findings, - sourceRepositoryUrl: other, - }; - await expect(prepare(wrong)).rejects.toThrow(/repository|remote/i); - } - const ordinary = structuredClone(projected); - ordinary.source_finding_id = "generic"; - delete ordinary.evidence.source_data; - const foreign = structuredClone(ordinary); - foreign.evidence.details!.repository!.url = other; - await expect( - prepare({ ...legacy, read: 2, findings: [projected, foreign] }), - ).rejects.toThrow(/report repository/i); - expect( - (await prepare({ ...legacy, findings: [ordinary] }, "wiz")).preview - .findings, - ).toHaveLength(1); - const sarif = normalizeSemgrepFindings( - semgrepSarif(), - { sourceRepositoryUrl: url }, - "local", - ); - expect( - (await prepare({ ...legacy, ...sarif })).preview.findings, - ).toHaveLength(1); -}); - test("hosted repository comparison accepts an opaque Git remote alias", () => { expect(hostedRepositoryUrlKey("invalid alias")).toBe("invalid alias"); }); @@ -860,35 +777,19 @@ async function checkProjectMapping(raw: string | null) { expect(saved.nativeProjectUrl).toBe(raw ?? undefined); expect(saved.ref).toBe(options.ref); expect(saved).toMatchObject({ sourceRepositoryUrl: url }); - const legacy = { - deployment: options.deployment, - ref: options.ref, - sourceRepositoryUrl: url, - product: "sast", - findingCount: 1, - format: "semgrep-platform-findings/v1", - project: { ...saved.project, url: raw }, - findings: [row], - }; await using fixture = await localReportFixture( "semgrep.json", { provider: "semgrep", sourceKey: "example/topic" }, { remote: url }, ); - for (const report of [saved, legacy]) { - await fixture.writeJson(report); - const prepared = await fixture.prepare(); - expect(prepared.preview.findings[0]?.evidence).toMatchObject({ - severity: raw ? "unknown" : "informational", - details: { repository: { url: resolved } }, - source_data: { project: { url: raw }, sourceRepositoryUrl: url }, - }); - expect((await prepared.import()).counts).toEqual( - report === saved - ? { created: 1, updated: 0, unchanged: 0 } - : { created: 0, updated: 0, unchanged: 1 }, - ); - } + await fixture.writeJson(saved); + const prepared = await fixture.prepare(); + expect(prepared.preview.findings[0]?.evidence).toMatchObject({ + severity: raw ? "unknown" : "informational", + details: { repository: { url: resolved } }, + source_data: { project: { url: raw }, sourceRepositoryUrl: url }, + }); + expect((await prepared.import()).counts.created).toBe(1); if (raw) expect(() => normalizeSemgrepFindings( @@ -997,7 +898,7 @@ test("replays a direct occurrence with a collector-style SSH project binding", ( ); }); -test("matches GitHub project-path aliases across casing in collected, v1 and normalized rows", async () => { +test("matches GitHub project-path aliases across casing in collected and normalized rows", async () => { const name = "semgrep-logical/Prod"; const native = { name, @@ -1025,12 +926,6 @@ test("matches GitHub project-path aliases across casing in collected, v1 and nor } const saved = await collect(native.url); expect(saved.repositoryAliases).toEqual([row.repository.name]); - const { repositoryAliases: _aliases, ...v1 } = saved; - const legacy = { - ...v1, - format: "semgrep-platform-findings/v1", - findings: [row], - }; // The Semgrep project label and any witnessed display label retain their original casing. const named = (name: string) => ({ ...row, repository: { name } }); const exact = await collect(native.url, [named("Prod")]); @@ -1070,31 +965,27 @@ test("matches GitHub project-path aliases across casing in collected, v1 and nor } const custom = "https://git.example.test/Example/Service"; await expect(collect(custom)).rejects.toThrow(/repository/); - for (const report of [saved, legacy]) { - const finding = normalizeSemgrepFindings(report, {}, "local").findings[0]!; - expect(semgrepPlatformFindingRepository(finding, "semgrep")).toBe( - native.url, - ); - expect( - finding.evidence.source_data?.["repositoryWitnessJson"], - ).toBeUndefined(); - const mapped = { ...report, project: { ...report.project, url: custom } }; - expect(() => normalizeSemgrepFindings(mapped, {}, "local")).toThrow( - /repository/, - ); - const options = { - sourceRepositories: [{ url: custom, platform: "github" as const }], - }; - const verified = normalizeSemgrepFindings(mapped, options, "local") - .findings[0]!; - expect( - semgrepPlatformFindingRepository( - verified, - "semgrep", - options.sourceRepositories, - ), - ).toBe(custom); - } + const finding = normalizeSemgrepFindings(saved, {}, "local").findings[0]!; + expect(semgrepPlatformFindingRepository(finding, "semgrep")).toBe(native.url); + expect( + finding.evidence.source_data?.["repositoryWitnessJson"], + ).toBeUndefined(); + const mapped = { ...saved, project: { ...saved.project, url: custom } }; + expect(() => normalizeSemgrepFindings(mapped, {}, "local")).toThrow( + /repository/, + ); + const mapping = { + sourceRepositories: [{ url: custom, platform: "github" as const }], + }; + const verified = normalizeSemgrepFindings(mapped, mapping, "local") + .findings[0]!; + expect( + semgrepPlatformFindingRepository( + verified, + "semgrep", + mapping.sourceRepositories, + ), + ).toBe(custom); }); test("a URL-less SAST row uses its Git path for selection and keeps other verified labels in the full report", () => { @@ -1246,22 +1137,13 @@ test("blank display names replay when a native ID or URL binds the repository", responses(new Response(page)), ); expect(saved.repositoryAliases).toEqual([]); - const { repositoryAliases: _aliases, ...legacy } = saved; - const prior = { - ...legacy, - format: "semgrep-platform-findings/v1", - findings, - project, - }; - for (const report of [saved, prior]) { - const normalized = normalizeSemgrepFindings(report, {}, "local"); - expect(normalized.excluded).toEqual([]); - expect( - normalized.findings.map((row) => - semgrepPlatformFindingRepository(row, "semgrep"), - ), - ).toEqual([url, url, url]); - } + const normalized = normalizeSemgrepFindings(saved, {}, "local"); + expect(normalized.excluded).toEqual([]); + expect( + normalized.findings.map((row) => + semgrepPlatformFindingRepository(row, "semgrep"), + ), + ).toEqual([url, url, url]); const selected = { ...options, projectResponse: JSON.stringify({ project }), @@ -1730,133 +1612,6 @@ test("CLI checks the token and its origin before fetching", async () => { expect(fetch).not.toHaveBeenCalled(); }); -test("saved v1 findings keep exact source IDs and original JSON numeric types on local replay", async () => { - const ids = ["9007199254740992", "9007199254740993"]; - const tokens = [ids[0]!, "9.007199254740993e15"]; - const row = { - rule: { name: "example.rule" }, - ref: options.ref, - repository: { name: "service" }, - }; - const v1 = { - format: "semgrep-platform-findings/v1", - deployment: options.deployment, - project: { name: project.name, id: ids[1], url: null }, - sourceRepositoryUrl: url, - ref: options.ref, - product: "sast", - findingCount: 2, - findings: tokens.map((id) => ({ - ...row, - id, - vendor: { score: ["1.00000000000000001"], label: id }, - })), - numericFields: { - project: { id: ids[1] }, - findings: Object.fromEntries( - tokens.map((id, i) => [ - i, - { id, vendor: { score: { 0: "1.00000000000000001" } } }, - ]), - ), - }, - }; - await using fixture = await localReportFixture( - "semgrep.json", - { - provider: "semgrep", - sourceKey: "example/topic", - }, - { remote: url }, - ); - await fixture.writeJson(v1); - const prepared = await fixture.prepare(); - const { preview } = prepared; - expect(preview.excluded).toEqual([]); - const numericRows = ids.map( - (id) => - `{"id":${id},"rule":{"name":"example.rule"},"ref":"${options.ref}","repository":{"url":"${url}"}}`, - ); - const selection = selectSemgrepPlatformFindings({ - deployment: options.deployment, - ref: options.ref, - projectResponse: `{"project":{"name":"${project.name}","id":${ids[1]},"url":"${url}"}}`, - findingsResponses: [`{"findings":[${numericRows.join(",")}]}`], - findingIds: ids, - }); - const selected = normalizeSemgrepFindings( - selection, - { sourceRepositoryUrl: url }, - "local", - ); - const sourceIds = (rows: typeof preview.findings) => - rows.map((row) => row.source_finding_id); - expect(new Set(sourceIds(preview.findings)).size).toBe(2); - expect(sourceIds(preview.findings)).toEqual(sourceIds(selected.findings)); - for (const [i, finding] of preview.findings.entries()) { - const raw = finding.evidence.source_data?.["semgrepPlatformFindingJson"]; - expect(raw).toContain(`"id":${ids[i]}`); - expect(raw).toContain('"score":[1.00000000000000001]'); - expect(raw).toContain(`"label":"${tokens[i]}"`); - } - expect((await prepared.import()).counts).toEqual({ - created: 2, - updated: 0, - unchanged: 0, - }); - expect((await (await fixture.prepare()).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 2, - }); - const v2 = { - ...v1, - format: "semgrep-platform-findings/v2", - repositoryAliases: ["service"], - findings: ids.map( - (id, i) => - `{"id":${id},"rule":{"name":"example.rule"},"ref":"${options.ref}","repository":{"name":"service"},"vendor":{"score":[1.00000000000000001],"label":"${tokens[i]}"}}`, - ), - }; - for (const equivalent of [v2, v1]) { - await fixture.writeJson(equivalent); - expect((await (await fixture.prepare()).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 2, - }); - } - await fixture.writeJson({ - ...v1, - numericFields: { findings: { 0: { id: ids[1] } } }, - }); - await expect(fixture.prepare()).rejects.toThrow(/exact scanner number/); - await fixture.writeJson({ - ...v1, - findings: v1.findings.map((row) => ({ - ...row, - repository: { name: "other" }, - })), - }); - await expect(fixture.prepare()).rejects.toThrow(/repository/); - await fixture.writeJson({ - ...v1, - project: { ...v1.project, url: "https://github.com/other/service" }, - }); - await expect(fixture.prepare()).rejects.toThrow(/repository/); - expect(() => - normalizeSemgrepFindings( - { - ...v1, - project: { ...v1.project, url: "git@git.example.test:team/service" }, - sourceRepositoryUrl: "https://git.example.test/team/service", - }, - {}, - "local", - ), - ).toThrow(/HTTP\(S\) repository/); -}); - test("a verified alias remains replayable with or without a reader platform hint", async () => { const remote = "https://git.example.test/Team/Repo"; const repository = { name: "repo" }; diff --git a/sdk/typescript/tests-ts/semgrep-platform-normalization.test.ts b/sdk/typescript/tests-ts/semgrep-platform-normalization.test.ts new file mode 100644 index 0000000000..1c971eb8e8 --- /dev/null +++ b/sdk/typescript/tests-ts/semgrep-platform-normalization.test.ts @@ -0,0 +1,303 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { semgrepJson } from "./support/semgrep-import-fixtures.js"; +import { prepareLocalExternalImport } from "../src/external-findings-import.js"; +import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; +import { isolatedGitReportFixture as localReportFixture } from "./external-findings-local-helpers.js"; +import { + normalizeSemgrepFindings, + readSemgrepFindings, +} from "../src/semgrep-findings.js"; +import { normalizeSemgrepPlatformFindings } from "../src/semgrep-platform-findings.js"; + +const url = "https://github.com/example/project"; +const scope = { + deployment: "demo", + project: { id: "10", name: "example/project", url }, + ref: "refs/heads/topic", + product: "sast", +}; +const code = { + ref: scope.ref, + repository: { url }, + rule: { name: "fixture.rule" }, +}; + +test("the standalone Cloud normalizer verifies the port on reader mappings", () => { + const projectUrl = "https://scm.example.test:443/Example/Project"; + const report = { + ...scope, + format: "semgrep-platform-selection/v1", + project: { ...scope.project, url: projectUrl }, + findings: [ + JSON.stringify({ ...code, id: 12, repository: { url: projectUrl } }), + ], + }; + const mapped = (url: string) => ({ + sourceRepositories: [{ url, platform: "github" as const }], + }); + const different = mapped(projectUrl.replace(":443", "")); + expect(() => + normalizeSemgrepPlatformFindings(report, different, "cloud"), + ).toThrow(/reader's source repository/); + expect( + normalizeSemgrepPlatformFindings(report, different).findings, + ).toHaveLength(1); + expect( + normalizeSemgrepPlatformFindings( + report, + mapped(projectUrl.toLowerCase()), + "cloud", + ).findings, + ).toHaveLength(1); +}); + +test("strongly bound current SAST rows have identical selection and backlog exclusions", () => { + const location = { file_path: "src/service.ts", line: 2, end_line: 3 }; + const rows = [ + { ...code, id: 11, severity: "unrecognized" }, + { ...code, id: 12, location: { ...location, line: "2" } }, + { ...code, id: 13, location: { ...location, end_line: "3" } }, + ]; + const read = (format: string, findings: unknown[]) => + normalizeSemgrepFindings( + { + ...scope, + format, + findingCount: findings.length, + findings, + repositoryAliases: [], + }, + {}, + "local", + ); + const json = rows.map((row) => JSON.stringify(row)); + const selection = read("semgrep-platform-selection/v1", json); + const backlog = read("semgrep-platform-findings/v2", json); + expect(selection.excluded.map((row) => row.position)).toEqual([2, 3]); + expect(backlog.excluded).toEqual(selection.excluded); + expect(backlog.findings).toEqual(selection.findings); + expect(selection.findings).toHaveLength(1); + const evidence = selection.findings[0]!.evidence; + expect(evidence.severity).toBe("unknown"); + expect( + JSON.parse(String(evidence.source_data?.["semgrepPlatformFindingJson"])) + .severity, + ).toBe("unrecognized"); + const weak = read( + "semgrep-platform-findings/v2", + rows.map((row) => JSON.stringify({ ...row, ref: undefined })), + ); + expect(weak.excluded).toEqual([]); + expect(weak.findings[0]!.evidence.severity).toBe("unknown"); + expect(weak.findings[1]!.evidence.locations?.[0]?.line).toBe(2); + expect(weak.findings[2]!.evidence.details!.code!.end_line).toBe(3); +}); + +test("a GitHub mapping reprojects generated severity when a backlog finding becomes selectable", async () => { + const url = "https://git.example.test/Team/Repo"; + const native = { + ...code, + id: 12, + repository: { id: 10, name: "team/repo" }, + severity: "informational", + }; + const rows = [ + native, + { ...native, id: 13, location: { file_path: "src/service.ts", line: "3" } }, + { + ...native, + id: 14, + location: { file_path: "src/service.ts", line: 3, end_line: "5" }, + }, + ]; + const report = { + ...scope, + project: { ...scope.project, name: "Project", url }, + format: "semgrep-platform-findings/v2", + repositoryAliases: [native.repository.name], + findingCount: rows.length, + findings: rows.map((row) => JSON.stringify(row)), + }; + const batch = normalizeSemgrepFindings(report, {}, "local"); + expect(batch.findings[0]!.evidence).toMatchObject({ + severity: "informational", + source_data: { format: report.format }, + }); + const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; + await using fixture = await localReportFixture("semgrep.json", source, { + remote: url, + }); + const options = { ...source, repository: fixture.repository }; + const dependencies = { + environment: fixture.environment, + sourceRepositories: [{ url, platform: "github" as const }], + }; + await fixture.writeJson(report); + const file = await prepareLocalExternalImport( + fixture.file, + options, + dependencies, + ); + const direct = () => + prepareNormalizedLocalImport(batch, options, dependencies); + expect(file.preview.findings[0]!.evidence).toMatchObject({ + severity: "unknown", + source_data: { format: "semgrep-platform-selection/v1" }, + }); + for (const row of file.preview.findings.slice(1)) { + expect(row.evidence).toMatchObject({ + severity: "informational", + locations: [{ path: "src/service.ts", line: 3 }], + source_data: { format: report.format }, + }); + } + expect(file.preview.excluded).toEqual([]); + expect((await file.import()).counts.created).toBe(rows.length); + expect((await (await direct()).import()).counts.unchanged).toBe(rows.length); + expect((await file.import()).counts.unchanged).toBe(rows.length); + expect((await fixture.read()).map((row) => row.version)).toEqual([1, 1, 1]); + batch.findings[0]!.evidence.severity = "high"; + expect((await direct()).preview.findings[0]!.evidence.severity).toBe("high"); +}); + +test.each(["sca", "weak SAST", "selected SAST"])( + "%s file and normalized SDK replays retain equivalent native number values once", + async (kind) => { + const strong = kind === "selected SAST"; + const product = kind === "sca" ? "sca" : "sast"; + const id = strong ? "12" : "1.2e1"; + const repository = strong ? JSON.stringify({ id: 10, url }) : '{"id":1e1}'; + const ref = strong ? `,"ref":${JSON.stringify(scope.ref)}` : ""; + const rule = product === "sast" ? ',"rule_name":"example.rule"' : ""; + const native = `{"id":${id}${rule},"repository":${repository},"metadata":{"source":"demo","score":10.0}${ref}}`; + const replayRepository = strong ? repository : '{"id":10.0}'; + const equivalent = `{"metadata":{"score":1e1,"source":"demo"},"repository":${replayRepository},"id":12${rule}${ref}}`; + const report = (row: string) => ({ + ...scope, + format: "semgrep-platform-findings/v2", + product, + repositoryAliases: [], + findingCount: 1, + findings: [row], + }); + const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; + await using fixture = await localReportFixture("semgrep.json", source, { + remote: url, + }); + await fixture.writeJson(report(native)); + const initial = await fixture.prepare(); + expect((await initial.import()).counts.created).toBe(1); + const result = normalizeSemgrepFindings(report(native), {}, "local"); + const data = result.findings[0]!.evidence.source_data!; + const prepare = () => fixture.prepareNormalized(result); + const unchanged = { created: 0, updated: 0, unchanged: 1 }; + data["semgrepPlatformFindingJson"] = equivalent; + expect((await (await prepare()).import()).counts).toEqual(unchanged); + await fixture.writeJson(report(equivalent)); + expect((await (await fixture.prepare()).import()).counts).toEqual( + unchanged, + ); + const [stored] = await fixture.read(); + expect(stored?.version).toBe(1); + if (!strong) { + const raw = stored!.evidence.source_data!["semgrepPlatformFindingJson"]; + expect(raw).toContain('"id":12,'); + expect(raw).toContain('"repository":{"id":10}'); + } + + data["callerAnnotation"] = { group: "demo" }; + expect((await (await prepare()).import()).counts.updated).toBe(1); + expect((await (await prepare()).import()).counts).toEqual(unchanged); + expect( + (await fixture.read())[0]?.evidence.source_data?.["callerAnnotation"], + ).toEqual({ group: "demo" }); + }, +); + +test("the retained repository witness stays stable when the API returns rows in another order", () => { + const name = "display"; + const dependent = { ...code, id: 12, repository: { name } }; + const byId = { ...code, id: 13, repository: { name, id: 10 } }; + const byUrl = { ...code, id: 14, repository: { name, url } }; + const project = (witnesses: (typeof byId | typeof byUrl)[]) => + normalizeSemgrepFindings( + { + ...scope, + format: "semgrep-platform-findings/v2", + findingCount: 3, + repositoryAliases: [name], + findings: [dependent, ...witnesses].map((row) => JSON.stringify(row)), + }, + {}, + "local", + ).findings[0]!; + const first = project([byId, byUrl]); + expect(first.evidence.source_data?.["repositoryWitnessJson"]).toBeDefined(); + expect(project([byUrl, byId])).toEqual(first); +}); + +test("v2 files require exact counts and string project IDs", async () => { + const dir = await mkdtemp(join(tmpdir(), "semgrep-v2-controls-")); + const path = join(dir, "report.json"); + const report = (projectId: string, count: string, rows: string[]) => + `{"deployment":"demo","project":{"id":${projectId},"name":"example/project","url":"${url}"},"ref":"refs/heads/topic","product":"sast","repositoryAliases":[],"findingCount":${count},"format":"semgrep-platform-findings/v2","metadata":{"findingCount":1},"findings":${JSON.stringify(rows)}}`; + const rows = [JSON.stringify({ ...code, id: 12 })]; + try { + await writeFile(path, report('"10"', "1e0", rows)); + expect( + (await readSemgrepFindings(path, {}, "local")).findings, + ).toHaveLength(1); + await writeFile(path, report('"10"', "0", [])); + expect((await readSemgrepFindings(path, {}, "local")).findings).toEqual([]); + // An escaped format name is still the same Platform document. + const rounded = report('"10"', "1.00000000000000001", rows); + for (const json of [ + rounded, + rounded.replace( + "semgrep-platform-findings", + "\\u0073emgrep-platform-findings", + ), + ]) { + await writeFile(path, json); + await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( + "finding count", + ); + } + for (const projectId of ["10", "10.00000000000000001"]) { + await writeFile(path, report(projectId, "1", rows)); + await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( + /project ID/i, + ); + } + const native = semgrepJson(); + Object.assign(native.results[0]!.extra.metadata, { + nested: { format: "semgrep-platform-findings/v2", number: 42 }, + }); + await writeFile(path, JSON.stringify(native)); + expect( + await readSemgrepFindings(path, { sourceRepositoryUrl: url }, "local"), + ).toEqual( + normalizeSemgrepFindings(native, { sourceRepositoryUrl: url }, "local"), + ); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + +test("retired object-row exports cannot be saved as current Platform findings", () => { + expect(() => + normalizeSemgrepFindings( + { + ...scope, + format: "semgrep-platform-findings/v1", + findingCount: 1, + findings: [{ ...code, id: 12 }], + }, + {}, + "local", + ), + ).toThrow(/recollect/i); +}); diff --git a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts index 3f9b0da1d5..e85e78eba4 100644 --- a/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts +++ b/sdk/typescript/tests-ts/semgrep-platform-selection-file.test.ts @@ -141,34 +141,6 @@ test("Semgrep message titles stay intact by default and fall back above 512 Clou }); }); -test.each(["", ".0", "e0"])( - "an archived selection keeps its original occurrence ID for an integer token with suffix %s", - (suffix) => { - const finding = - '{"id":1000000000000000000001,"ref":"refs/heads/release","rule":{"name":"example.rule"},"repository":{"id":1000000000000000000002,"url":"https://github.com/example/project"}}'.replace( - "1000000000000000000001,", - "1000000000000000000001" + suffix + ",", - ); - const archived = { - ...selectedRepository(url, url), - project: { ...project, id: "1000000000000000000002" }, - findings: [finding], - }; - const row = normalizeSemgrepFindings( - archived, - { sourceRepositoryUrl: url }, - "local", - ).findings[0]!; - expect(row.source_finding_id).toBe( - "semgrep-platform:702c8f5d11896dc0316d2cc74426c02af20b6292374c3e08b0cf7e717c6fce4a", - ); - expect(row.evidence.title).toBe("example.rule"); - expect(row.evidence.source_data![rawFindingKey]).toContain( - '"id":1000000000000000000001', - ); - }, -); - test("two selected Platform SAST IDs from separate pages replay unchanged in SQLite", async () => { const report = selection([ page("8"), @@ -1101,11 +1073,13 @@ test("direct SDK imports respect the native source even with a caller-selected I expect(prepared.preview.findings[0]!.source_finding_id).toBe("chosen-id"); const genericOptions = { ...options, provider: "wiz" as const }; + const currentFormat = row.evidence.source_data!["format"]; delete row.evidence.source_data!["format"]; await expect( prepareNormalizedLocalImport(normalized, genericOptions, deps), ).rejects.toThrow("semgrep provider"); const data = row.evidence.source_data!; + data["format"] = currentFormat; const original = data[rawFindingKey] as string; data[rawFindingKey] = JSON.stringify({ ...JSON.parse(original), diff --git a/sdk/typescript/tests-ts/semgrep-platform-selection.test.ts b/sdk/typescript/tests-ts/semgrep-platform-selection.test.ts index 94e1aaa118..dcc10c8e83 100644 --- a/sdk/typescript/tests-ts/semgrep-platform-selection.test.ts +++ b/sdk/typescript/tests-ts/semgrep-platform-selection.test.ts @@ -122,6 +122,13 @@ test("a fully excluded SDK selection still verifies the checkout against the sel }); expect(selected.findings).toHaveLength(0); expect(selected.excluded).toHaveLength(1); + // SDK readers may expose the collection scope outside the enumerable JSON payload. + for (const field of [ + "sourceFormat", + "semgrepPlatformScope", + "sourceRepositoryUrl", + ]) + Object.defineProperty(selected, field, { enumerable: false }); await using fixture = await localFindingsFixture({ remote: url + "-other" }); await expect( prepareNormalizedLocalImport( diff --git a/sdk/typescript/tests-ts/semgrep-platform-v1-compat.test.ts b/sdk/typescript/tests-ts/semgrep-platform-v1-compat.test.ts deleted file mode 100644 index 0fab457f6a..0000000000 --- a/sdk/typescript/tests-ts/semgrep-platform-v1-compat.test.ts +++ /dev/null @@ -1,1294 +0,0 @@ -import { expect, test } from "bun:test"; -import { mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { gzipSync } from "node:zlib"; -import { semgrepJson } from "./support/semgrep-import-fixtures.js"; -import { prepareLocalExternalImport } from "../src/external-findings-import.js"; -import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; -import { bundledPluginRoot, runWorkbench } from "../src/runtime.js"; -import { - cloudRepositoryUrlKey, - validateExternalEvidence, -} from "../src/external-import-contract.js"; -import { isolatedGitReportFixture as localReportFixture } from "./external-findings-local-helpers.js"; -import { - normalizeSemgrepFindings, - readSemgrepFindings, -} from "../src/semgrep-findings.js"; -import { - selectSemgrepPlatformFindings, - normalizeSemgrepPlatformFindings, - semgrepPlatformFindingRepository, -} from "../src/semgrep-platform-findings.js"; - -const url = "https://github.com/example/project"; -const scope = { - deployment: "demo", - project: { id: "10", name: "example/project", url }, - ref: "refs/heads/topic", - product: "sast", -}; -const code = { - ref: scope.ref, - repository: { url }, - rule: { name: "fixture.rule" }, -}; - -test("the standalone Cloud normalizer verifies the port on reader mappings", () => { - const projectUrl = "https://scm.example.test:443/Example/Project"; - const report = { - ...scope, - format: "semgrep-platform-selection/v1", - project: { ...scope.project, url: projectUrl }, - findings: [ - JSON.stringify({ ...code, id: 12, repository: { url: projectUrl } }), - ], - }; - const mapped = (url: string) => ({ - sourceRepositories: [{ url, platform: "github" as const }], - }); - const different = mapped(projectUrl.replace(":443", "")); - expect(() => - normalizeSemgrepPlatformFindings(report, different, "cloud"), - ).toThrow(/reader's source repository/); - expect( - normalizeSemgrepPlatformFindings(report, different).findings, - ).toHaveLength(1); - expect( - normalizeSemgrepPlatformFindings( - report, - mapped(projectUrl.toLowerCase()), - "cloud", - ).findings, - ).toHaveLength(1); -}); - -test("Cloud shortens a long v1 message title using its original textual ID", () => { - const description = "é".repeat(257); - const report = { - ...scope, - format: "semgrep-platform-findings/v1", - findingCount: 1, - findings: [{ id: "1.3e1", rule_message: description }], - }; - expect( - normalizeSemgrepFindings(report, {}, "cloud").findings[0]?.evidence, - ).toMatchObject({ title: "Semgrep finding 1.3e1", description }); -}); - -test("strongly bound current SAST rows have identical selection and backlog exclusions", () => { - const location = { file_path: "src/service.ts", line: 2, end_line: 3 }; - const rows = [ - { ...code, id: 11, severity: "unrecognized" }, - { ...code, id: 12, location: { ...location, line: "2" } }, - { ...code, id: 13, location: { ...location, end_line: "3" } }, - ]; - const read = (format: string, findings: unknown[]) => - normalizeSemgrepFindings( - { - ...scope, - format, - findingCount: findings.length, - findings, - repositoryAliases: [], - }, - {}, - "local", - ); - const json = rows.map((row) => JSON.stringify(row)); - const selection = read("semgrep-platform-selection/v1", json); - const backlog = read("semgrep-platform-findings/v2", json); - expect(selection.excluded.map((row) => row.position)).toEqual([2, 3]); - expect(backlog.excluded).toEqual(selection.excluded); - expect(backlog.findings).toEqual(selection.findings); - expect(selection.findings).toHaveLength(1); - const evidence = selection.findings[0]!.evidence; - expect(evidence.severity).toBe("unknown"); - expect( - JSON.parse(String(evidence.source_data?.["semgrepPlatformFindingJson"])) - .severity, - ).toBe("unrecognized"); - const legacy = read("semgrep-platform-findings/v1", rows); - expect(legacy.excluded.map((row) => row.position)).toEqual([2, 3]); - expect(legacy.findings[0]!.evidence.severity).toBe("unknown"); - const weak = read( - "semgrep-platform-findings/v2", - rows.map((row) => JSON.stringify({ ...row, ref: undefined })), - ); - expect(weak.excluded).toEqual([]); - expect(weak.findings[0]!.evidence.severity).toBe("unknown"); - expect(weak.findings[1]!.evidence.locations?.[0]?.line).toBe(2); - expect(weak.findings[2]!.evidence.details!.code!.end_line).toBe(3); -}); - -test("a GitHub mapping reprojects generated severity when a backlog finding becomes selectable", async () => { - const url = "https://git.example.test/Team/Repo"; - const native = { - ...code, - id: 12, - repository: { id: 10, name: "team/repo" }, - severity: "informational", - }; - const rows = [ - native, - { ...native, id: 13, location: { file_path: "src/service.ts", line: "3" } }, - { - ...native, - id: 14, - location: { file_path: "src/service.ts", line: 3, end_line: "5" }, - }, - ]; - const report = { - ...scope, - project: { ...scope.project, name: "Project", url }, - format: "semgrep-platform-findings/v2", - repositoryAliases: [native.repository.name], - findingCount: rows.length, - findings: rows.map((row) => JSON.stringify(row)), - }; - const batch = normalizeSemgrepFindings(report, {}, "local"); - expect(batch.findings[0]!.evidence).toMatchObject({ - severity: "informational", - source_data: { format: report.format }, - }); - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("semgrep.json", source, { - remote: url, - }); - const options = { ...source, repository: fixture.repository }; - const dependencies = { - environment: fixture.environment, - sourceRepositories: [{ url, platform: "github" as const }], - }; - await fixture.writeJson(report); - const file = await prepareLocalExternalImport( - fixture.file, - options, - dependencies, - ); - const direct = () => - prepareNormalizedLocalImport(batch, options, dependencies); - expect(file.preview.findings[0]!.evidence).toMatchObject({ - severity: "unknown", - source_data: { format: "semgrep-platform-selection/v1" }, - }); - for (const row of file.preview.findings.slice(1)) { - expect(row.evidence).toMatchObject({ - severity: "informational", - locations: [{ path: "src/service.ts", line: 3 }], - source_data: { format: report.format }, - }); - } - expect(file.preview.excluded).toEqual([]); - expect((await file.import()).counts.created).toBe(rows.length); - expect((await (await direct()).import()).counts.unchanged).toBe(rows.length); - expect((await file.import()).counts.unchanged).toBe(rows.length); - expect((await fixture.read()).map((row) => row.version)).toEqual([1, 1, 1]); - batch.findings[0]!.evidence.severity = "high"; - expect((await direct()).preview.findings[0]!.evidence.severity).toBe("high"); -}); - -test.each(["sca", "weak SAST", "selected SAST"])( - "%s file and normalized SDK replays retain equivalent native number values once", - async (kind) => { - const strong = kind === "selected SAST"; - const product = kind === "sca" ? "sca" : "sast"; - const id = strong ? "12" : "1.2e1"; - const repository = strong ? JSON.stringify({ id: 10, url }) : '{"id":1e1}'; - const ref = strong ? `,"ref":${JSON.stringify(scope.ref)}` : ""; - const rule = product === "sast" ? ',"rule_name":"example.rule"' : ""; - const native = `{"id":${id}${rule},"repository":${repository},"metadata":{"source":"demo","score":10.0}${ref}}`; - const replayRepository = strong ? repository : '{"id":10.0}'; - const equivalent = `{"metadata":{"score":1e1,"source":"demo"},"repository":${replayRepository},"id":12${rule}${ref}}`; - const report = (row: string) => ({ - ...scope, - format: "semgrep-platform-findings/v2", - product, - repositoryAliases: [], - findingCount: 1, - findings: [row], - }); - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("semgrep.json", source, { - remote: url, - }); - await fixture.writeJson(report(native)); - const initial = await fixture.prepare(); - expect((await initial.import()).counts.created).toBe(1); - const result = normalizeSemgrepFindings(report(native), {}, "local"); - const data = result.findings[0]!.evidence.source_data!; - const prepare = () => fixture.prepareNormalized(result); - const unchanged = { created: 0, updated: 0, unchanged: 1 }; - data["semgrepPlatformFindingJson"] = equivalent; - for (const format of strong - ? [data["format"], undefined, "another-format"] - : [data["format"]]) { - if (format === undefined) delete data["format"]; - else data["format"] = format; - expect((await (await prepare()).import()).counts).toEqual(unchanged); - } - await fixture.writeJson(report(equivalent)); - expect((await (await fixture.prepare()).import()).counts).toEqual( - unchanged, - ); - const [stored] = await fixture.read(); - expect(stored?.version).toBe(1); - if (!strong) { - const raw = stored!.evidence.source_data!["semgrepPlatformFindingJson"]; - expect(raw).toContain('"id":12,'); - expect(raw).toContain('"repository":{"id":10}'); - } - - data["callerAnnotation"] = { group: "demo" }; - expect((await (await prepare()).import()).counts.updated).toBe(1); - expect((await (await prepare()).import()).counts).toEqual(unchanged); - expect( - (await fixture.read())[0]?.evidence.source_data?.["callerAnnotation"], - ).toEqual({ group: "demo" }); - }, -); - -test.each(["sast", "ai_sast", "sca"])( - "v1 %s requires the original title sources", - (product) => { - const untitled = { id: 11, ref: scope.ref, repository: { url } }; - const rows = [ - untitled, - { id: 12, rule: { name: "named", message: "description" } }, - { id: 13, rule_message: "message" }, - ...(product === "sca" - ? [ - { - id: 14, - vulnerability_identifier: "advisory", - found_dependency: { package: "dependency" }, - rule_message: "description", - }, - { - id: 15, - found_dependency: { package: "dependency" }, - rule_message: "description", - }, - ] - : []), - ]; - const report = { ...scope, product, findingCount: rows.length }; - const legacy = normalizeSemgrepFindings( - { ...report, format: "semgrep-platform-findings/v1", findings: rows }, - {}, - "local", - ); - expect(legacy.excluded).toMatchObject([ - { position: 1, reason: "Semgrep finding title must be nonempty text." }, - ]); - expect(legacy.findings.map((row) => row.evidence.title)).toEqual([ - "named", - "message", - ...(product === "sca" ? ["advisory", "dependency"] : []), - ]); - for (const format of [ - "semgrep-platform-findings/v2", - ...(product === "sast" ? ["semgrep-platform-selection/v1"] : []), - ]) { - const current = normalizeSemgrepFindings( - { - ...report, - format, - findingCount: 1, - findings: [JSON.stringify(untitled)], - repositoryAliases: [], - }, - {}, - "local", - ); - if (product === "sast") { - expect(current.findings).toEqual([]); - expect(current.excluded).toEqual(legacy.excluded); - } else { - expect(current.findings[0]?.evidence.title).toBe("Semgrep finding 11"); - } - } - }, -); - -test("a v1 project SSH URL requires a matching GitHub Enterprise reader reference", () => { - const source = { - url: "https://git.example.test/team/service", - platform: "github" as const, - }; - const report = { - ...scope, - format: "semgrep-platform-findings/v1", - sourceRepositoryUrl: source.url + ".git", - project: { ...scope.project, url: "git@git.example.test:team/service" }, - findingCount: 1, - findings: [ - { - id: 12, - rule_name: "example.rule", - repository: { name: "service.git" }, - }, - ], - }; - const read = (sourceRepositories: (typeof source)[]) => - normalizeSemgrepFindings(report, { sourceRepositories }, "local"); - for (const reference of [ - source.url, - "git@git.example.test:team/service", - "ssh://git@git.example.test/team/service", - ]) { - expect(read([{ ...source, url: reference }]).findings).toHaveLength(1); - } - const [stored] = read([source]).findings; - expect(semgrepPlatformFindingRepository(stored!, "semgrep", [source])).toBe( - source.url, - ); - stored!.evidence.source_data = { ...report, finding: report.findings[0] }; - expect(semgrepPlatformFindingRepository(stored!, "semgrep", [source])).toBe( - source.url, - ); - for (const repository of [ - { name: "unrelated.git" }, - { name: "service.git", id: 999 }, - ]) { - expect(() => - normalizeSemgrepFindings( - { ...report, findings: [{ ...report.findings[0], repository }] }, - { sourceRepositories: [source] }, - "local", - ), - ).toThrow(/repository/); - } - expect(() => read([])).toThrow(/repository/); - for (const foreign of [ - "git@git.example.test:team/other", - "ssh://git@other.example.test/team/service", - ]) { - expect(() => read([{ ...source, url: foreign }])).toThrow(/repository/); - } -}); - -test("legacy SSH repository projections replay unchanged alongside file imports", async () => { - const original = { - ...scope, - format: "semgrep-platform-findings/v1", - project: { ...scope.project, url: "git@github.com:example/project.git" }, - sourceRepositoryUrl: url, - }; - const row = { - ...code, - id: 12, - location: { file_path: "src\\service.ts", line: 2, end_line: 4 }, - }; - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("v1.json", source, { - remote: url, - }); - await fixture.writeJson({ ...original, findings: [row], findingCount: 1 }); - const file = await fixture.prepare(); - expect((await file.import()).counts.created).toBe(1); - const batch = structuredClone(file.preview); - const evidence = batch.findings[0]!.evidence; - evidence.source_data = { ...original, finding: row }; - evidence.details!.repository!.url = url; - evidence.locations = []; - evidence.details!.code!.end_line = null; - const prepare = () => fixture.prepareNormalized(batch); - const direct = await prepare(); - expect(direct.preview.findings[0]!.evidence.details).toMatchObject({ - code: { end_line: 4 }, - repository: { url: url + ".git" }, - }); - expect((await direct.import()).counts.unchanged).toBe(1); - expect((await (await fixture.prepare()).import()).counts.unchanged).toBe(1); - - const override = "https://github.com/Example/Project"; - evidence.details!.repository!.url = override; - expect( - (await prepare()).preview.findings[0]!.evidence.details?.repository?.url, - ).toBe(override); - evidence.details!.repository!.url = "https://github.com/example/other"; - await expect(prepare()).rejects.toThrow(/repository and ref/); - // Replaying the same unmapped archive with a matching declaration keeps its version. - const { sourceRepositoryUrl: _saved, ...unmapped } = original; - await fixture.writeJson({ ...unmapped, findings: [row], findingCount: 1 }); - for (const [i, declared] of [undefined, url, undefined].entries()) { - const replay = await fixture.prepare({ sourceRepositoryUrl: declared }); - expect(replay.preview.findings[0]!.evidence.source_data).not.toHaveProperty( - "sourceRepositoryUrl", - ); - expect((await replay.import()).counts).toMatchObject({ - created: 0, - unchanged: i === 0 ? 0 : 1, - }); - } -}); - -test("v1 files preserve native JSON numbers without witnesses and reject a rounded finding count", async () => { - const dir = await mkdtemp(join(tmpdir(), "semgrep-v1-")); - const path = join(dir, "report.gz"); - try { - const first = "9007199254740992", - second = "9007199254740993"; - const rows = [first, second].map( - (id) => - `{"id":${id},"metadata":{"score":1.00000000000000001,"key":${id}},"rule":{"name":"fixture.rule"}}`, - ); - const report = (count: string) => - `{"format":"semgrep-platform-findings/v1","deployment":"demo","project":{"id":9007199254740993,"name":"example/project","url":"${url}"},"ref":"refs/heads/topic","product":"sast","findings":[${rows}],"findingCount":${count}}`; - await writeFile(path, gzipSync(report("2"))); - const result = await readSemgrepFindings(path, {}, "local"); - expect(result.excluded).toEqual([]); - expect(new Set(result.findings.map((f) => f.source_finding_id)).size).toBe( - 2, - ); - for (const [index, finding] of result.findings.entries()) { - const retained = finding.evidence.source_data!; - expect(retained["project"]).toMatchObject({ id: second }); - expect(retained["semgrepPlatformFindingJson"]).toContain( - '"score":1.00000000000000001', - ); - expect(retained["semgrepPlatformFindingJson"]).toContain( - `"key":${[first, second][index]}`, - ); - } - await writeFile( - path, - gzipSync( - report("2").replace( - '"metadata"', - '"repository":{"id":9007199254740992},"metadata"', - ), - ), - ); - await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( - "repository", - ); - // A direct SDK input that already went through JSON.parse cannot recover an unsafe ID. - const rounded = { - ...scope, - project: { ...scope.project, id: first }, - format: "semgrep-platform-findings/v1", - findingCount: 1, - findings: [JSON.parse('{"id":1,"repository":{"id":9007199254740993}}')], - }; - expect(() => normalizeSemgrepFindings(rounded, {}, "local")).toThrow( - "exact, positive", - ); - await writeFile(path, gzipSync(report("2.00000000000000001"))); - await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( - "finding count", - ); - const mixed = { - ...scope, - format: "semgrep-platform-findings/v1", - findingCount: 2, - findings: [ - { ...code, id: 1.5 }, - { ...code, id: 12 }, - ], - }; - await writeFile(path, gzipSync(JSON.stringify(mixed))); - const file = await readSemgrepFindings(path, {}, "local"); - expect(file.findings).toHaveLength(1); - expect(file.excluded).toMatchObject([{ position: 1 }]); - expect(normalizeSemgrepFindings(mixed, {}, "local")).toEqual(file); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - -test("a legacy object witness cannot turn a native number into repository evidence", async () => { - const dir = await mkdtemp(join(tmpdir(), "semgrep-v1-witness-")); - try { - const path = join(dir, "report.json"); - await writeFile( - path, - JSON.stringify({ - ...scope, - format: "semgrep-platform-findings/v1", - findingCount: 1, - findings: [{ ...code, id: 12, repository: 42 }], - numericFields: { findings: { 0: { repository: {} } } }, - }), - ); - await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( - "Invalid scanner numeric fields", - ); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - -test("v1 SDK rows reject numbers that cannot be retained exactly", () => { - const report = (token: string) => ({ - ...scope, - format: "semgrep-platform-findings/v1", - findingCount: 1, - findings: [ - { ...code, id: 12, metadata: { values: JSON.parse(`[${token}]`) } }, - ], - }); - for (const token of ["9007199254740993", "1e400"]) { - expect(() => normalizeSemgrepFindings(report(token), {}, "local")).toThrow( - "original Semgrep export", - ); - } - const result = normalizeSemgrepFindings(report("0.125"), {}, "local"); - expect( - result.findings[0]!.evidence.source_data?.["semgrepPlatformFindingJson"], - ).toContain('"values":[0.125]'); -}); - -test("textual decimal and exponent IDs stay specific to v1 artifacts", () => { - const historical = normalizeSemgrepFindings( - { - ...scope, - project: { ...scope.project, id: "1e1" }, - format: "semgrep-platform-findings/v1", - findings: [{ ...code, id: "1.2e1", repository: { url, id: "1e1" } }], - findingCount: 1, - }, - {}, - "local", - ); - const v2 = (id: string | number) => ({ - ...scope, - format: "semgrep-platform-findings/v2", - findingCount: 1, - repositoryAliases: [], - findings: [JSON.stringify({ ...code, id })], - }); - const modern = normalizeSemgrepFindings(v2(12), {}, "local"); - expect(historical.excluded).toEqual([]); - expect(historical.findings[0]!.source_finding_id).toBe( - modern.findings[0]!.source_finding_id, - ); - const raw = - historical.findings[0]!.evidence.source_data?.[ - "semgrepPlatformFindingJson" - ]; - expect(raw).toContain('"id":"1.2e1"'); - expect(raw).toContain('"id":"1e1"'); - expect( - semgrepPlatformFindingRepository(historical.findings[0]!, "semgrep"), - ).toBe(url); - expect( - normalizeSemgrepFindings(v2("1.2e1"), {}, "local").findings, - ).toHaveLength(0); - expect(() => - selectSemgrepPlatformFindings({ - deployment: scope.deployment, - ref: scope.ref, - projectResponse: JSON.stringify({ project: scope.project }), - findingsResponses: [JSON.stringify({ findings: [{ ...code, id: 12 }] })], - findingIds: ["1.2e1"], - }), - ).toThrow("exact, positive"); -}); - -test("v1 files, original projected rows and v2 share one local occurrence version", async () => { - const rows = [ - { ...code, id: 12, repository: { name: "display", id: "10" } }, - { ...code, id: "13", repository: { name: "display" } }, - ]; - // In a Git HTTP path these escapes are repository name bytes, never a query or a fragment. - const project = { ...scope.project, url: url + "%23topic%3Fone" }; - const report = { - ...scope, - project, - repositoryAliases: ["display"], - findingCount: 2, - }; - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("semgrep.json", source, { - remote: project.url, - }); - await fixture.writeJson({ - ...report, - format: "semgrep-platform-findings/v1", - sourceRepositoryUrl: "https://github.com/Example/Project%23topic%3Fone.git", - findings: rows, - }); - const original = await fixture.prepare(); - expect((await original.import()).counts).toEqual({ - created: 2, - updated: 0, - unchanged: 0, - }); - const projected = structuredClone(original.preview); - for (const [i, finding] of projected.findings.entries()) { - finding.evidence.source_data = { - ...scope, - project, - format: "semgrep-platform-findings/v1", - repositoryAliases: report.repositoryAliases, - finding: rows[i], - }; - } - const prepareProjected = () => fixture.prepareNormalized(projected); - const unchanged = { created: 0, updated: 0, unchanged: 2 }; - expect((await (await prepareProjected()).import()).counts).toEqual(unchanged); - expect((await (await fixture.prepare()).import()).counts).toEqual(unchanged); - expect((await fixture.read()).map((row) => row.version)).toEqual([1, 1]); - await fixture.writeJson({ - ...report, - format: "semgrep-platform-findings/v2", - findings: rows.map((row) => JSON.stringify(row)), - }); - const replay = await fixture.prepare(); - expect(replay.preview.findings).toEqual(original.preview.findings); - expect((await replay.import()).counts).toEqual(unchanged); - - // A change in the retained vendor row is meaningful even when its title and location stay the same. - projected.findings[0]!.evidence.source_data!["finding"] = { - ...rows[0], - metadata: { revision: 2 }, - }; - expect((await (await prepareProjected()).import()).counts).toEqual({ - created: 0, - updated: 1, - unchanged: 1, - }); - expect((await fixture.read()).map((row) => row.version).sort()).toEqual([ - 1, 2, - ]); -}); - -test("original v1 normalized rows can bind a name-only alias through a sibling in the same scope", async () => { - const label = "display"; - const legacy = { - ...scope, - format: "semgrep-platform-findings/v1", - repositoryAliases: [label], - }; - const native = [ - { ...code, id: 12, repository: { name: label } }, - { ...code, id: 13, repository: { name: label, id: scope.project.id, url } }, - ]; - const batch = normalizeSemgrepFindings( - { ...legacy, findingCount: native.length, findings: native }, - {}, - "local", - ); - const currentWitness = structuredClone( - batch.findings[1]!.evidence.source_data!, - ); - for (const [i, finding] of batch.findings.entries()) { - // #2085 stored the vendor object without the later per-row witness string. - finding.evidence.source_data = { ...legacy, finding: native[i] }; - finding.source_finding_id = "retained-" + i; - } - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("v1.json", source, { - remote: url, - }); - const prepare = fixture.prepareNormalized; - const retained = { - ...legacy, - finding: native[0], - sourceRepositoryUrl: url, - operatorNote: "reviewed", - repositoryWitnessJson: "stale", - idEncoding: "stale", - project: { - ...scope.project, - url: null, - labels: { group: "demo", sequence: 1 }, - }, - }; - batch.findings[0]!.evidence.source_data = retained; - batch.findings[0]!.evidence.title = "Title chosen by the caller"; - const first = await prepare(batch); - const preparedData = first.preview.findings[0]!.evidence.source_data!; - expect(preparedData["repositoryWitnessJson"]).toBe( - JSON.stringify({ id: scope.project.id, name: label }), - ); - expect(preparedData).not.toHaveProperty("idEncoding"); - expect((await first.import()).counts).toEqual({ - created: 2, - updated: 0, - unchanged: 0, - }); - // A partially migrated batch retains the same independent repository proof. - batch.findings[1]!.evidence.source_data = currentWitness; - expect((await (await prepare(batch)).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 2, - }); - expect( - (await fixture.read()).find((row) => row.sourceFindingId === "retained-0"), - ).toMatchObject({ - version: 1, - evidence: { - title: "Title chosen by the caller", - source_data: { - operatorNote: "reviewed", - sourceRepositoryUrl: url, - project: { url: null, labels: { group: "demo", sequence: 1 } }, - }, - }, - }); - expect(batch.findings[0]!.evidence.source_data?.["finding"]).toEqual( - native[0], - ); - - retained.project.labels.sequence++; - expect((await (await prepare(batch)).import()).counts).toEqual({ - created: 0, - updated: 1, - unchanged: 1, - }); - expect((await (await prepare(batch)).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 2, - }); - expect( - (await fixture.read()).find((row) => row.sourceFindingId === "retained-0"), - ).toMatchObject({ - version: 2, - evidence: { - source_data: { project: { labels: { group: "demo", sequence: 2 } } }, - }, - }); - - for (const other of [ - { ...scope, deployment: "other" }, - { ...scope, project: { ...scope.project, id: "11" } }, - { ...scope, ref: "refs/heads/other" }, - { - ...scope, - project: { ...scope.project, url: "https://github.com/other/project" }, - }, - ]) { - const foreign = structuredClone(batch); - const witness = foreign.findings[1]!.evidence; - witness.source_data = { - ...legacy, - ...other, - finding: { - ...native[1], - ref: other.ref, - repository: { - name: label, - id: other.project.id, - url: other.project.url, - }, - }, - }; - witness.branch = other.ref; - witness.details!.repository!.url = other.project.url; - await expect(prepare(foreign)).rejects.toThrow(/alias|witness|repository/i); - } -}); - -test("a stored decimal witness remains equivalent to its canonical binding on local replay", async () => { - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("semgrep.json", source, { - remote: url, - }); - await fixture.writeJson({ - ...scope, - format: "semgrep-platform-findings/v1", - repositoryAliases: ["display"], - findingCount: 2, - findings: [ - { ...code, id: 12, repository: { name: "display", id: "1e1" } }, - { ...code, id: 13, repository: { name: "display" } }, - ], - }); - const saved = await fixture.prepare(); - expect((await saved.import()).counts.created).toBe(2); - const result = saved.preview; - expect(result.excluded).toEqual([]); - expect(result.findings).toHaveLength(2); - const [proof, dependent] = result.findings; - expect(proof!.evidence.source_data?.["format"]).toBe( - "semgrep-platform-findings/v1", - ); - expect(dependent!.evidence.source_data?.["format"]).toBe( - "semgrep-platform-findings/v2", - ); - for (const finding of result.findings) - expect(semgrepPlatformFindingRepository(finding, "semgrep")).toBe(url); - - const previous = structuredClone(dependent!); - Object.assign(previous.evidence.source_data!, { - format: "semgrep-platform-findings/v1", - idEncoding: "v1-decimal", - repositoryWitnessJson: JSON.stringify({ name: "display", id: "1e1" }), - }); - expect(semgrepPlatformFindingRepository(previous, "semgrep")).toBe(url); - const direct = await fixture.prepareNormalized({ - read: 1, - findings: [previous], - excluded: [], - }); - expect((await direct.import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 1, - }); - expect((await (await fixture.prepare()).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 2, - }); - expect( - (await fixture.read()).find( - (row) => row.sourceFindingId === previous.source_finding_id, - )?.version, - ).toBe(1); -}); - -test("a legacy report verifies each display label against that row's repository ID or URL", async () => { - const name = "display"; - const row = (repository: Record, id = 12) => ({ - ...code, - id, - repository, - }); - const report = (...findings: ReturnType[]) => ({ - ...scope, - format: "semgrep-platform-findings/v1", - findings, - findingCount: findings.length, - }); - await using fixture = await localReportFixture( - "v1.json", - { provider: "semgrep", sourceKey: "demo/topic" }, - { remote: url }, - ); - await fixture.writeJson(report(row({ name, id: 10 }))); - const prepared = await fixture.prepare(); - expect((await prepared.import()).counts).toEqual({ - created: 1, - updated: 0, - unchanged: 0, - }); - for (const rows of [ - [row({ name, id: 11, url })], - [row({ name, id: 10, url: "https://github.com/example/other" })], - [row({ name, url }), row({ name }, 13)], - ]) { - await fixture.writeJson(report(...rows)); - await expect(fixture.prepare()).rejects.toThrow(/repository/); - } -}); - -test("v1 repository labels retain decodeURI spellings without trusting unrelated labels", () => { - const archive = { - ...scope, - format: "semgrep-platform-findings/v1", - project: { - ...scope.project, - name: "logical", - url: "https://scm.example.test/group/R%C3%A9po%2B", - }, - findingCount: 1, - }; - const name = "group/Répo%2B"; - const row = { ...code, id: 12, repository: { name } }; - const source = { ...archive, findings: [row] }; - for (const aliases of [{}, { repositoryAliases: [name] }]) { - const [finding] = normalizeSemgrepFindings( - { ...source, ...aliases }, - {}, - "local", - ).findings; - expect(semgrepPlatformFindingRepository(finding!, "semgrep")).toBe( - archive.project.url, - ); - finding!.evidence.source_data = { ...archive, ...aliases, finding: row }; - expect(semgrepPlatformFindingRepository(finding!, "semgrep")).toBe( - archive.project.url, - ); - } - for (const invalid of [ - { - ...source, - repositoryAliases: ["group/Other%2B"], - findings: [{ ...row, repository: { name: "group/Other%2B" } }], - }, - { ...source, findings: [{ ...row, repository: { name, id: 11 } }] }, - { - ...source, - findings: [ - { - ...row, - repository: { name, url: "https://scm.example.test/other/repo" }, - }, - ], - }, - { - ...source, - format: "semgrep-platform-findings/v2", - findings: [JSON.stringify(row)], - repositoryAliases: [name], - }, - ]) { - expect(() => normalizeSemgrepFindings(invalid, {}, "local")).toThrow( - /repository/, - ); - } -}); - -test("projected legacy textual IDs retain their verified occurrence", () => { - const report = { - ...scope, - format: "semgrep-platform-findings/v1", - findings: [{ ...code, id: "1.2e1" }], - findingCount: 1, - }; - const [finding] = normalizeSemgrepFindings(report, {}, "local").findings; - const data = finding!.evidence.source_data!; - delete data["format"]; - expect(semgrepPlatformFindingRepository(finding!, "semgrep")).toBe(url); - data["format"] = "semgrep-platform-findings/v2"; - expect(() => semgrepPlatformFindingRepository(finding!, "semgrep")).toThrow( - "exact, positive", - ); -}); - -test("direct historic object-row evidence honors the caller's repository comparator", () => { - const lower = "https://ghe.example.test/org/service"; - const upper = "https://ghe.example.test/ORG/Service"; - const original = { ...code, id: 12, repository: { url: upper } }; - const report = { - ...scope, - format: "semgrep-platform-findings/v1", - project: { ...scope.project, url: upper }, - sourceRepositoryUrl: lower, - }; - const [finding] = normalizeSemgrepFindings( - { ...report, findings: [original], findingCount: 1 }, - { - sourceRepositoryUrl: lower, - sourceRepositories: [{ url: lower, platform: "github" }], - }, - "local", - ).findings; - finding!.evidence.source_data = { ...report, finding: original }; - expect(() => semgrepPlatformFindingRepository(finding!, "semgrep")).toThrow( - /repository/, - ); - expect( - semgrepPlatformFindingRepository( - finding!, - "semgrep", - cloudRepositoryUrlKey, - ), - ).toBe(upper); -}); - -test("the retained repository witness stays stable when the API returns rows in another order", () => { - const name = "display"; - const dependent = { ...code, id: 12, repository: { name } }; - const byId = { ...code, id: 13, repository: { name, id: 10 } }; - const byUrl = { ...code, id: 14, repository: { name, url } }; - const project = (witnesses: (typeof byId | typeof byUrl)[]) => - normalizeSemgrepFindings( - { - ...scope, - format: "semgrep-platform-findings/v2", - findingCount: 3, - repositoryAliases: [name], - findings: [dependent, ...witnesses].map((row) => JSON.stringify(row)), - }, - {}, - "local", - ).findings[0]!; - const first = project([byId, byUrl]); - expect(first.evidence.source_data?.["repositoryWitnessJson"]).toBeDefined(); - expect(project([byUrl, byId])).toEqual(first); -}); - -test("legacy rows with a generic format retain their repository for the Semgrep provider", () => { - const finding = { - source_finding_id: "opaque", - evidence: { - title: "Example", - severity: "low" as const, - branch: scope.ref, - details: { kind: "sast" as const, repository: { url } }, - source_data: { ...scope, format: "normalized/v1", finding: { id: 12 } }, - }, - }; - expect(semgrepPlatformFindingRepository(finding, "semgrep")).toBe(url); - expect(semgrepPlatformFindingRepository(finding, "wiz")).toBeUndefined(); -}); - -test("v2 files require exact counts and string project IDs", async () => { - const dir = await mkdtemp(join(tmpdir(), "semgrep-v2-controls-")); - const path = join(dir, "report.json"); - const report = (projectId: string, count: string, rows: string[]) => - `{"deployment":"demo","project":{"id":${projectId},"name":"example/project","url":"${url}"},"ref":"refs/heads/topic","product":"sast","repositoryAliases":[],"findingCount":${count},"format":"semgrep-platform-findings/v2","metadata":{"findingCount":1},"findings":${JSON.stringify(rows)}}`; - const rows = [JSON.stringify({ ...code, id: 12 })]; - try { - await writeFile(path, report('"10"', "1e0", rows)); - expect( - (await readSemgrepFindings(path, {}, "local")).findings, - ).toHaveLength(1); - await writeFile(path, report('"10"', "0", [])); - expect((await readSemgrepFindings(path, {}, "local")).findings).toEqual([]); - // An escaped format name is still the same Platform document. - const rounded = report('"10"', "1.00000000000000001", rows); - for (const json of [ - rounded, - rounded.replace( - "semgrep-platform-findings", - "\\u0073emgrep-platform-findings", - ), - ]) { - await writeFile(path, json); - await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( - "finding count", - ); - } - for (const projectId of ["10", "10.00000000000000001"]) { - await writeFile(path, report(projectId, "1", rows)); - await expect(readSemgrepFindings(path, {}, "local")).rejects.toThrow( - /project ID/i, - ); - } - const native = semgrepJson(); - Object.assign(native.results[0]!.extra.metadata, { - nested: { format: "semgrep-platform-findings/v2", number: 42 }, - }); - await writeFile(path, JSON.stringify(native)); - expect( - await readSemgrepFindings(path, { sourceRepositoryUrl: url }, "local"), - ).toEqual( - normalizeSemgrepFindings(native, { sourceRepositoryUrl: url }, "local"), - ); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); - -const historicalPaths = [ - ["src\\service.ts", null, "src\\service.ts", "all", 3, 5], - ["src\\backlog.ts", null, "src\\backlog.ts", "backlog", "3", "5"], - ["src\\strong.ts", null, "src\\strong.ts", "legacy", "3", "5"], - ["src\\zero.ts", null, null, "legacy", 0, 5], - ["src\\reversed.ts", null, null, "legacy", 3, 2], - ["pkg:service.ts", "pkg:service.ts", null, "all", 3, 5], - ["src/", "src/", null, "all", 3, 5], -] as const; - -test.each(historicalPaths)( - "migrates persisted Semgrep location %s once across historical and current replays", - verifyLegacyLocationReplay, -); - -async function verifyLegacyLocationReplay( - path: string, - oldPath: string | null, - newPath: string | null, - mode: "all" | "legacy" | "backlog", - line: string | number, - endLine: string | number, -) { - const backlogOnly = mode === "backlog"; - const legacyOnly = mode === "legacy"; - const locations = (path: string | null) => (path ? [{ path, line: 3 }] : []); - - const source = { provider: "semgrep" as const, sourceKey: "demo/topic" }; - await using fixture = await localReportFixture("semgrep.json", source, { - remote: url, - }); - const row = { - repository: code.repository, - rule: code.rule, - ...(backlogOnly ? {} : { ref: scope.ref }), - id: 12, - severity: "HIGH", - location: { file_path: path, line, end_line: endLine }, - }; - const legacy = { - ...scope, - project: { ...scope.project, id: 10 }, - format: "semgrep-platform-findings/v1", - findingCount: 1, - findings: [row], - }; - // The original object-row reader hashed numeric IDs, mapped HIGH to unknown, and used its hosted path rules. - const id = - "semgrep-platform:a7db3d7539422c25f776693a34ab2f916e31cc218f407c8c7faa7f209cad7ba3"; - const original = { - source_finding_id: id, - evidence: validateExternalEvidence( - { - title: code.rule.name, - severity: "unknown", - branch: scope.ref, - locations: locations(oldPath), - details: { - kind: "sast", - scanner_origin: "Semgrep", - repository: { url }, - rule: { id: code.rule.name }, - code: { end_line: oldPath ? 5 : null }, - }, - source_data: { - format: legacy.format, - deployment: scope.deployment, - project: legacy.project, - ref: scope.ref, - product: scope.product, - finding: row, - }, - }, - "local", - ), - }; - await runWorkbench( - { - pluginRoot: await bundledPluginRoot(), - environment: fixture.environment, - stateDirectory: fixture.stateDirectory, - }, - ["store-external-findings"], - JSON.stringify({ - ...source, - repository: fixture.repository, - findings: [original], - }), - ); - const [historical] = await fixture.read(); - expect(historical).toMatchObject({ - sourceFindingId: id, - version: 1, - evidence: { severity: "unknown", locations: locations(oldPath) }, - }); - const direct = (finding = original) => - fixture.prepareNormalized({ read: 1, excluded: [], findings: [finding] }); - const next = await direct(); - expect((await next.import()).counts).toEqual({ - created: 0, - updated: 1, - unchanged: 0, - }); - const [stored] = await fixture.read(); - expect(stored).toMatchObject({ - id: historical!.id, - sourceFindingId: id, - version: 2, - evidence: { - severity: "high", - locations: locations(newPath), - details: { code: { end_line: newPath ? 5 : null } }, - }, - }); - const raw = stored!.evidence.source_data!["semgrepPlatformFindingJson"]; - expect(JSON.parse(String(raw)).location).toEqual(row.location); - const modern = { - ...scope, - repositoryAliases: [], - findingCount: 1, - findings: [JSON.stringify(row)], - format: "semgrep-platform-findings/v2", - }; - const selected = { ...modern, format: "semgrep-platform-selection/v1" }; - if (backlogOnly) - expect(() => normalizeSemgrepFindings(selected, {}, "local")).toThrow( - /ref/u, - ); - const unchanged = { created: 0, updated: 0, unchanged: 1 }; - for (const report of [ - legacy, - modern, - ...(backlogOnly ? [] : [selected]), - legacy, - ]) { - await fixture.writeJson(report); - const file = await fixture.prepare(); - if (legacyOnly && report !== legacy) { - expect(file.preview.excluded.map((row) => row.reason)).toEqual([ - typeof endLine === "number" && endLine < Number(line) - ? "The code end line requires a first source location line and cannot precede it." - : "Semgrep source line must be a positive integer.", - ]); - expect((await file.import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 0, - }); - } else { - expect((await file.import()).counts).toEqual(unchanged); - } - if (backlogOnly && report === modern) - expect( - (await (await direct(file.preview.findings[0]!)).import()).counts, - ).toEqual(unchanged); - expect((await (await direct()).import()).counts).toEqual(unchanged); - } - if (legacyOnly && !newPath) { - // Both spellings were projected by the original reader and required valid coordinates. - for (const file_path of ["src/service.ts", "pkg:service.ts"]) { - await fixture.writeJson({ - ...legacy, - findings: [{ ...row, location: { ...row.location, file_path } }], - }); - const rejected = await fixture.prepare(); - expect(rejected.preview.excluded).toHaveLength(1); - expect((await rejected.import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 0, - }); - } - } - expect(await fixture.read()).toEqual([stored!]); - if (oldPath) { - const extended = structuredClone(original); - extended.evidence.locations!.push({ path: "another.ts", line: 7 }); - expect( - (await direct(extended)).preview.findings[0]!.evidence.locations, - ).toEqual(extended.evidence.locations); - } - const authored = structuredClone(original); - authored.evidence.source_data!["finding"] = { - ...row, - severity: "informational", - }; - authored.evidence.severity = "informational"; - expect((await direct(authored)).preview.findings[0]!.evidence).toMatchObject({ - severity: "informational", - locations: locations(newPath), - details: { code: { end_line: newPath ? 5 : null } }, - }); - const custom = structuredClone(original); - if (!legacyOnly) custom.evidence.severity = "low"; - custom.evidence.locations = [{ path: "manual.ts", line: 7 }]; - custom.evidence.details!.code!.end_line = 9; - const manual = (await direct(custom)).preview.findings[0]!.evidence; - expect(manual).toMatchObject({ - severity: legacyOnly ? "high" : "low", - locations: custom.evidence.locations, - details: { code: { end_line: 9 } }, - }); - const changed = { ...row, severity: "CRITICAL" }; - await fixture.writeJson( - legacyOnly - ? { ...legacy, findings: [changed] } - : { ...modern, findings: [JSON.stringify(changed)] }, - ); - expect((await (await fixture.prepare()).import()).counts).toEqual({ - created: 0, - updated: 1, - unchanged: 0, - }); - expect((await (await fixture.prepare()).import()).counts).toEqual(unchanged); - expect(await fixture.read()).toMatchObject([ - { - id: historical!.id, - sourceFindingId: id, - version: 3, - evidence: { severity: "critical" }, - }, - ]); -} diff --git a/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts b/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts index 35724fc2ae..ccd1e991b1 100644 --- a/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts +++ b/sdk/typescript/tests-ts/snyk-rest-boundaries.test.ts @@ -4,10 +4,7 @@ import { snykJson, snykSarif } from "./support/snyk-import-fixtures.js"; import { prepareNormalizedExternalPublication } from "../src/external-findings-publish.js"; import type { ExternalFindings } from "../src/external-findings.js"; import { prepareNormalizedLocalImport } from "../src/external-findings-local.js"; -import { - normalizeSnykIssues, - snykRestImportRepository, -} from "../src/snyk-rest-findings.js"; +import { normalizeSnykIssues } from "../src/snyk-rest-findings.js"; import { snykIssue, snykLocalFixture, @@ -415,13 +412,8 @@ test("a hosted selection cannot store another normalized project's projected ass } const license = JSON.parse(String(native["issueJson"])); license.attributes.type = "license"; - for (const projected of [ - { issue: license }, - { issueJson: JSON.stringify(license) }, - ]) { - row.evidence.source_data = projected; - await expect(prepare()).rejects.toThrow("selected project"); - } + row.evidence.source_data = { issueJson: JSON.stringify(license) }; + await expect(prepare()).rejects.toThrow("selected project"); delete row.evidence.source_data; await expect(prepare()).rejects.toThrow("Original Snyk REST issue evidence"); expect(await fixture.read()).toBeEmpty(); @@ -507,11 +499,7 @@ test("retagged original Snyk records remain scoped without a matching source ID prepareNormalizedLocalImport(projected, opts, { environment: f.environment, }), - ).rejects.toThrow( - "issue" in resources || "issueJson" in resources - ? /different project/ - : /Snyk/, - ); + ).rejects.toThrow("issueJson" in resources ? /different project/ : /Snyk/); } for (const missing of ["issueJson", "projectJson"]) { row.evidence.source_data = { ...original, format: "reader/v1" }; @@ -548,7 +536,7 @@ test("retagged original Snyk records remain scoped without a matching source ID { ...opts, provider: "snyk" }, { environment: f.environment }, ), - ).rejects.toThrow("both v1 and v2"); + ).rejects.toThrow("current collector"); } }); @@ -789,294 +777,103 @@ test("rows from a different Snyk monitor cannot use the report's source key even await expect(prepareCloud(input)).rejects.toThrow("Snyk project scope"); } } - const saved = structuredClone(original); - saved.source_finding_id = "saved-reader-id"; - saved.evidence.source_data = { - format: capture.format, - projectJson: capture.projectJson, - targetJson: capture.targetJson, - }; - const checkSaved = (monitor: typeof capture) => - snykRestImportRepository( - { ...normalizeSnykIssues(monitor), findings: [saved] }, - "snyk", - [], - [], - undefined, - true, - ); - expect(checkSaved(capture)?.url).toBe(capture.sourceRepositoryUrl); - for (const id of [capture.projectId, capture.organizationId]) { - expect(() => checkSaved(replace(id))).toThrow( - "retained Snyk project scope", - ); - } - for (const id of [target.id, integrationId]) { - expect(() => checkSaved(replace(id))).toThrow( - "conflicting targets or integrations", - ); - } - saved.evidence.source_data = { - format: capture.format, - targetJson: capture.targetJson.replace(`"${integrationId}"`, "null"), - }; - expect(checkSaved(capture)?.url).toBe(capture.sourceRepositoryUrl); - expect(checkSaved(replace("github-cloud-app", "github"))?.url).toBe( - capture.sourceRepositoryUrl, - ); - expect(() => - checkSaved(replace("github-cloud-app", "github-server-app")), - ).toThrow("conflicting targets or integrations"); expect(await f.read()).toEqual(stored); }); -test("a sparse approved Snyk record still checks its retained repository URLs", () => { - const url = "https://scm.example.test/Team/Project"; - const { projectJson } = snykRestFixture("open-source", { - sourceRepositoryUrl: url, - }); - const project = JSON.parse(projectJson); - project.attributes.origin = "gitlab"; - const target = project.relationships.target.data; - for (const source of [ - { format: "snyk-rest-issues/v2", sourceRepositoryUrl: url }, - { format: "snyk-rest-issues/v2", projectJson: JSON.stringify(project) }, - { format: "snyk-rest-issues/v2", targetJson: JSON.stringify(target) }, - { format: "snyk-rest-issues/v1", project }, - { format: "snyk-rest-issues/v1", target }, - ]) { - const input = { - read: 1, - excluded: [], - findings: [ - { - source_finding_id: "reader-selected", - evidence: { - title: "Issue", - severity: "low" as const, - source_data: source, - }, - }, - ], - }; - const check = (destination?: string) => - snykRestImportRepository( - input, - "snyk", - [], - destination ? [{ url: destination, platform: "github" }] : [], - undefined, - true, - destination, - ); - expect(check()).toBeUndefined(); - expect(check(url)?.url).toBe(url); - if ("project" in source || "projectJson" in source) - expect(() => check(url.toLowerCase())).toThrow( - "does not match the original Snyk project", - ); - expect(() => check("https://scm.example.test/Other/Project")).toThrow( - "does not match the original Snyk project", - ); - } -}); - -test("a sparse approved project retains its known product and target linkage", () => { - const report = normalizeSnykIssues(snykRestFixture("code")); +test("a linked issue projection uses its report's project and requires the Snyk provider", async () => { + const report = normalizeSnykIssues(snykRestFixture("open-source")); const row = report.findings[0]!; - row.source_finding_id = "reader-selected"; - const project = JSON.parse(String(report.snykRestContext!["projectJson"])); - const check = () => { - row.evidence.source_data = { - format: "snyk-rest-issues/v2", - projectJson: JSON.stringify(project), + const full = structuredClone(report); + const nativeId = row.source_finding_id; + const json = String(row.evidence.source_data!["issueJson"]); + const project = (id: string) => { + const issue = JSON.parse(json); + issue.relationships.scan_item.data.id = id; + return { + ...snykOptions, + issueJson: JSON.stringify(issue), }; - return snykRestImportRepository(report, "snyk", [], [], undefined, true); }; - expect(check()?.url).toBe(report.sourceRepositoryUrl); - project.attributes.type = "npm"; - expect(check).toThrow("retained Snyk project scope"); - delete project.attributes.type; - project.relationships.target.data = null; - expect(check).toThrow("conflicting targets"); - delete project.relationships.target; - expect(check()?.url).toBe(report.sourceRepositoryUrl); -}); - -test("opaque legacy issues retain sibling scope checks on a checkpoint retry", () => { - const report = normalizeSnykIssues(snykRestFixture("open-source")); - const native = report.findings[0]!; - const context = native.evidence.source_data!; - report.findings.push({ - ...native, - source_finding_id: "snyk-rest:opaque", - evidence: { - ...native.evidence, - source_data: { format: context["format"] }, - }, + row.source_finding_id = "reader-row"; + row.evidence.source_data = project("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); + await using fixture = await snykLocalFixture({ + remote: snykOptions.sourceRepositoryUrl, }); - const check = (allowOpaqueLegacyRows: boolean) => - snykRestImportRepository( - report, - "snyk", - [], - [], - undefined, - allowOpaqueLegacyRows, - ); - report.snykRestContext = context; - expect(() => check(false)).toThrow(); - expect(check(true)?.url).toBe(report.sourceRepositoryUrl); - // A different Snyk project can monitor exactly the same Git repository. - report.snykRestContext = JSON.parse( - JSON.stringify(context).replaceAll( - snykOptions.projectId, - "44444444-4444-4444-8444-444444444444", + const options = { + provider: "snyk" as const, + sourceKey: fixture.sourceKey, + repository: fixture.repository, + }; + const { read, findings, excluded } = report; + await expect( + prepareNormalizedExternalPublication( + { read, findings, excluded }, + { ...options, repository: "repo-example", provider: "wiz" }, + cloudCatalog(fixture.environment, snykOptions.sourceRepositoryUrl), ), + ).rejects.toThrow(/requires the snyk provider/i); + const prepare = () => fixture.prepareNormalized(report); + await expect(prepare()).rejects.toThrow("selected project"); + row.evidence.source_data = project(snykOptions.projectId); + await expect(prepare()).rejects.toThrow("normalized finding ID"); + row.source_finding_id = nativeId; + const targetlessProject = JSON.parse( + String(full.snykRestContext!["projectJson"]), ); - expect(() => check(true)).toThrow("retained Snyk project scope"); -}); - -test.each(["v1", "v2"] as const)( - "a linked %s issue projection uses its report's project and requires the Snyk provider", - async (version) => { - const report = normalizeSnykIssues(snykRestFixture("open-source")); - const row = report.findings[0]!; - const full = structuredClone(report); - const nativeId = row.source_finding_id; - const json = String(row.evidence.source_data!["issueJson"]); - const project = (id: string) => { - const issue = JSON.parse(json); - issue.relationships.scan_item.data.id = id; - return { - ...snykOptions, - ...(version === "v1" - ? { issue } - : { issueJson: JSON.stringify(issue) }), - }; - }; - row.source_finding_id = "reader-row"; - row.evidence.source_data = project("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"); - await using fixture = await snykLocalFixture({ - remote: snykOptions.sourceRepositoryUrl, - }); - const options = { - provider: "snyk" as const, - sourceKey: fixture.sourceKey, - repository: fixture.repository, - }; - const { read, findings, excluded } = report; - await expect( - prepareNormalizedExternalPublication( - { read, findings, excluded }, - { ...options, repository: "repo-example", provider: "wiz" }, - cloudCatalog(fixture.environment, snykOptions.sourceRepositoryUrl), - ), - ).rejects.toThrow(/requires the snyk provider/i); - const prepare = () => fixture.prepareNormalized(report); - await expect(prepare()).rejects.toThrow("selected project"); - const retry = () => - snykRestImportRepository(report, "snyk", [], [], undefined, true); - expect(retry).toThrow("selected project"); - row.evidence.source_data = project(snykOptions.projectId); - expect(retry()?.url).toBe(snykOptions.sourceRepositoryUrl); - await expect(prepare()).rejects.toThrow("normalized finding ID"); - row.source_finding_id = nativeId; - const issue = JSON.parse(json); - const targetlessProject = JSON.parse( - String(full.snykRestContext!["projectJson"]), - ); - delete targetlessProject.relationships; - Object.assign( - row.evidence.source_data, - version === "v1" - ? { project: targetlessProject } - : { projectJson: JSON.stringify(targetlessProject) }, - ); - await expect(prepare()).rejects.toThrow( - "missing the report's native target", - ); - const foreign = snykRestFixture("open-source", { - sourceRepositoryUrl: "https://github.com/example/other", - }); - const alternateId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; - const original = normalizeSnykIssues( - JSON.parse( - JSON.stringify(foreign).replaceAll(foreign.projectId, alternateId), - ), - ).snykRestContext!; - row.evidence.source_data = - version === "v1" - ? { issue, projectJson: original["projectJson"] } - : { - issueJson: json, - project: JSON.parse(String(original["projectJson"])), - }; - await expect(prepare()).rejects.toThrow("both v1 and v2"); - expect(await fixture.read()).toBeEmpty(); - const asset = report.snykRestContext!; - const selected = - version === "v1" - ? { - issue, - project: JSON.parse(String(asset["projectJson"])), - target: null, - } - : { - issueJson: json, - projectJson: asset["projectJson"], - targetJson: asset["targetJson"], - }; - row.evidence.source_data = { ...selected, projectId: alternateId }; - await expect(prepare()).rejects.toThrow("project"); - row.evidence.source_data = selected; - expect((await (await prepare()).import()).counts.created).toBe(1); - expect((await (await prepare()).import()).counts.unchanged).toBe(1); - const [stored] = await fixture.read(); - const listed = { - read: 1, - findings: [ - { - source_finding_id: stored!.sourceFindingId, - evidence: stored!.evidence, - }, - ], - excluded: [], - }; - const replay = await fixture.prepareNormalized(listed); - expect((await replay.import()).counts.unchanged).toBe(1); - await expect( - prepareNormalizedExternalPublication( - listed, - { ...options, repository: "repo-example", provider: "wiz" }, - cloudCatalog(fixture.environment, snykOptions.sourceRepositoryUrl), - ), - ).rejects.toThrow(/requires the snyk provider/i); - const complete = await fixture.prepareNormalized(full); - expect((await complete.import()).counts.created).toBe(0); - expect(await fixture.read()).toHaveLength(1); - }, -); - -test("unmarked native v1 rows survive unrelated issueJson data", async () => { - const { read, findings, excluded } = normalizeSnykIssues( - snykRestFixture("open-source"), - ); - const row = findings[0]!; - const original = row.evidence.source_data!; + delete targetlessProject.relationships; + Object.assign(row.evidence.source_data, { + projectJson: JSON.stringify(targetlessProject), + }); + await expect(prepare()).rejects.toThrow("missing the report's native target"); + const foreign = snykRestFixture("open-source", { + sourceRepositoryUrl: "https://github.com/example/other", + }); + const alternateId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const original = normalizeSnykIssues( + JSON.parse( + JSON.stringify(foreign).replaceAll(foreign.projectId, alternateId), + ), + ).snykRestContext!; row.evidence.source_data = { - ...snykOptions, - apiVersion: original["apiVersion"], + issueJson: json, project: JSON.parse(String(original["projectJson"])), - target: null, - issue: JSON.parse(String(original["issueJson"])), - issueJson: "{}", }; - await using fixture = await snykLocalFixture({ - remote: snykOptions.sourceRepositoryUrl, - }); - const result = await fixture.prepareNormalized({ read, findings, excluded }); - expect((await result.import()).counts.created).toBe(1); + await expect(prepare()).rejects.toThrow("current collector"); + expect(await fixture.read()).toBeEmpty(); + const asset = report.snykRestContext!; + const selected = { + issueJson: json, + projectJson: asset["projectJson"], + targetJson: asset["targetJson"], + }; + row.evidence.source_data = { ...selected, projectId: alternateId }; + await expect(prepare()).rejects.toThrow("project"); + row.evidence.source_data = selected; + expect((await (await prepare()).import()).counts.created).toBe(1); + expect((await (await prepare()).import()).counts.unchanged).toBe(1); + const [stored] = await fixture.read(); + const listed = { + read: 1, + findings: [ + { + source_finding_id: stored!.sourceFindingId, + evidence: stored!.evidence, + }, + ], + excluded: [], + }; + const replay = await fixture.prepareNormalized(listed); + expect((await replay.import()).counts.unchanged).toBe(1); + await expect( + prepareNormalizedExternalPublication( + listed, + { ...options, repository: "repo-example", provider: "wiz" }, + cloudCatalog(fixture.environment, snykOptions.sourceRepositoryUrl), + ), + ).rejects.toThrow(/requires the snyk provider/i); + const complete = await fixture.prepareNormalized(full); + expect((await complete.import()).counts.created).toBe(0); + expect(await fixture.read()).toHaveLength(1); }); test("a complete SDK row replays with reformatted native JSON and reader annotations", async () => { diff --git a/sdk/typescript/tests-ts/snyk-rest-findings.test.ts b/sdk/typescript/tests-ts/snyk-rest-findings.test.ts index b94e0aae95..3fc934f6fd 100644 --- a/sdk/typescript/tests-ts/snyk-rest-findings.test.ts +++ b/sdk/typescript/tests-ts/snyk-rest-findings.test.ts @@ -5,10 +5,7 @@ import { normalizeSnykIssues, type CollectSnykIssuesOptions, } from "../src/snyk-rest-findings.js"; -import { - createAndReplayFinding, - updateStoredFinding, -} from "./external-findings-local-helpers.js"; +import { createAndReplayFinding } from "./external-findings-local-helpers.js"; import { snykLocalFixture, @@ -828,31 +825,13 @@ test.each(["git+ssh", "ssh+git"])( }, ); -test("a v1 Code location stays unchanged until v2 adds its reported source line to the same row", async () => { - const { projectJson, targetJson, pages, ...scope } = snykRestFixture("code"); - const v1 = { - ...scope, - format: "snyk-rest-issues/v1", - project: JSON.parse(projectJson), - target: JSON.parse(targetJson), - pages: pages.map(({ startingAfter, responseJson }) => ({ - startingAfter, - response: JSON.parse(responseJson), - })), - }; - await using fixture = await snykLocalFixture({ - remote: options.sourceRepositoryUrl, - }); - await fixture.writeJson(v1); - const stored = await createAndReplayFinding(fixture); - expect(stored.evidence.locations).toEqual([ - { path: "src/app.ts", line: null }, - ]); - await fixture.writeJson(snykRestFixture("code")); - const updated = await updateStoredFinding(fixture, stored); - expect(updated.evidence.locations).toEqual([{ path: "src/app.ts", line: 3 }]); - expect((await (await fixture.prepare()).import()).counts.unchanged).toBe(1); - +test("the Snyk reader requires the current collection format and a numeric source line", () => { + expect(() => + normalizeSnykIssues({ + ...snykRestFixture("code"), + format: "snyk-rest-issues/v1", + }), + ).toThrow("snyk-rest-issues/v2"); const v2 = snykRestFixture("code"); v2.pages[0]!.responseJson = v2.pages[0]!.responseJson.replace( '"line":3', diff --git a/sdk/typescript/tests-ts/snyk-rest-v1.test.ts b/sdk/typescript/tests-ts/snyk-rest-v1.test.ts deleted file mode 100644 index 50a6028dc0..0000000000 --- a/sdk/typescript/tests-ts/snyk-rest-v1.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { expect, test } from "bun:test"; -import { writeFile } from "node:fs/promises"; -import { gzipSync } from "node:zlib"; -import { normalizeSnykFile } from "../src/snyk-findings.js"; -import { parseSnykReport } from "../src/snyk-rest-v1.js"; -import { - normalizeSnykIssues, - snykRestFindingRepository, -} from "../src/snyk-rest-findings.js"; -import { - createAndReplayFinding, - updateStoredFinding, -} from "./external-findings-local-helpers.js"; -import { - snykLocalFixture, - snykOptions, - snykRestFixture, - snykIssue, - snykProject, -} from "./support/snyk-rest-fixtures.js"; - -test("replays a v1 Snyk export with its original evidence, then updates the same stored ID from v2", async () => { - const token = "9007199254740993"; - const projectToken = "9007199254740993e1"; - const project = { ...snykProject().data, counter: projectToken }; - project.attributes.target_file = "src/"; - const issue = snykIssue(false); - const original = { - ...issue, - attributes: { ...issue.attributes, description: token, label: token }, - }; - const issueFields = { attributes: { description: token } }; - const v1 = { - ...snykOptions, - format: "snyk-rest-issues/v1", - apiVersion: "2026-03-25", - complete: true, - project, - target: null, - pages: [{ startingAfter: null, response: { data: [original] } }], - numericFields: { - project: { counter: projectToken }, - pages: { 0: { response: { data: { 0: issueFields } } } }, - }, - }; - await using fixture = await snykLocalFixture({ - remote: snykOptions.sourceRepositoryUrl, - }); - await fixture.writeJson(v1); - const stored = await createAndReplayFinding(fixture); - expect(stored.evidence.packages?.[0]?.manifest_path).toBe("src/"); - const source = stored.evidence.source_data!; - expect(stored.evidence.description).toBeNull(); - expect(source).toEqual({ - ...snykOptions, - format: v1.format, - apiVersion: v1.apiVersion, - project, - target: null, - issue: original, - numericFields: { project: { counter: projectToken }, issue: issueFields }, - }); - expect( - snykRestFindingRepository({ - source_finding_id: stored.sourceFindingId, - evidence: stored.evidence, - }).url, - ).toBe(snykOptions.sourceRepositoryUrl); - const selected = normalizeSnykIssues(v1); - expect(selected).not.toHaveProperty("sourceRepositoryPlatform"); - const row = selected.findings[0]!; - delete row.evidence.source_data!["format"]; - const unmarked = await fixture.prepareNormalized(selected); - expect((await unmarked.import()).counts.unchanged).toBe(1); - expect((await fixture.read())[0]).toEqual(stored); - row.evidence.source_data = { - issue: original, - project: undefined, - target: undefined, - }; - for (const fields of [{ issue: issueFields }, source["numericFields"]]) { - row.evidence.source_data["numericFields"] = fields; - const projected = await fixture.prepareNormalized(selected); - expect((await projected.import()).counts.unchanged).toBe(1); - expect((await fixture.read())[0]).toEqual(stored); - } - row.evidence.source_data["numericFields"] = { - project: { counter: "9007199254740994" }, - }; - await expect(fixture.prepareNormalized(selected)).rejects.toThrow( - "Invalid scanner numeric fields", - ); - - // A historical native project still identifies a hosted report after a reader removes the issue and ID. - const assetOnly = { ...source }; - delete assetOnly["format"]; - delete assetOnly["issue"]; - assetOnly["numericFields"] = { project: { counter: projectToken } }; - row.source_finding_id = "reader-id"; - row.evidence.source_data = assetOnly; - await expect( - fixture.prepareNormalized({ - read: 1, - findings: [row], - excluded: [], - }), - ).rejects.toThrow("Snyk original issue"); - expect(() => - normalizeSnykIssues({ - ...v1, - numericFields: { project: { counter: "9007199254740994" } }, - }), - ).toThrow("Invalid exact scanner number"); - - // Earlier v1 files also used native numbers and no sidecar. Both forms replay the same row. - const native = JSON.stringify({ ...v1, numericFields: undefined }) - .replace(`"counter":"${projectToken}"`, `"counter":${projectToken}`) - .replace(`"description":"${token}"`, `"description":${token}`); - await writeFile(fixture.file, gzipSync(native)); - expect((await (await fixture.prepare()).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 1, - }); - - const v2 = snykRestFixture("open-source"); - v2.projectJson = JSON.stringify(project).replace( - `"counter":"${projectToken}"`, - `"counter":${projectToken}`, - ); - v2.pages[0]!.responseJson = JSON.stringify(v1.pages[0]!.response).replace( - `"description":"${token}"`, - `"description":${token}`, - ); - await fixture.writeJson(v2); - const updated = await updateStoredFinding(fixture, stored); - expect(updated.evidence.packages?.[0]?.manifest_path).toBeNull(); - const retained = updated.evidence.source_data?.["issueJson"]; - expect(retained).toContain(`"description":${token}`); - expect(retained).toContain(`"label":"${token}"`); - expect((await (await fixture.prepare()).import()).counts).toEqual({ - created: 0, - updated: 0, - unchanged: 1, - }); -}); - -test("a second projection preserves both native numbers and v1 type witnesses", async () => { - const token = "9007199254740995"; - const document = parseSnykReport( - JSON.stringify({ - ...snykOptions, - format: "snyk-rest-issues/v1", - apiVersion: "2026-03-25", - complete: true, - project: { ...snykProject().data, counter: token }, - target: null, - pages: [ - { - startingAfter: null, - response: { - data: [{ ...snykIssue(false), counter: "native-number" }], - }, - }, - ], - numericFields: { project: { counter: token } }, - }).replace('"native-number"', token), - ); - const first = await normalizeSnykFile(document, "cloud", []); - expect(first.findings[0]?.evidence.source_data?.["numericFields"]).toEqual({ - project: { counter: token }, - issue: { counter: token }, - }); - expect(await normalizeSnykFile(document, "cloud", [])).toEqual(first); -}); diff --git a/sdk/typescript/tests-ts/stored-native-private-report.test.ts b/sdk/typescript/tests-ts/stored-native-private-report.test.ts new file mode 100644 index 0000000000..1d63a74742 --- /dev/null +++ b/sdk/typescript/tests-ts/stored-native-private-report.test.ts @@ -0,0 +1,97 @@ +import { expect, test } from "bun:test"; +import { normalizeEndorFindings } from "../src/endor-findings.js"; +import { + selectPrivateReport, + type PrivateImportContext, +} from "../src/private-finding-import.js"; +import { normalizeWizJson } from "../src/wiz-findings.js"; +import { endorFinding } from "./support/endor-import-fixtures.js"; + +const structured = "security.findings.v1"; +const context = (formats: string[]): PrivateImportContext => ({ + object: "security.finding_import_context", + context_version: "current", + can_upload: true, + can_create_import_jobs: true, + can_read: true, + formats, + limits: { max_direct_file_bytes: 1_000_000, max_record_bytes: 100_000 }, +}); + +const wiz = { + id: "wiz-id", + name: "CVE-2099-0001", + severity: "HIGH", + detailedName: "lodash", + vulnerableAsset: { id: "asset" }, +}; +for (const { provider, format, normalized, payload } of [ + { + provider: "wiz", + format: "wiz.vulnerability_json.v1", + normalized: normalizeWizJson([wiz], "cloud"), + payload: [wiz], + }, + { + provider: "endor", + format: "endor.findings_json.v1", + normalized: normalizeEndorFindings(endorFinding()), + payload: { list: { objects: [endorFinding()] } }, + }, +] as const) { + test(`${provider} encodes native-only uploads from reviewed rows and pins the format on retry`, () => { + const selected = normalized.findings; + const native = context([format]); + const report = selectPrivateReport(selected, provider, native); + expect(report.format).toBe(format); + expect(JSON.parse(report.bytes)).toEqual(payload); + + const both = context([format, structured]); + expect(selectPrivateReport(selected, provider, both).format).toBe( + structured, + ); + expect(selectPrivateReport(selected, provider, both, format)).toEqual( + report, + ); + const edited = structuredClone(selected); + edited[0]!.evidence.title += " (reviewed)"; + for (const limit of [ + "max_record_bytes", + "max_direct_file_bytes", + ] as const) { + const constrained = context([format, structured]); + constrained.limits[limit] = Buffer.byteLength(report.bytes); + expect(selectPrivateReport(selected, provider, constrained)).toEqual( + report, + ); + expect(() => + selectPrivateReport(selected, provider, constrained, structured), + ).toThrow("limit"); + expect(() => selectPrivateReport(edited, provider, constrained)).toThrow( + "limit", + ); + } + expect(() => selectPrivateReport(edited, provider, native)).toThrow( + "represented exactly", + ); + native.limits.max_record_bytes = 1; + expect(() => selectPrivateReport(selected, provider, native)).toThrow( + "limits", + ); + }); +} + +test("Endor rows that omit oversized native enrichment require structured upload", () => { + const base = endorFinding(); + const { target_dependency_name: _, ...spec } = base.spec; + const raw = { + ...base, + spec: { ...spec, target_dependency_package_name: "a".repeat(513) }, + }; + const selection = normalizeEndorFindings(raw); + const native = context(["endor.findings_json.v1"]); + expect(selection.findings).toHaveLength(1); + expect(() => + selectPrivateReport(selection.findings, "endor", native), + ).toThrow(structured); +}); diff --git a/sdk/typescript/tests-ts/trivy-findings.test.ts b/sdk/typescript/tests-ts/trivy-findings.test.ts index 1d9e3d6d69..f05dd1842e 100644 --- a/sdk/typescript/tests-ts/trivy-findings.test.ts +++ b/sdk/typescript/tests-ts/trivy-findings.test.ts @@ -4,8 +4,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { - needsPreviousTrivyProjection, - normalizePreviousTrivyFindings, normalizeTrivyFindings, readTrivyFindings, trivyRepositoryUrl, @@ -582,13 +580,8 @@ test("Cloud keeps a repeated lockfile dependency with all native declaration ran const ranges = report.Results[1]!.Packages[0]!.Locations; const current = normalizeAtRoot(report); const local = normalizeTrivyFindings(report, "local", "."); - const previous = normalizePreviousTrivyFindings(report, "cloud", "."); - const [ordinary, repeated] = current.findings; + const [, repeated] = current.findings; expect(current.excluded).toEqual([]); - expect(previous.findings).toEqual([ordinary!]); - expect(previous.excluded).toMatchObject([ - { position: 2, source_finding_id: repeated!.source_finding_id }, - ]); expect(repeated!.source_finding_id).toBe( local.findings[1]!.source_finding_id, ); @@ -599,8 +592,6 @@ test("Cloud keeps a repeated lockfile dependency with all native declaration ran expect(repeated!.evidence.source_data?.["package"]).toMatchObject({ Locations: ranges, }); - expect(needsPreviousTrivyProjection(current)).toBe(true); - expect(needsPreviousTrivyProjection(previous)).toBe(false); }); test("only ID-less detections use a unique name/version match from inventory", () => { diff --git a/sdk/typescript/tests-ts/veracode-findings.test.ts b/sdk/typescript/tests-ts/veracode-findings.test.ts index 53bf841485..fb8a777fb2 100644 --- a/sdk/typescript/tests-ts/veracode-findings.test.ts +++ b/sdk/typescript/tests-ts/veracode-findings.test.ts @@ -1,11 +1,9 @@ -import { hasPreviousCloudProjection } from "../src/reader-selection.js"; import { expect, test } from "bun:test"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { gzipSync } from "node:zlib"; import { - normalizePreviousVeracodeFindings, normalizeVeracodeFindings, readVeracodeFindings, veracodeReportRepositories, @@ -78,29 +76,14 @@ test("accepts schema and expanded CWE and category fields under one policy flaw for (const target of ["local", "cloud"] as const) { const expanded = normalize(page, target); const result = normalize(mixed, target); - expect(hasPreviousCloudProjection(expanded)).toBe(false); - expect(hasPreviousCloudProjection(result)).toBe(target === "cloud"); expect(result.excluded).toEqual([]); // Only the native row spelling should distinguish the two projections. expanded.findings[1]!.evidence.source_data!["row"] = scalar; expect(result.findings).toEqual(expanded.findings); - const previous = normalizePreviousVeracodeFindings( - mixed, - repository, - target, - ); - expect(previous.findings).toEqual([expanded.findings[0]!]); - expect(previous.excluded).toMatchObject([ - { - position: 2, - source_finding_id: expanded.findings[1]!.source_finding_id, - }, - ]); } mixed._links.self.href += "&cwe=79"; const filtered = normalize(mixed); expect(filtered.findings).toHaveLength(2); - expect(hasPreviousCloudProjection(filtered)).toBe(false); mixed._links.self.href += "&finding_category=4"; expect(() => normalize(mixed)).toThrow(/finding_category ID/); }); diff --git a/sdk/typescript/tests-ts/wiz-findings.test.ts b/sdk/typescript/tests-ts/wiz-findings.test.ts index 923f125a42..db926c7fa3 100644 --- a/sdk/typescript/tests-ts/wiz-findings.test.ts +++ b/sdk/typescript/tests-ts/wiz-findings.test.ts @@ -1,13 +1,5 @@ import { hash } from "node:crypto"; -import { - mkdtemp, - mkdir, - readFile, - readdir, - rename, - rm, - writeFile, -} from "node:fs/promises"; +import { mkdtemp, mkdir, readdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Writable } from "node:stream"; @@ -18,24 +10,18 @@ import { main } from "../src/cli.js"; import { ExternalPublicationError, prepareExternalPublication, - prepareNormalizedExternalPublication, } from "../src/external-findings-publish.js"; -import { DEFAULT_CLOUD_BASE_URL } from "../src/cloud-endpoint.js"; import { validateExternalEvidence } from "../src/external-import-contract.js"; import type { ExternalFindingEvidence, FindingImportRequest, SourceReport, } from "../src/external-import-models.js"; -import { - readVendorFindings, - readVendorFindingsForPublication, -} from "../src/wiz-findings.js"; +import { readVendorFindings } from "../src/wiz-findings.js"; import { prepareLocalExternalImport } from "../src/external-findings-import.js"; import { localFindingsFixture } from "./external-findings-local-helpers.js"; import { capture, dependencies } from "./cli-fixtures.js"; import { createCliTest } from "./support/cli-run.js"; -import previewCheckpoint from "./fixtures/wiz-preview-checkpoint.json" with { type: "json" }; import roundtripFixtures from "./fixtures/wiz-details-roundtrip.json" with { type: "json" }; const directories: string[] = []; @@ -156,7 +142,7 @@ async function parse(payload: unknown) { } test.each(["local", "cloud"] as const)( - "mixed Wiz reports preserve duplicate IDs found only in the historical projection (%s)", + "mixed Wiz reports preserve valid findings alongside unusable inventory rows (%s)", async (target) => { const row = { ...sast, repository: { id: repository.id } }; const { file } = await fixture({ @@ -209,9 +195,6 @@ test.each(["local", "cloud"] as const)( unchanged: 1, }); } - await expect( - readVendorFindingsForPublication(file, target), - ).rejects.toThrow('Duplicate source finding id "sast:occurrence-1"'); }, ); @@ -507,22 +490,19 @@ test.each([ ).toThrow("repository-relative paths"); }); -test("legacy evidence defaults retain their saved-submission bytes, including explicit null details", () => { +test("external evidence accepts explicit null details", () => { const input = { title: "Legacy evidence", severity: "high", locations: [{ path: "package.json" }], packages: [{ name: "example-package" }], }; - const legacy = validateExternalEvidence(structuredClone(input)); - expect(hash("sha256", JSON.stringify(legacy))).toBe( - "88f2d96656a5ad322d5f16fce7704366e359a8101d035b311a1917d4f6e31caf", - ); + const plain = validateExternalEvidence(structuredClone(input)); const withNull = validateExternalEvidence({ ...structuredClone(input), details: null, }); - expect(withNull).toEqual(legacy); + expect(withNull).toEqual(plain); expect(withNull).not.toHaveProperty("details"); }); @@ -1523,10 +1503,8 @@ test.each(["named collection", "normalized JSONL"])( await writeFile( runner, ` - import { readVendorFindings, readVendorFindingsForPublication } from "../src/wiz-findings.js"; - const parsed = process.argv[3] === "normalized JSONL" - ? (await readVendorFindingsForPublication(process.argv[2]!)).current - : await readVendorFindings(process.argv[2]!); + import { readVendorFindings } from "../src/wiz-findings.js"; + const parsed = await readVendorFindings(process.argv[2]!); console.log(JSON.stringify({ read: parsed.read, ready: parsed.findings.length, excluded: parsed.excluded.length, first: parsed.findings[0]?.source_finding_id, @@ -1568,277 +1546,6 @@ test.each(["named collection", "normalized JSONL"])( }, ); -// Captured after the pre-release writer at a431663b5e1 received a receipt and -// readback failed. Its filenames depended on deployment and host collation. -async function previewCheckpointFixture(base = DEFAULT_CLOUD_BASE_URL) { - const request = previewCheckpoint.requests[0]!; - const item = request.items[0]!; - const { title, severity, source_data } = item.evidence; - const f = await cloudFixture([ - { - source_finding_id: item.source_finding_id, - evidence: { title, severity, source_data }, - }, - ]); - // Seed the injected Cloud transport with the observation in the saved receipt. - await f.deps.fetch(`${base}/finding_imports`, { - method: "POST", - body: JSON.stringify(request), - }); - const directory = join( - f.environment.CODEX_SECURITY_STATE_DIR, - "external-finding-publications", - ); - await mkdir(directory, { recursive: true, mode: 0o700 }); - return { - ...f, - directory, - prepare: () => - prepareExternalPublication(f.file, f.options, { - ...f.deps, - environment: { ...f.environment, CODEX_SECURITY_CLOUD_BASE_URL: base }, - }), - }; -} - -const savedPreview = JSON.stringify(previewCheckpoint); - -// Use the host's collation to recover the original filename in any locale. -function localPreviewFilename() { - const request = previewCheckpoint.requests[0]!; - const selection = request.items.map(({ source_finding_id, evidence }) => ({ - source_finding_id, - evidence, - })); - const identity = [ - previewCheckpoint.accountId, - request.repository.id, - request.repository.repo_connector_id, - request.source, - selection, - ]; - return `${legacyCheckpointKey(identity)}.pending.json`; -} - -function legacyCheckpointKey(identity: unknown[]) { - const encoded = JSON.stringify(identity, (_key, child: unknown) => - child && typeof child === "object" && !Array.isArray(child) - ? Object.fromEntries( - Object.entries(child).sort(([a], [b]) => a.localeCompare(b)), - ) - : child, - ); - return hash("sha256", encoded); -} - -test("a normalized Wiz retry requires a known locale to reuse its acknowledged receipt", async () => { - const f = await previewCheckpointFixture(); - const input = await readVendorFindings(f.file); - const data = input.findings[0]!.evidence.source_data!; - Object.defineProperty(data, "toJSON", { value: () => ({ ...data, Z: 7 }) }); - const prepare = (selected = input) => - prepareNormalizedExternalPublication(selected, f.options, { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, - }, - }); - const early = await prepare(); - const foreign = join(f.directory, hostedForeignName); - await writeFile(foreign, savedPreview); - await expect(early.publish()).rejects.toThrow( - "original deployment and locale", - ); - await expect(prepare()).rejects.toThrow("original deployment and locale"); - const opaque = await readVendorFindings(f.file); - const getter = opaque.findings[0]!.evidence.source_data!; - Object.defineProperty(getter, "Z", { get: () => 1 }); - Object.defineProperty(getter, "toJSON", { value: () => ({ ...data, Z: 9 }) }); - await expect(prepare(opaque)).rejects.toThrow("cannot be matched"); - expect(f.posts).toHaveLength(1); - expect(await readFile(foreign, "utf8")).toBe(savedPreview); - const distinct = await prepare(JSON.parse(JSON.stringify(opaque))); - expect(distinct.preview.resumed).toBe(false); - - await rename(foreign, join(f.directory, localPreviewFilename())); - const retry = await prepare(); - expect(retry.preview.resumed).toBe(true); - expect(retry.preview.requests).toEqual(previewCheckpoint.requests); - const completed = await retry.publish(); - expect(completed.verified).toBe(1); - expect(completed.receipts).toEqual(previewCheckpoint.receipts); - expect(f.posts).toHaveLength(1); // Only the original import seeded by the fixture. - const fresh = await prepare(); - expect(fresh.preview.resumed).toBe(false); - expect(fresh.preview.requests[0]!.items[0]!.evidence.source_data?.["Z"]).toBe( - 7, - ); -}); - -test.each(["distinct", "duplicate"] as const)( - "a migrated preview checkpoint completes readback with a %s older file", - async (kind) => { - const f = await previewCheckpointFixture(); - const oldName = localPreviewFilename(); - const oldPath = join(f.directory, oldName); - await writeFile(oldPath, savedPreview); - const migrated = await f.prepare(); - expect(migrated.preview.resumed).toBe(true); - expect(migrated.preview.requests).toEqual( - previewCheckpoint.requests, - ); - const pendingNames = (await readdir(f.directory)).filter((name) => - name.endsWith(".pending.json"), - ); - expect(pendingNames).toHaveLength(1); - expect(pendingNames).not.toContain(oldName); - const pendingPath = join(f.directory, pendingNames[0]!); - expect(await readFile(pendingPath, "utf8")).toBe(savedPreview); - - // A crash can leave both names. Only an identical request can be removed. - const old = structuredClone(previewCheckpoint); - if (kind === "distinct") { - old.requests[0]!.request_id = "synthetic-older-request"; - old.receipts = []; - } - const older = JSON.stringify(old); - await writeFile(oldPath, older); - const resumed = await f.prepare(); - expect(resumed.preview.requests).toEqual( - previewCheckpoint.requests, - ); - expect(await readFile(pendingPath, "utf8")).toBe(savedPreview); - if (kind === "duplicate") { - await expect(readFile(oldPath, "utf8")).rejects.toMatchObject({ - code: "ENOENT", - }); - } else { - expect(await readFile(oldPath, "utf8")).toBe(older); - } - - const report = f.reports.get("unicode-evidence")!; - report.evidence.source_data = Object.fromEntries( - Object.entries(report.evidence.source_data!).reverse(), - ); - const beforePublish = f.calls.length; - expect((await resumed.publish()).verified).toBe(1); - expect(f.calls.slice(beforePublish)).toContainEqual( - expect.stringContaining(`/source_reports/${report.id}`), - ); - expect(f.posts).toHaveLength(1); - if (kind === "distinct") { - expect(await readFile(oldPath, "utf8")).toBe(older); - } else { - expect((await f.prepare()).preview.resumed).toBe(false); - } - }, -); - -// Pick a filename the historical writer produced under a different collation. -function foreignFilename(englishKey: string, swedishKey: string) { - return `${"ä".localeCompare("Z") < 0 ? swedishKey : englishKey}.pending.json`; -} -const hostedForeignName = foreignFilename( - "cde1eadb671511a7a309b2d76cf3e6f95fbb452e86543547d841be981002fbcc", - "253163ba65ed2f1929f28c25e3d129e111ec119851244e8dcbe32a1bb4ed952a", -); -const customForeignName = foreignFilename( - "2dcaa1dba62bf389461d0a02692f34ec3f3119bb4aacb8b21502466ae6ce0d33", - "d64d6d01223834e06cf10a7b71fa31c730c3f3c5d08cc189f6a0d5ab6d3b6b3f", -); -const alternateBase = "https://cloud.example.test"; - -test.each([ - ["hosted, another locale", DEFAULT_CLOUD_BASE_URL, hostedForeignName, false], - [ - "custom, another locale after preview", - alternateBase, - customForeignName, - true, - ], - [ - "custom, recognizable hosted filename", - alternateBase, - localPreviewFilename(), - false, - ], -] as const)( - "an unidentified preview checkpoint stops before upload: %s", - async (_name, target, filename, afterPreview) => { - const f = await previewCheckpointFixture(target); - const path = join(f.directory, filename); - const newer = f.reports.get("unicode-evidence")!; - newer.version++; - newer.evidence.description = "Newer Cloud evidence"; - const fresh = afterPreview ? await f.prepare() : undefined; - await writeFile(path, savedPreview); - await expect(fresh ? fresh.publish() : f.prepare()).rejects.toThrow( - "original deployment and locale", - ); - expect(f.posts).toHaveLength(1); - expect(newer.evidence.description).toBe("Newer Cloud evidence"); - expect(await readFile(path, "utf8")).toBe(savedPreview); - }, -); - -test.each([ - "account", - "repository", - "source", - "evidence", - "projected", - "invalid", -])( - "unrelated checkpoints do not block a fresh publication (%s)", - async (difference) => { - const f = await cloudFixture([ - { - source_finding_id: "scope-evidence", - evidence: { title: "Synthetic scope evidence", severity: "high" }, - }, - ]); - const prepared = await prepareExternalPublication( - f.file, - f.options, - f.deps, - ); - const saved: { - accountId: string; - requests: FindingImportRequest[]; - inputProjection?: { inputDigest: string; savedEvidenceDigest: string }; - } = { - accountId: prepared.preview.accountId, - requests: structuredClone(prepared.preview.requests), - }; - if (difference === "account") saved.accountId = "another-account"; - if (difference === "repository") - saved.requests[0]!.repository.id = "another-repository"; - if (difference === "source") - saved.requests[0]!.source.source_key = "another-source"; - if (difference === "evidence") - saved.requests[0]!.items[0]!.evidence.title = "Different evidence"; - if (difference === "projected") - saved.inputProjection = { - inputDigest: "another-deployment", - savedEvidenceDigest: "another-deployment", - }; - const directory = join( - f.environment.CODEX_SECURITY_STATE_DIR, - "external-finding-publications", - ); - await mkdir(directory, { recursive: true, mode: 0o700 }); - const path = join(directory, `${"a".repeat(64)}.pending.json`); - const serialized = - difference === "invalid" ? "invalid JSON" : JSON.stringify(saved); - await writeFile(path, serialized); - const fresh = await prepareExternalPublication(f.file, f.options, f.deps); - expect(fresh.preview.resumed).toBe(false); - expect(await readFile(path, "utf8")).toBe(serialized); - expect(f.posts).toHaveLength(0); - }, -); - test("Node resumes the same uncertain publication across locales and Unicode key order", async () => { const input = { source_finding_id: "locale-evidence", @@ -2431,212 +2138,6 @@ test.each(["raw", "normalized"])( }, ); -async function prepareLegacyInventoryPublication( - sastNodes: unknown[], - lastInventoryNode: unknown, -) { - // The earlier mapper used the last inventory row and stripped branch prefixes. - const f = await cloudFixture({ - data: { - sastFindings: { nodes: sastNodes }, - versionControlResources: { nodes: [lastInventoryNode] }, - }, - }); - const deps = { - ...f.deps, - environment: { - ...f.environment, - CODEX_SECURITY_CLOUD_BASE_URL: DEFAULT_CLOUD_BASE_URL, - }, - }; - const historical = await readVendorFindingsForPublication(f.file); - await writeFile(f.file, JSON.stringify(historical.legacy.findings)); - const original = await prepareExternalPublication(f.file, f.options, deps); - f.state.failReadback = true; - return { f, deps, historical, original }; -} - -async function savedPathAfterReadbackFailure( - publication: Awaited>, -) { - try { - await publication.publish(); - } catch (error) { - expect(error).toBeInstanceOf(ExternalPublicationError); - return (error as ExternalPublicationError).result.savedSubmission!; - } - throw new Error("Expected publication readback to fail"); -} - -test.each([ - "resume", - "reset", - "coverage", - "rebind", - "prepend", - "excluded", - "qualifier", -])( - "inventory normalization upgrades saved immutable requests and receipts (%s)", - async (scenario) => { - const record = { - ...sast, - vendorMetadata: { z: 1, ä: 2 }, - repository: { id: repository.id }, - ...(scenario === "qualifier" - ? { - repositoryBranch: { - ...sast.repositoryBranch, - name: "project/component/Feature", - }, - } - : {}), - }; - const selected = - scenario === "coverage" - ? [ - { ...record, id: "already-ready", repository }, - { ...record, id: "newly-ready" }, - ] - : scenario === "excluded" - ? [ - record, - { - source_finding_id: "sast:occurrence-1", - evidence: { title: "Excluded input", severity: "invalid" }, - }, - ] - : [record]; - const nodes = [ - { - ...inventoryNode("GITHUB"), - repository: { - ...repository, - name: - scenario === "qualifier" ? "project/component" : "Example/Project", - url: - scenario === "qualifier" - ? repositoryUrl - : "https://github.com/Example/Project.git", - }, - }, - { - ...inventoryNode("GITHUB"), - repository: { - ...repository, - name: - scenario === "coverage" - ? "z".repeat(513) - : scenario === "qualifier" - ? "project" - : repository.name, - url: repositoryUrl, - }, - }, - ]; - const payload = { - data: { - sastFindings: { nodes: selected }, - versionControlResources: { nodes }, - }, - }; - const { f, deps, historical, original } = - await prepareLegacyInventoryPublication(selected, nodes[1]); - expect(historical.legacy.findings[0]!.evidence.branch).toBe( - scenario === "qualifier" ? "component/Feature" : "feature/parser", - ); - const originalPath = await savedPathAfterReadbackFailure(original); - const saved = JSON.parse(await readFile(originalPath, "utf8")); - delete saved.inputProjection; - await writeFile(originalPath, JSON.stringify(saved)); - expect(saved.receipts).toHaveLength(1); - const identity = [ - original.preview.accountId, - original.preview.destination.id, - original.preview.destination.repo_connector_id, - original.preview.source, - original.preview.findings, - ]; - const oldKey = legacyCheckpointKey(identity); - const oldPath = join( - f.environment.CODEX_SECURITY_STATE_DIR, - "external-finding-publications", - `${oldKey}.pending.json`, - ); - if (oldPath !== originalPath) await rename(originalPath, oldPath); - await writeFile(f.file, JSON.stringify(payload)); - if (scenario === "coverage") { - expect((await readVendorFindings(f.file)).findings).toHaveLength(2); - await expect( - prepareExternalPublication(f.file, f.options, deps), - ).rejects.toThrow("selects different findings"); - expect(JSON.parse(await readFile(oldPath, "utf8"))).toEqual(saved); - expect(f.posts).toHaveLength(1); - return; - } - if (scenario === "reset") { - f.destination.reset_marker = "generation-2"; - await expect( - prepareExternalPublication(f.file, f.options, deps), - ).rejects.toThrow("saved request was retired"); - await expect(readFile(oldPath, "utf8")).rejects.toMatchObject({ - code: "ENOENT", - }); - expect(f.posts).toHaveLength(1); - return; - } - const upgraded = await prepareExternalPublication(f.file, f.options, deps); - expect(upgraded.preview.resumed).toBe(true); - expect(upgraded.preview.requests).toEqual(original.preview.requests); - expect(upgraded.preview.findings).toEqual(original.preview.findings); - const migratedPath = await savedPathAfterReadbackFailure(upgraded); - const migrated = await readFile(migratedPath, "utf8"); - const migratedData = JSON.parse(migrated); - expect(migratedData.requests).toEqual(saved.requests); - expect(migratedData.receipts).toEqual(saved.receipts); - if (scenario !== "prepend" && scenario !== "excluded") nodes.reverse(); - await writeFile(f.file, JSON.stringify(payload)); - const reordered = await prepareExternalPublication(f.file, f.options, deps); - expect(reordered.preview.resumed).toBe(true); - expect(reordered.preview.requests).toEqual(original.preview.requests); - migratedData.requests[0].items[0].evidence.description = - "Modified saved evidence"; - await writeFile(migratedPath, JSON.stringify(migratedData)); - await expect( - prepareExternalPublication(f.file, f.options, deps), - ).rejects.toThrow("Saved import evidence"); - await writeFile(migratedPath, migrated); - const incomplete = JSON.parse(migrated); - delete incomplete.inputProjection.savedEvidenceDigest; - await writeFile(migratedPath, JSON.stringify(incomplete)); - await expect( - prepareExternalPublication(f.file, f.options, deps), - ).rejects.toThrow("Saved import evidence"); - await writeFile(migratedPath, migrated); - let replay = reordered; - if (["rebind", "prepend", "excluded"].includes(scenario)) { - nodes.unshift({ - ...nodes[1]!, - repository: { ...nodes[1]!.repository, name: "EXAMPLE/PROJECT" }, - }); - await writeFile(f.file, JSON.stringify(payload)); - replay = await prepareExternalPublication(f.file, f.options, deps); - expect(replay.preview.resumed).toBe(true); - expect(replay.preview.requests).toEqual(original.preview.requests); - expect(replay.preview.findings).toEqual(original.preview.findings); - } - const newer = f.reports.get("sast:occurrence-1")!; - newer.version += 1; - newer.evidence.description = "Newer Cloud evidence"; - f.state.failReadback = false; - expect((await replay.publish()).verified).toBe(1); - expect(f.posts).toHaveLength(1); - expect(f.reports.get("sast:occurrence-1")!.evidence.description).toBe( - "Newer Cloud evidence", - ); - }, -); - test.each([ ["GITHUB", "GITLAB"], ["GITLAB", "GITHUB"], @@ -2695,149 +2196,8 @@ test("GITHUB inventory provenance does not permit a different repository URL", a }), ).rejects.toThrow("conflicting URLs"); }); -test("concurrent legacy inventory resumers retain the acknowledged request after migration finishes", async () => { - const name = - "concurrent legacy inventory resumers retain the acknowledged request after migration finishes"; - const { runTestInSubprocess } = await import("./support/test-subprocess.js"); - if (runTestInSubprocess(import.meta.filename, name)) return; - const fs = await import("node:fs/promises"); - const { spyOn } = await import("bun:test"); - const record = { ...sast, repository: { id: repository.id } }; - const nodes = [ - { - ...inventoryNode("GITHUB"), - repository: { - ...repository, - name: "Example/Project", - url: "https://github.com/Example/Project.git", - }, - }, - { - ...inventoryNode("GITHUB"), - repository: { ...repository, url: repositoryUrl }, - }, - ]; - const payload = { - data: { - sastFindings: { nodes: [record] }, - versionControlResources: { nodes }, - }, - }; - const { f, deps, historical, original } = - await prepareLegacyInventoryPublication([record], nodes[1]); - const originalPath = await savedPathAfterReadbackFailure(original); - const saved = JSON.parse(await readFile(originalPath, "utf8")); - expect(saved.receipts).toHaveLength(1); - const identity = [ - original.preview.accountId, - original.preview.destination.id, - original.preview.destination.repo_connector_id, - original.preview.source, - ]; - const encode = (value: unknown) => - JSON.stringify(value, (_key, child) => - child !== null && typeof child === "object" && !Array.isArray(child) - ? Object.fromEntries( - Object.keys(child) - .sort() - .map((key) => [key, child[key]]), - ) - : child, - ); - const directory = join( - f.environment.CODEX_SECURITY_STATE_DIR, - "external-finding-publications", - ); - saved.inputProjection.inputDigest = hash( - "sha256", - encode(historical.legacyIdentity), - ); - await writeFile(originalPath, JSON.stringify(saved)); - const oldKey = legacyCheckpointKey([...identity, original.preview.findings]); - const oldPath = join(directory, `${oldKey}.pending.json`); - if (oldPath !== originalPath) await rename(originalPath, oldPath); - await writeFile(f.file, JSON.stringify(payload)); - const { currentIdentity } = await readVendorFindingsForPublication(f.file); - const key = hash("sha256", encode([...identity, currentIdentity])); - const pendingPath = join(directory, `${key}.pending.json`); - const lockPath = join(directory, `${key}.lock.sqlite`); - expect(pendingPath).not.toBe(oldPath); - const firstAtLock = Promise.withResolvers(); - const secondAtLock = Promise.withResolvers(); - const releaseSecond = Promise.withResolvers(); - const originalLstat = fs.lstat; - let lockEntries = 0; - const metadata = spyOn(fs, "lstat").mockImplementation((async ( - ...args: Parameters - ) => { - if (String(args[0]) === lockPath) { - lockEntries += 1; - if (lockEntries === 1) { - firstAtLock.resolve(); - await secondAtLock.promise; - } else if (lockEntries === 2) { - secondAtLock.resolve(); - await releaseSecond.promise; - } - } - return await originalLstat(...args); - }) as typeof fs.lstat); - let first: ReturnType | undefined; - let second: ReturnType | undefined; - try { - first = prepareExternalPublication(f.file, f.options, deps); - await firstAtLock.promise; - second = prepareExternalPublication(f.file, f.options, deps); - await secondAtLock.promise; - const leading = await first; - expect(leading.preview.resumed).toBe(true); - expect(leading.preview.requests).toEqual(saved.requests); - expect(JSON.parse(await readFile(pendingPath, "utf8")).receipts).toEqual( - saved.receipts, - ); - f.state.failReadback = false; - expect((await leading.publish()).verified).toBe(1); - await expect(readFile(pendingPath, "utf8")).rejects.toMatchObject({ - code: "ENOENT", - }); - await expect(readFile(oldPath, "utf8")).rejects.toMatchObject({ - code: "ENOENT", - }); - const newer = structuredClone(f.reports.get("sast:occurrence-1")!); - newer.version += 1; - newer.evidence.description = - "Newer Cloud evidence after the first resumer completed"; - f.reports.set("sast:occurrence-1", newer); - const otherPath = join( - directory, - `${hash("sha256", "other-deployment")}.pending.json`, - ); - const projectionless = JSON.stringify({ - accountId: saved.accountId, - requests: saved.requests, - }); - await writeFile(otherPath, projectionless, { mode: 0o600 }); - releaseSecond.resolve(); - const trailing = await second; - const result = await trailing.publish(); - expect(f.posts).toHaveLength(1); - expect(trailing.preview.resumed).toBe(true); - expect(trailing.preview.requests).toEqual(saved.requests); - expect(result.receipts).toEqual(saved.receipts); - expect(result.verified).toBe(1); - expect(f.reports.get("sast:occurrence-1")).toEqual(newer); - expect(await readFile(otherPath, "utf8")).toBe(projectionless); - } finally { - secondAtLock.resolve(); - releaseSecond.resolve(); - await Promise.allSettled( - [first, second].filter((value) => value !== undefined), - ); - metadata.mockRestore(); - } -}); -test("resume preview is derived from the saved upload evidence", async () => { +test("resuming a Wiz upload requires the original projected inventory evidence", async () => { const payload = { data: { sastFindings: { nodes: [{ ...sast, repository: { id: repository.id } }] }, @@ -2862,6 +2222,14 @@ test("resume preview is derived from the saved upload evidence", async () => { payload.data.versionControlResources.nodes[0]!.repository.name = "updated-display-name"; await writeFile(f.file, JSON.stringify(payload)); + const changed = await prepareExternalPublication(f.file, f.options, f.deps); + expect(changed.preview.resumed).toBe(false); + expect(changed.preview.findings).not.toEqual(original.preview.findings); + expect(f.posts).toHaveLength(1); + + payload.data.versionControlResources.nodes[0]!.repository.name = + "original-display-name"; + await writeFile(f.file, JSON.stringify(payload)); const resumed = await prepareExternalPublication(f.file, f.options, f.deps); expect(resumed.preview.resumed).toBe(true); expect(resumed.preview.requests).toEqual(original.preview.requests); From 742cf1f470683869ea2197d04d4440ca0ad4fedf Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 13:36:30 +0000 Subject: [PATCH 2/3] test(imports): remove an empty reader import --- sdk/typescript/tests-ts/external-findings-publish.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/typescript/tests-ts/external-findings-publish.test.ts b/sdk/typescript/tests-ts/external-findings-publish.test.ts index 1c52f25f4f..f4a6eb72cf 100644 --- a/sdk/typescript/tests-ts/external-findings-publish.test.ts +++ b/sdk/typescript/tests-ts/external-findings-publish.test.ts @@ -95,7 +95,6 @@ import { import { createCliTest } from "./support/cli-run.js"; import { normalizeTrivyFindings } from "../src/trivy-findings.js"; import { normalizeInspectorFindings } from "../src/inspector-findings.js"; -import {} from "../src/continuum-findings.js"; import { trivyReport, trivyRepositoryHeader, From 7590375164d20686bf617e832b9f5015d28e14e4 Mon Sep 17 00:00:00 2001 From: Michael D'Angelo Date: Wed, 11 Nov 2026 13:39:35 +0000 Subject: [PATCH 3/3] refactor(imports): project saved checkpoint items once --- .../src/external-findings-publish.ts | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/sdk/typescript/src/external-findings-publish.ts b/sdk/typescript/src/external-findings-publish.ts index 2252ef7e75..60ef3fc40a 100644 --- a/sdk/typescript/src/external-findings-publish.ts +++ b/sdk/typescript/src/external-findings-publish.ts @@ -1139,18 +1139,6 @@ async function preparePublicationInput( options.provider === "semgrep" && "sourceFormat" in parsed && isSemgrepPlatformFormat(parsed.sourceFormat); - function matchesSavedDestination(requests: FindingImportRequest[]): boolean { - return ( - requests.length > 0 && - requests.every( - (batch) => - batch.repository.id === destination.id && - batch.repository.repo_connector_id === - destination.repo_connector_id && - canonicalJson(batch.source) === canonicalJson(source), - ) - ); - } function wasReset(content: SavedSubmission) { return content.requests.some( ({ repository }) => repository.reset_marker !== destination.reset_marker, @@ -1196,11 +1184,19 @@ async function preparePublicationInput( ) throw new Error("Saved import account or requests do not match."); content.requests = content.requests.map(validateImportRequest); - for (const finding of submissionFindings(content.requests)) + const actual = submissionFindings(content.requests); + for (const finding of actual) checkRetainedTrivyProvider(options.provider, finding); - if (!matchesSavedDestination(content.requests)) + if ( + content.requests.some( + (batch) => + batch.repository.id !== destination.id || + batch.repository.repo_connector_id !== + destination.repo_connector_id || + canonicalJson(batch.source) !== canonicalJson(source), + ) + ) throw new Error("Saved import destination does not match."); - const actual = submissionFindings(content.requests); checkFindingProvider({ ...parsed, findings: actual }, options.provider); const expected = repositoryFindings(parsed, content.repositorySeverities); if (canonicalJson(actual) !== canonicalJson(expected.map(wireFinding)))