diff --git a/.github/workflows/recover-protocol-cdn.yml b/.github/workflows/recover-protocol-cdn.yml index 7cc73a26a49..b289e421c35 100644 --- a/.github/workflows/recover-protocol-cdn.yml +++ b/.github/workflows/recover-protocol-cdn.yml @@ -1,4 +1,4 @@ -name: Recover protocol release on CDN +name: Recover release artifacts on CDN on: workflow_dispatch: @@ -7,7 +7,19 @@ on: description: Exact stable semver to recover (for example, 3.0.22) required: true type: string - + surfaces: + description: >- + protocol = the four signed tuple files only. all = also the immutable + schemas// and compliance// trees. + required: false + default: protocol + type: choice + options: + - protocol + - all + +# One recovery per version at a time, whatever the requested surfaces, so two +# dispatches cannot interleave reads and conditional writes on the same objects. concurrency: group: recover-protocol-cdn-${{ inputs.version }} cancel-in-progress: false @@ -21,7 +33,7 @@ permissions: jobs: recover: - name: Recover immutable protocol tuple + name: Recover immutable release artifacts runs-on: ubuntu-latest steps: - name: Checkout repository history @@ -29,10 +41,15 @@ jobs: with: fetch-depth: 0 + # The tag checked out below carries its own copies of these scripts, and + # an old release predates them entirely. Recovery logic must come from + # the current reviewed branch, never from the artifact being recovered. - name: Preserve current workflow publication fence run: | node scripts/check-release-state.cjs current cp scripts/check-release-state.cjs "$RUNNER_TEMP/check-release-state.cjs" + cp scripts/check-protocol-bundle-parity.cjs "$RUNNER_TEMP/check-protocol-bundle-parity.cjs" + cp scripts/recover-cdn-version-trees.sh "$RUNNER_TEMP/recover-cdn-version-trees.sh" - name: Install cosign uses: sigstore/cosign-installer@v3 @@ -164,3 +181,29 @@ jobs: --certificate-identity-regexp '^https://github\.com/adcontextprotocol/adcp/\.github/workflows/release\.yml@refs/heads/.*$' \ --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ "${VERSION}.tgz" + + # The signed tuple above is the only cryptographic authority available + # for a historical version; R2 objects are unsigned. The script below + # proves the trees it publishes are byte-for-byte the trees inside that + # tarball before it writes anything, so one cosign verification covers + # every object. See RELEASING.md "Recovering a release's CDN artifacts". + - name: Upload only missing immutable schema and compliance objects + if: inputs.surfaces == 'all' + env: + VERSION: ${{ inputs.version }} + FENCE_SCRIPT: ${{ runner.temp }}/check-release-state.cjs + PARITY_SCRIPT: ${{ runner.temp }}/check-protocol-bundle-parity.cjs + BUCKET: ${{ vars.ADCP_ARTIFACT_R2_BUCKET || 'adcp-artifacts' }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + run: bash "$RUNNER_TEMP/recover-cdn-version-trees.sh" upload + + - name: Verify recovered public CDN trees + if: inputs.surfaces == 'all' + env: + VERSION: ${{ inputs.version }} + FENCE_SCRIPT: ${{ runner.temp }}/check-release-state.cjs + PARITY_SCRIPT: ${{ runner.temp }}/check-protocol-bundle-parity.cjs + run: bash "$RUNNER_TEMP/recover-cdn-version-trees.sh" verify diff --git a/RELEASING.md b/RELEASING.md index c550744c64e..02a1ba84007 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -424,8 +424,100 @@ Live unscoped bulk backfills are disabled. `--dry-run` remains available for inventory; it is not release authorization. Historical CDN recovery must use an existing approved tag/release, a current tested branch fence, unchanged committed bytes, and conditional missing-object creation. The stable-only -`recover-protocol-cdn.yml` recovers four protocol files, not schemas/compliance -and not RC releases. +`recover-protocol-cdn.yml` is the single dispatch for that; it does not +recover RC releases. + +### Recovering a release's CDN artifacts + +A stable release whose objects never reached R2 does not announce itself. The +worker's pinned-semver fallback resolves `/schemas/3.0.28/...` to the newest +published release at-or-below it on the same line, so the request returns 200 +with an *older release's* bytes. Only two signals distinguish that from a real +hit: the response carries `cache-control: public, no-cache, must-revalidate` +instead of the immutable policy an exact directory gets, and +`build-schema-discovery-pointers.cjs` refuses to advertise the release, so the +deploy log carries `::warning title=Schema versions missing from the CDN` +naming the gap and the line's alias stays pinned one release back (#8318, +#8324, #8347). + +Nothing in the routine paths can close that gap. `release.yml` publishes only +the version in the current checkout's `package.json`; `--latest-only` excludes +every semver path; live bulk backfill is disabled. Recovery is a separate, +maintainer-dispatched authority: + +``` +gh workflow run recover-protocol-cdn.yml --ref main \ + -f version= -f surfaces=all +``` + +`surfaces` defaults to `protocol` (the four signed tuple files, the original +scope of this workflow). `all` additionally recovers the immutable +`schemas//` and `compliance//` trees through +`scripts/recover-cdn-version-trees.sh`. The run: + +- checks out tag `v` and requires its `package.json` to hold exactly + that version. The fence, the parity checker and the recovery script are + copied from the dispatched branch before the tag checkout overwrites them, so + recovery logic never comes from the artifact being recovered; +- verifies the committed signed tuple with `cosign verify-blob` against the + `release.yml` identity and runs `check-release-state.cjs published`, which + binds the tag to its GitHub Release and to the committed tuple bytes. Both + happen on the protocol path, which always runs, so tree recovery inherits + them; +- proves that the committed `dist/schemas/` and `dist/compliance/` trees + are byte-for-byte the `schemas/` and `compliance/` trees inside that signed + tarball (`scripts/check-protocol-bundle-parity.cjs`). R2 objects are + unsigned, so this is what makes the release signature cover them. cosign's + identity regexp binds the signing workflow, not the version, so the bundle's + `adcp-/` root directory is the load-bearing version binding. The + only tolerated divergence is the bounded build timestamp + `build-protocol-tarball.cjs` re-pins in `schemas/manifest.json` and + `compliance/index.json` — the one field the signature does not cover, so it + is constrained by shape and size instead. Any other difference, extra file or + missing file fails the run before a single write; +- reads every published byte with `git cat-file blob`, never the working tree + and never `git archive`, whose output is filtered by the tag's own + `.gitattributes` (`export-subst` splices commit text into the bytes). It + refuses any path that is not a regular blob with an unambiguous name and a + known published content type, and re-checks each file's digest immediately + before its write; +- writes conditionally (`If-None-Match: *`) and only where the object is + confirmed missing. An existing object must already match byte-for-byte *and* + carry the right content type; an inconclusive existence check aborts rather + than reading as absent. Nothing is overwritten or deleted, and the root + discovery pointers stay with the deploy path; +- re-checks the current-branch publication fence before and after every object, + so a branch move mid-run stops publication; +- then fetches every recovered object from the public origin and requires the + bytes, the immutable cache policy **and** the `ETag` to match. The ETag is + the decisive one: R2 reports a single-part object's ETag as the body's hex + MD5, so it proves a bucket object with exactly those bytes answered. Bytes + and cache policy alone cannot, because on a miss the worker proxies to the + Fly origin, which serves the same committed tree and stamps the same + immutable policy for an exact pinned hit + (`server/src/schemas-middleware.ts`). + +The dispatch is idempotent and safe to re-run: a complete recovery reports +every object as already verified. + +Operational notes: + +- **It is slow.** Recovery fences the current branch before and after every + object, exactly as release publication does, so ~770 objects means ~2,300 + `git ls-remote` round trips plus the S3 calls. Budget tens of minutes, and + expect a transient failure to abort mid-run; re-dispatch resumes safely. +- **A branch move fails a successful run.** Anything merging to `main` during + the upload trips the fence in the verification step, so the job can go red + after publication actually succeeded. Re-dispatch to confirm. +- **The line alias moves as soon as the first object lands.** The worker + resolves `v3`/`v3.0` from the R2 prefix listing, and a prefix exists after + the first write, so mid-run those aliases already resolve to the version + being recovered while its tree is incomplete. Objects not yet uploaded miss + R2 and fall through to the origin. This is the same progressive-publication + window `release.yml` has for a new release, not a new one, but it is a window + — prefer running recovery when the line is quiet. +- The discovery pointers advance on the next successful `main` deploy that sees + the new prefix, not as part of recovery. ### Explicit recovery of a superseded release merge diff --git a/package.json b/package.json index 150fa903ef5..dc309f8d380 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,7 @@ "test:build-schemas-hoist-by-identity": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-hoist-by-identity.test.cjs", "test:build-schemas-async-response-refs": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-async-response-refs.test.cjs", "test:build-schemas-vendor-metric-uniqueness": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-vendor-metric-uniqueness.test.cjs", - "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs tests/maintenance-release-publication.test.cjs tests/changesets-fence.test.cjs tests/release-supersession.test.cjs tests/skip-withdrawn-release.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs && node --test --test-timeout=30000 tests/cdn-artifacts.test.cjs tests/artifact-cdn-worker.test.cjs tests/schema-discovery-pointers.test.cjs", + "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs tests/maintenance-release-publication.test.cjs tests/changesets-fence.test.cjs tests/release-supersession.test.cjs tests/skip-withdrawn-release.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs && node --test --test-timeout=30000 tests/cdn-artifacts.test.cjs tests/artifact-cdn-worker.test.cjs tests/schema-discovery-pointers.test.cjs && node --test --test-timeout=120000 tests/recover-cdn-version-trees.test.cjs", "promote:rc": "node scripts/promote-release-candidate.mjs", "test:immutable-release-artifacts": "node tests/immutable-release-artifacts.test.cjs", "test:runtime-artifacts": "node --test --test-force-exit --test-timeout=30000 tests/runtime-artifact-pruning.test.cjs", diff --git a/scripts/check-protocol-bundle-parity.cjs b/scripts/check-protocol-bundle-parity.cjs new file mode 100755 index 00000000000..838fbbf4b93 --- /dev/null +++ b/scripts/check-protocol-bundle-parity.cjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +// Prove that a release's committed schema and compliance trees are exactly the +// trees inside that release's signed protocol tarball. +// +// CDN recovery (RELEASING.md "Recovering a release's CDN artifacts") republishes +// hundreds of individual objects for a historical version. Nothing in R2 is +// signed, so the only cryptographic authority available is the cosign signature +// over `dist/protocol/.tgz`. `build-protocol-tarball.cjs` stages that +// tarball by copying `dist/schemas/` and `dist/compliance/` +// verbatim, so proving tree parity makes that one signature transitively cover +// every object recovery is about to write — every byte except the value of one +// bounded timestamp field in the two files named below, which the bundle build +// re-pins and which is therefore constrained by shape and size rather than by +// the signature. +// +// Usage: +// node check-protocol-bundle-parity.cjs +// +// released-root directory holding dist/schemas/ and +// dist/compliance/, materialized from the tag's +// Git blobs +// bundle-root the extracted `adcp-/` directory from the tarball +// +// Exits non-zero, with no partial success, on any divergence. + +const fs = require("node:fs"); +const path = require("node:path"); + +const STABLE_SEMVER = /^\d+\.\d+\.\d+$/; +// Bounded fractional seconds. The published bytes for the two files below are +// the committed ones, so this field's value is the only thing the release +// signature does not cover; an unbounded fraction would let a release PR +// commit a multi-megabyte "timestamp" that passes parity and is then +// unfixable under the no-overwrite rule. +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; +// Belt and braces on the same bound, in bytes rather than shape. +const MAX_PINNED_DELTA_BYTES = 64; + +// The only divergence `build-protocol-tarball.cjs` is allowed to introduce. +// `pinGeneratedAt` rewrites the staged copy's build timestamp, which is written +// by a later build step than the one that wrote the committed tree. Its sibling +// `pinPublishedVersion` only runs when the bundle is republished under a +// different version, which a release build never does, so nothing else may +// differ. Keyed by surface so a compliance file named manifest.json, or a +// schema file named index.json, gets no exemption. +const PINNED_BUILD_TIMESTAMP_FILES = new Map([ + ["schemas", "manifest.json"], + ["compliance", "index.json"], +]); + +function fail(message) { + throw new Error(message); +} + +// Reject anything that is not a plain file or directory, so a symlink cannot +// smuggle bytes from outside the compared tree into the comparison or the +// upload that follows it. +function walk(root, surface) { + const files = []; + const visit = (relative) => { + const absolute = path.join(root, relative || "."); + const metadata = fs.lstatSync(absolute); + if (metadata.isSymbolicLink()) + fail(`Symlinked ${surface} entry: ${relative}`); + if (metadata.isDirectory()) { + for (const name of fs.readdirSync(absolute).sort()) + visit(relative ? path.join(relative, name) : name); + return; + } + if (!metadata.isFile()) fail(`Unsupported ${surface} entry: ${relative}`); + files.push(relative); + }; + visit(""); + if (files.length === 0) fail(`Empty ${surface} tree: ${root}`); + return files; +} + +function parsePinnedDocument(bytes, label) { + let document; + try { + document = JSON.parse(bytes.toString("utf8")); + } catch { + return fail(`Pinned build-timestamp file is not JSON: ${label}`); + } + if ( + !document || + typeof document !== "object" || + Array.isArray(document) || + typeof document.generated_at !== "string" || + !ISO_TIMESTAMP.test(document.generated_at) + ) + fail(`Pinned build-timestamp file has no ISO generated_at: ${label}`); + return document; +} + +// Byte equality after substituting exactly the one value the bundle build is +// allowed to rewrite. Deep JSON equality would also accept reordered keys and +// reformatted bytes; this does not. +function matchesExceptPinnedTimestamp(releasedBytes, bundleBytes, label) { + if (Math.abs(releasedBytes.length - bundleBytes.length) > MAX_PINNED_DELTA_BYTES) + fail(`Pinned build-timestamp file differs by more than a timestamp: ${label}`); + const released = parsePinnedDocument(releasedBytes, label); + const bundle = parsePinnedDocument(bundleBytes, label); + const needle = Buffer.from( + `"generated_at": ${JSON.stringify(bundle.generated_at)}`, + "utf8", + ); + const replacement = Buffer.from( + `"generated_at": ${JSON.stringify(released.generated_at)}`, + "utf8", + ); + const first = bundleBytes.indexOf(needle); + if (first === -1) return false; + if (bundleBytes.indexOf(needle, first + needle.length) !== -1) + fail(`Ambiguous generated_at occurrences in ${label}`); + const normalized = Buffer.concat([ + bundleBytes.subarray(0, first), + replacement, + bundleBytes.subarray(first + needle.length), + ]); + return normalized.equals(releasedBytes); +} + +function compareSurface(surface, releasedRoot, bundleRoot) { + for (const [label, root] of [ + [`released ${surface}`, releasedRoot], + [`bundled ${surface}`, bundleRoot], + ]) + if (!fs.existsSync(root)) fail(`Missing ${label} tree: ${root}`); + + const released = walk(releasedRoot, `released ${surface}`); + const bundle = walk(bundleRoot, `bundled ${surface}`); + const bundled = new Set(bundle); + const expected = new Set(released); + const extra = bundle.filter((file) => !expected.has(file)); + const missing = released.filter((file) => !bundled.has(file)); + if (missing.length || extra.length) + fail( + `Bundled ${surface} tree does not match the released tree` + + (missing.length ? `; missing ${missing.slice(0, 5).join(", ")}` : "") + + (extra.length ? `; unexpected ${extra.slice(0, 5).join(", ")}` : ""), + ); + + const pinned = PINNED_BUILD_TIMESTAMP_FILES.get(surface); + for (const file of released) { + const releasedBytes = fs.readFileSync(path.join(releasedRoot, file)); + const bundleBytes = fs.readFileSync(path.join(bundleRoot, file)); + if (releasedBytes.equals(bundleBytes)) continue; + const label = `${surface}/${file}`; + if ( + file === pinned && + matchesExceptPinnedTimestamp(releasedBytes, bundleBytes, label) + ) + continue; + fail(`Released ${label} differs from the signed protocol bundle`); + } + return released.length; +} + +function checkBundleParity(version, releasedRoot, bundleRoot) { + if (!STABLE_SEMVER.test(version || "")) + fail("Bundle parity requires an exact stable semver version."); + const counts = {}; + for (const surface of ["schemas", "compliance"]) { + counts[surface] = compareSurface( + surface, + path.join(releasedRoot, "dist", surface, version), + path.join(bundleRoot, surface), + ); + } + return counts; +} + +if (require.main === module) { + const [version, releasedRoot, bundleRoot] = process.argv.slice(2); + if (!version || !releasedRoot || !bundleRoot) { + console.error( + "Usage: check-protocol-bundle-parity.cjs ", + ); + process.exitCode = 2; + } else { + try { + const counts = checkBundleParity(version, releasedRoot, bundleRoot); + console.log( + `Signed protocol bundle ${version} covers ${counts.schemas} schema and ${counts.compliance} compliance files.`, + ); + } catch (error) { + console.error( + `::error::${error.message}\nRecovery stopped: the released trees are not the signed bundle's trees, so the release signature does not cover them. See RELEASING.md.`, + ); + process.exitCode = 1; + } + } +} + +module.exports = { checkBundleParity }; diff --git a/scripts/recover-cdn-version-trees.sh b/scripts/recover-cdn-version-trees.sh new file mode 100755 index 00000000000..030a4c99422 --- /dev/null +++ b/scripts/recover-cdn-version-trees.sh @@ -0,0 +1,401 @@ +#!/usr/bin/env bash +# Recover a released stable version's immutable schema and compliance trees on +# the artifact CDN, or verify that the public CDN serves them byte-exact. +# +# Why this exists: a release whose `schemas//` and +# `compliance//` objects never reached R2 does not 404. The CDN +# worker's pinned-semver fallback resolves the request to the nearest published +# release on the line, so a consumer pinning the missing version silently gets +# an older release's bytes, and the schema discovery pointers keep that line +# pinned one release back (#8318, #8324, #8347). `release.yml` cannot republish +# a historical version (`check-release-state.cjs committed` requires the local, +# current-tested and original committed package versions to be equal), routine +# deployment excludes every semver path, and live bulk backfill is disabled. +# +# Authority model (RELEASING.md "Publication authority and ordering"): +# +# - The caller checks out tag `v$VERSION`, verifies the signed protocol tuple +# with cosign bound to `release.yml`, and runs `check-release-state.cjs +# published` to bind that tag to its GitHub Release and to the committed +# tuple bytes. This script re-checks the tarball checksum, then proves that +# the trees it is about to publish are byte-for-byte the trees inside that +# signed tarball, so the release signature transitively covers every object +# it writes. +# - Every byte comes from `git cat-file blob`, never the working tree and never +# `git archive` (which applies the tag's own `.gitattributes`, so +# `export-subst` would splice commit text into published bytes). +# - This script, the fence and the parity checker are supplied by the caller +# from the current reviewed branch, never from the checked-out tag. +# - Writes are conditional (`If-None-Match: *`). An object that already exists +# must already match byte-for-byte; nothing is ever overwritten or deleted. +# - The current-branch publication fence runs before and after every write. +# +# Usage: recover-cdn-version-trees.sh +# +# Environment: +# VERSION exact stable semver (required) +# FENCE_SCRIPT path to the preserved check-release-state.cjs +# PARITY_SCRIPT path to the preserved check-protocol-bundle-parity.cjs +# BUCKET destination R2 bucket (upload only) +# R2_ENDPOINT or R2_ACCOUNT_ID (upload only) +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (upload only) +# CDN_BASE_URL public origin to verify (default adcontextprotocol.org) +# CDN_VERIFY_ATTEMPTS verification attempts before failing (default 5) + +set -euo pipefail + +mode="${1:-}" +if [[ "$mode" != upload && "$mode" != verify ]]; then + echo "::error::Usage: recover-cdn-version-trees.sh " >&2 + exit 2 +fi + +die() { + echo "::error::$1" >&2 + echo "Recovery stopped. See RELEASING.md \"Recovering a release's CDN artifacts\"." >&2 + exit 1 +} + +VERSION="${VERSION:-}" +# Stable only: prereleases are never recovered, matching recover-protocol-cdn.yml. +[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + die "VERSION must be an exact stable semver (for example, 3.0.27)" +version_re="${VERSION//./\\.}" + +FENCE_SCRIPT="${FENCE_SCRIPT:-}" +PARITY_SCRIPT="${PARITY_SCRIPT:-}" +[[ -f "$FENCE_SCRIPT" ]] || die "FENCE_SCRIPT must point at the preserved publication fence" +[[ -f "$PARITY_SCRIPT" ]] || die "PARITY_SCRIPT must point at the preserved bundle parity checker" + +cdn_base_url="${CDN_BASE_URL:-https://adcontextprotocol.org}" +attempts="${CDN_VERIFY_ATTEMPTS:-5}" +[[ "$attempts" =~ ^[1-9][0-9]*$ ]] || die "CDN_VERIFY_ATTEMPTS must be a positive integer" +immutable_cache="public, max-age=31536000, immutable" + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d ' ' -f1 + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | cut -d ' ' -f1 + else + return 1 + fi +} + +# R2 reports a single-part object's ETag as the hex MD5 of its bytes, and +# `aws s3api put-object` is always single part. That makes the served ETag a +# direct statement about the object in the bucket, which is what verification +# below needs. MD5 is used here only as R2's object identifier, never as a +# security property: the bytes themselves are compared too. +md5_of() { + if command -v md5sum >/dev/null 2>&1; then + md5sum "$1" | cut -d ' ' -f1 + elif command -v md5 >/dev/null 2>&1; then + md5 -q "$1" + else + return 1 + fi +} + +work="" +cleanup() { + if [[ -n "$work" ]]; then rm -rf -- "$work"; fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +work="$(mktemp -d "${TMPDIR:-/tmp}/adcp-cdn-recovery.XXXXXX")" +chmod 700 "$work" + +# The fence proves the dispatch is still current for the supported release +# branch. It is the same gate release publication uses, and the only one this +# recovery shares with it. +node "$FENCE_SCRIPT" current /dev/null || + die "missing ${tuple}${suffix} in the checked-out release" +done + +# --------------------------------------------------------------------------- +# Enumerate the released trees from the tag's Git objects. +# --------------------------------------------------------------------------- + +schemas_root="dist/schemas/${VERSION}" +compliance_root="dist/compliance/${VERSION}" + +# A path's extension decides the content type the CDN will serve it with. An +# unmapped extension fails the whole run rather than publishing a tree with a +# silently mistyped hole in it, which is the same class of bug as the missing +# directory this recovers. +content_type_for() { + case "$1" in + *.jsonl) printf 'application/x-ndjson; charset=utf-8' ;; + *.json) printf 'application/json; charset=utf-8' ;; + *.yaml | *.yml) printf 'application/yaml; charset=utf-8' ;; + *.md | *.mdx) printf 'text/markdown; charset=utf-8' ;; + *.txt) printf 'text/plain; charset=utf-8' ;; + *) printf '' ;; + esac +} + +# Validate while the enumeration is still NUL-delimited. A path containing a +# newline would otherwise split into two plausible-looking lines downstream. +tree_list="$work/tree.list" +: > "$tree_list" +while IFS= read -r -d '' row; do + meta="${row%%$'\t'*}" + file="${row#*$'\t'}" + read -r mode_bits object_type _oid <<< "$meta" + # Regular blobs only: a symlink, submodule or subtree entry here would let + # publication read bytes the release never committed at that path. + [[ "$object_type" == blob && ( "$mode_bits" == 100644 || "$mode_bits" == 100755 ) ]] || + die "unsupported Git entry in the released trees: ${file}" + key="${file#dist/}" + # Keys become R2 keys and public URLs. Hold them to an unambiguous charset so + # neither needs escaping and neither can be reinterpreted. + [[ "$key" =~ ^(schemas|compliance)/${version_re}/[A-Za-z0-9][A-Za-z0-9._/-]*$ && "$key" != *..* ]] || + die "unsupported released artifact path: ${file}" + [[ -n "$(content_type_for "$key")" ]] || + die "no published content type for ${key}; recovery would leave the tree incomplete" + printf '%s\n' "$file" >> "$tree_list" +done < <(git ls-tree -r -z HEAD -- "$schemas_root" "$compliance_root") + +LC_ALL=C sort -o "$tree_list" "$tree_list" +[[ -s "$tree_list" ]] || die "release ${VERSION} commits no schema or compliance tree" +[[ "$(LC_ALL=C sort -u "$tree_list" | wc -l)" == "$(wc -l < "$tree_list")" ]] || + die "ambiguous duplicate paths in the released trees" + +for root in "$schemas_root" "$compliance_root"; do + awk -v prefix="${root}/" ' + index($0, prefix) == 1 { tree = 1 } + $0 == prefix "index.json" { manifest = 1 } + END { exit tree ? (manifest ? 0 : 2) : 1 } + ' "$tree_list" && continue + case "$?" in + 1) die "release ${VERSION} commits no ${root}" ;; + *) die "release ${VERSION} commits ${root} without an index.json" ;; + esac +done + +# Materialize each blob exactly as committed. `git archive` would be one +# command, but it runs the tag's own `.gitattributes` filters (`export-subst` +# splices the commit subject into the output, `export-ignore` drops files), so +# its bytes are not the blob's bytes. `git cat-file blob` is attribute-immune, +# which is why backfill-cdn-artifacts.sh uses it too. +released="$work/released" +while IFS= read -r file; do + mkdir -p "${released}/$(dirname "$file")" + git cat-file blob "HEAD:${file}" > "${released}/${file}" +done < "$tree_list" +[[ "$(find "$released" -type f | wc -l)" == "$(wc -l < "$tree_list")" ]] || + die "materialized release trees do not match the committed file set" + +# --------------------------------------------------------------------------- +# Bind the trees to the release signature through the signed tarball. +# --------------------------------------------------------------------------- + +tuple_dir="$work/tuple" +bundle="$work/bundle" +mkdir -p "$tuple_dir" "$bundle" +for suffix in "" .sha256; do + git cat-file blob "HEAD:${tuple}${suffix}" > "${tuple_dir}/${VERSION}.tgz${suffix}" +done +expected_digest="$(cut -d ' ' -f1 < "${tuple_dir}/${VERSION}.tgz.sha256")" +[[ "$(sha256_of "${tuple_dir}/${VERSION}.tgz")" == "$expected_digest" ]] || + die "committed protocol tarball ${VERSION}.tgz fails its committed checksum" +tar -xzf "${tuple_dir}/${VERSION}.tgz" -C "$bundle" --no-same-owner --no-same-permissions +# cosign's --certificate-identity-regexp binds the signing workflow, not the +# version: a renamed copy of another release's signed tuple would verify. The +# bundle's root directory name is the only thing that ties the signature to +# this version, so it is load-bearing, not a sanity check. +[[ -d "${bundle}/adcp-${VERSION}" ]] || + die "signed protocol bundle has no adcp-${VERSION} root directory" +node "$PARITY_SCRIPT" "$VERSION" "$released" "${bundle}/adcp-${VERSION}" "$digest_list" +while IFS= read -r file; do + printf '%s %s %s\n' \ + "$(sha256_of "${released}/${file}")" "$(md5_of "${released}/${file}")" "$file" >> "$digest_list" +done < "$tree_list" + +total="$(wc -l < "$tree_list" | tr -d ' ')" + +if [[ "$mode" == upload ]]; then + BUCKET="${BUCKET:-}" + endpoint="${R2_ENDPOINT:-}" + if [[ -z "$endpoint" && -n "${R2_ACCOUNT_ID:-}" ]]; then + endpoint="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" + fi + [[ -n "$BUCKET" ]] || die "Missing destination bucket" + [[ "$BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,62}$ ]] || + die "BUCKET must be a bucket name, not a URL: ${BUCKET}" + [[ -n "$endpoint" ]] || die "Missing R2 endpoint" + [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" ]] || + die "Missing R2 publication credentials" + export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-auto}" + # Bound every S3 call the way the verification fetches are bounded. The run + # makes three calls per object over a long loop; a single stalled socket on + # CLI defaults would hang the publication rather than fail it. + aws_timeouts=(--cli-connect-timeout 20 --cli-read-timeout 120) + + existing_body="$work/existing" + existing_type="$work/existing-type" + head_error="$work/head-error" + created=0 + verified=0 + + while read -r digest _md5 file <&3; do + key="${file#dist/}" + body_path="${released}/${file}" + content_type="$(content_type_for "$key")" + [[ "$(sha256_of "$body_path")" == "$digest" ]] || + die "publication body for ${key} changed after the signature proof" + + node "$FENCE_SCRIPT" current "$existing_type" 2>"$head_error"; then + aws s3 cp "s3://${BUCKET}/${key}" "$existing_body" \ + --endpoint-url "$endpoint" --region "$AWS_DEFAULT_REGION" \ + --no-cli-pager --no-progress --only-show-errors "${aws_timeouts[@]}" &2 + die "Could not safely determine whether R2 object ${key} exists" + fi + node "$FENCE_SCRIPT" current /dev/null + created=$((created + 1)) + fi + node "$FENCE_SCRIPT" current "$expected" +while read -r _digest md5 file; do + printf '%s 200 "%s" %s\n' "${file#dist/}" "$md5" "$immutable_cache" >> "$expected" +done < "$digest_list" +LC_ALL=C sort -o "$expected" "$expected" + +verify_once() { + local attempt="$1" + local bust file key line url rest + bust="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${attempt}-$(date +%s)" + rm -rf "$download_dir" + mkdir -p "$download_dir" + : > "$config" + while IFS= read -r file; do + key="${file#dist/}" + mkdir -p "${download_dir}/$(dirname "$key")" + printf 'fail\nlocation\nurl = "%s"\noutput = "%s"\n' \ + "${cdn_base_url}/${key}?cache_bust=${bust}" "${download_dir}/${key}" >> "$config" + done < "$tree_list" + + # Bound every transfer. Without this a server that accepts the connection and + # never answers hangs the job forever and the retry loop below never runs. + if ! curl --silent --show-error --fail-early \ + --parallel --parallel-immediate --parallel-max 16 \ + --connect-timeout 20 --max-time 300 --speed-limit 1024 --speed-time 60 \ + --write-out '%{url_effective} %{http_code} %header{etag} %header{cache-control}\n' \ + --config "$config" > "$report"; then + return 1 + fi + + # --parallel completes out of order, so correlate on the URL rather than + # comparing line by line. + : > "$observed" + while IFS= read -r line; do + url="${line%% *}" + rest="${line#* }" + key="${url#"${cdn_base_url}/"}" + key="${key%%\?*}" + printf '%s %s\n' "$key" "$rest" >> "$observed" + done < "$report" + LC_ALL=C sort -o "$observed" "$observed" + + if ! diff -q "$expected" "$observed" >/dev/null; then + echo "CDN responses do not match the recovered ${VERSION} objects:" >&2 + diff "$expected" "$observed" | head -n 10 >&2 + echo "A missing or mismatched ETag means the bytes did not come from the bucket." >&2 + return 1 + fi + + while IFS= read -r file; do + key="${file#dist/}" + if ! cmp -s "${released}/${file}" "${download_dir}/${key}"; then + echo "CDN object ${key} differs from the released bytes." >&2 + return 1 + fi + done < "$tree_list" + return 0 +} + +# The worker caches its published-version list for VERSION_CACHE_TTL_MS = 60s +# (workers/artifact-cdn/src/index.js), so a freshly created prefix can take +# that long to win an exact pinned hit. The cumulative backoff below reaches +# 60s at attempt 4; do not shrink it below that. +for attempt in $(seq 1 "$attempts"); do + if verify_once "$attempt"; then + echo "Public CDN serves all ${total} ${VERSION} schema and compliance objects byte-exact (attempt ${attempt})." + exit 0 + fi + if [[ "$attempt" != "$attempts" ]]; then + echo "CDN verification failed on attempt ${attempt}; retrying." + sleep $((attempt * 10)) + fi +done + +die "public CDN does not serve the recovered ${VERSION} trees byte-exact" diff --git a/tests/recover-cdn-version-trees.test.cjs b/tests/recover-cdn-version-trees.test.cjs new file mode 100644 index 00000000000..bb0738e2706 --- /dev/null +++ b/tests/recover-cdn-version-trees.test.cjs @@ -0,0 +1,797 @@ +// Recovery of a released version's immutable schema and compliance trees on +// the artifact CDN (#8347). The gap these cover: a missing version directory +// does not 404, the worker silently serves a neighbouring release, so a +// recovery that publishes the wrong bytes, overwrites an object, or merely +// uploads without checking what the CDN then serves is indistinguishable from +// the bug it is meant to fix. +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const http = require("node:http"); +const os = require("node:os"); +const path = require("node:path"); +const { execFileSync, spawn, spawnSync } = require("node:child_process"); +const YAML = require("yaml"); + +const root = path.resolve(__dirname, ".."); +const realGit = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); +const workflowPath = path.join( + root, + ".github/workflows/recover-protocol-cdn.yml", +); +const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")); +const recoverySteps = workflow.jobs.recover.steps; +const stepNamed = (name) => recoverySteps.find((s) => s.name === name); + +const VERSION = "3.0.27"; +const IMMUTABLE = "public, max-age=31536000, immutable"; +const REVALIDATE = "public, no-cache, must-revalidate"; + +// One representative file per published content type, plus a nested path. +const RELEASED_TREE = { + "schemas/index.json": '{\n "adcp_version": "3.0.27"\n}\n', + "schemas/manifest.json": + '{\n "published_version": "3.0.27",\n "file_count": 3,\n "generated_at": "2026-09-30T14:37:10.432Z"\n}\n', + "schemas/core/product.json": '{\n "title": "Product"\n}\n', + "compliance/index.json": + '{\n "adcp_version": "3.0.27",\n "generated_at": "2026-09-30T14:37:12.795Z"\n}\n', + "compliance/universal/pagination.yaml": "id: pagination\n", + "compliance/README.md": "# Compliance\n", + "compliance/test-kits/rows.jsonl": '{"row":1}\n', + "compliance/notes.txt": "notes\n", +}; + +function fixture(t, options = {}) { + const { + version = VERSION, + tree = RELEASED_TREE, + // Divergence injected into the staged bundle only, so the committed trees + // stop matching the signed tarball. + bundleOverrides = {}, + packageVersion = version, + } = options; + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "recover-cdn-trees-")); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + + const write = (file, value) => { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), value); + }; + const git = (...args) => + execFileSync(realGit, args, { + cwd: dir, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + + write("package.json", JSON.stringify({ version: packageVersion })); + for (const [relative, body] of Object.entries(tree)) { + const [surface, ...rest] = relative.split("/"); + write(`dist/${surface}/${version}/${rest.join("/")}`, body); + } + + // Stage and build the signed tarball exactly the way build-protocol-tarball + // does: the bundle's schemas/ and compliance/ are copies of the released + // trees, with only the build timestamps re-pinned. + const bundleRoot = `adcp-${version}`; + for (const [relative, body] of Object.entries(tree)) { + write(`stage/${bundleRoot}/${relative}`, body); + } + // Mirror build-protocol-tarball.cjs: the staged copies carry a later build + // timestamp than the committed trees. + for (const [relative, from] of [ + ["schemas/manifest.json", "2026-09-30T14:37:10.432Z"], + ["compliance/index.json", "2026-09-30T14:37:12.795Z"], + ]) + if (tree[relative]) + write( + `stage/${bundleRoot}/${relative}`, + tree[relative].replace(from, "2026-09-30T14:37:20.000Z"), + ); + write(`stage/${bundleRoot}/README.md`, "# AdCP Protocol Bundle\n"); + for (const [relative, body] of Object.entries(bundleOverrides)) { + if (body === null) fs.rmSync(path.join(dir, "stage", bundleRoot, relative)); + else write(`stage/${bundleRoot}/${relative}`, body); + } + + fs.mkdirSync(path.join(dir, "dist/protocol"), { recursive: true }); + const tarball = path.join(dir, `dist/protocol/${version}.tgz`); + execFileSync("tar", ["-czf", tarball, "-C", path.join(dir, "stage"), bundleRoot]); + const digest = crypto + .createHash("sha256") + .update(fs.readFileSync(tarball)) + .digest("hex"); + write(`dist/protocol/${version}.tgz.sha256`, `${digest} ${version}.tgz\n`); + write(`dist/protocol/${version}.tgz.sig`, "signature\n"); + write(`dist/protocol/${version}.tgz.crt`, "certificate\n"); + fs.rmSync(path.join(dir, "stage"), { recursive: true, force: true }); + + git("init", "-q"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.invalid"); + git("add", "."); + git("commit", "-qm", "Release"); + const head = git("rev-parse", "HEAD"); + write("remote", head); + write("calls", ""); + fs.mkdirSync(path.join(dir, "bucket"), { recursive: true }); + + // Recovery runs the preserved copies from the current branch, never the + // tag's. Mirror that: the scripts live outside the checkout. + for (const script of [ + "check-release-state.cjs", + "check-protocol-bundle-parity.cjs", + "recover-cdn-version-trees.sh", + ]) + write(`temp/${script}`, fs.readFileSync(path.join(root, "scripts", script))); + + const executable = (name, code) => { + write(`bin/${name}`, `#!/usr/bin/env node\n${code}`); + fs.chmodSync(path.join(dir, "bin", name), 0o755); + }; + const common = `const fs=require('node:fs'),path=require('node:path');const args=process.argv.slice(2);const read=f=>fs.readFileSync(f,'utf8');const log=s=>fs.appendFileSync('calls',s+'\\n');`; + executable( + "git", + common + + ` + if(args[0]==='ls-remote') { + if(process.env.REMOTE_ERROR) process.exit(1); + const advance=Number(process.env.ADVANCE_AFTER||'0'); + const count=(fs.existsSync('fence-count')?Number(read('fence-count')):0)+1; + fs.writeFileSync('fence-count',String(count)); + // Move the branch under the run, before this very fence answers. + if(advance&&count===advance) fs.writeFileSync('remote','f'.repeat(40)); + const ref=args.find(a=>a.startsWith('refs/heads/')); + console.log(read('remote')+'\\t'+ref); process.exit(0); + } + const r=require('node:child_process').spawnSync(${JSON.stringify(realGit)},args,{stdio:'inherit'});process.exit(r.status??1); + `, + ); + executable( + "aws", + common + + ` + const value=k=>args[args.indexOf(k)+1]; + const key=args.includes('--key')?value('--key'):'';const object=path.join('bucket',key); + const metaPath=object+'.meta'; + if(args[1]==='head-object') { + if(process.env.HEAD_ERROR) { console.error(process.env.HEAD_ERROR);process.exit(1); } + if(!fs.existsSync(object)) {console.error('An error occurred (404) when calling the HeadObject operation: Not Found');process.exit(1);} + const meta=fs.existsSync(metaPath)?JSON.parse(read(metaPath)):{ContentType:process.env.EXISTING_CONTENT_TYPE||'application/json; charset=utf-8'}; + if(args.includes('--query')&&value('--query')==='ContentType') console.log(meta.ContentType); + process.exit(0); + } + if(args[1]==='put-object') { + if(value('--if-none-match')!=='*') throw Error('unconditional write'); + if(fs.existsSync(object)) {console.error('PreconditionFailed');process.exit(1);} + fs.mkdirSync(path.dirname(object),{recursive:true});fs.copyFileSync(value('--body'),object); + fs.writeFileSync(metaPath,JSON.stringify({ContentType:value('--content-type'),CacheControl:value('--cache-control')})); + log('put '+key+' | '+value('--content-type')+' | '+value('--cache-control')); + process.exit(0); + } + if(args[0]==='s3'&&args[1]==='cp'&&args[2].startsWith('s3://')) { + fs.copyFileSync(path.join('bucket',args[2].split('/').slice(3).join('/')),args[3]); + process.exit(0); + } + throw Error('unexpected aws '+args.join(' ')); + `, + ); + + const env = { + PATH: `${dir}/bin:${process.env.PATH}`, + HOME: dir, + TESTED_SHA: head, + GITHUB_SHA: head, + PUBLICATION_BRANCH: "main", + GITHUB_REF_NAME: "main", + VERSION: version, + FENCE_SCRIPT: path.join(dir, "temp/check-release-state.cjs"), + PARITY_SCRIPT: path.join(dir, "temp/check-protocol-bundle-parity.cjs"), + BUCKET: "test", + R2_ENDPOINT: "https://r2.invalid", + AWS_ACCESS_KEY_ID: "test", + AWS_SECRET_ACCESS_KEY: "test", + CDN_VERIFY_ATTEMPTS: "1", + }; + + const argv = (mode) => [ + "--noprofile", + "--norc", + path.join(dir, "temp/recover-cdn-version-trees.sh"), + mode, + ]; + const run = (mode, extra = {}) => + spawnSync("bash", argv(mode), { + cwd: dir, + env: { ...env, ...extra }, + encoding: "utf8", + }); + // The verification tests answer the script's own HTTP requests from this + // process, so they cannot block its event loop on spawnSync. + const runAsync = (mode, extra = {}) => + new Promise((resolve, reject) => { + const child = spawn("bash", argv(mode), { + cwd: dir, + env: { ...env, ...extra }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (status) => resolve({ status, stdout, stderr })); + }); + + return { + dir, + head, + version, + write, + git, + run, + runAsync, + bucketKeys: () => + fs.existsSync(path.join(dir, "bucket")) + ? listFiles(path.join(dir, "bucket")) + .filter((key) => !key.endsWith(".meta")) + .sort() + : [], + calls: () => + fs + .readFileSync(path.join(dir, "calls"), "utf8") + .split("\n") + .filter(Boolean), + }; +} + +function listFiles(base, prefix = "") { + const out = []; + for (const entry of fs.readdirSync(base, { withFileTypes: true })) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) + out.push(...listFiles(path.join(base, entry.name), relative)); + else out.push(relative); + } + return out; +} + +const expectedKeys = Object.keys(RELEASED_TREE) + .map((relative) => { + const [surface, ...rest] = relative.split("/"); + return `${surface}/${VERSION}/${rest.join("/")}`; + }) + .sort(); + +// --- upload ----------------------------------------------------------------- + +test("recovery publishes every missing version object conditionally and immutably", (t) => { + const f = fixture(t); + const result = f.run("upload"); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(f.bucketKeys(), expectedKeys); + assert.match(result.stdout, /created 8 missing object\(s\), verified 0/); + + const published = new Map( + f.calls().map((line) => { + const [key, contentType, cacheControl] = line + .replace(/^put /, "") + .split(" | "); + return [key, { contentType, cacheControl }]; + }), + ); + assert.deepEqual([...published.keys()].sort(), expectedKeys); + for (const entry of published.values()) + assert.equal(entry.cacheControl, IMMUTABLE); + assert.equal( + published.get(`schemas/${VERSION}/index.json`).contentType, + "application/json; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/universal/pagination.yaml`).contentType, + "application/yaml; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/README.md`).contentType, + "text/markdown; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/test-kits/rows.jsonl`).contentType, + "application/x-ndjson; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/notes.txt`).contentType, + "text/plain; charset=utf-8", + ); +}); + +test("an already published object is verified, never rewritten", (t) => { + const f = fixture(t); + f.write( + `bucket/schemas/${VERSION}/index.json`, + RELEASED_TREE["schemas/index.json"], + ); + const result = f.run("upload"); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /created 7 missing object\(s\), verified 1/); + assert.ok( + !f.calls().some((line) => line.includes(`schemas/${VERSION}/index.json`)), + ); +}); + +test("a published object with different bytes stops recovery instead of overwriting", (t) => { + const f = fixture(t); + f.write(`bucket/compliance/${VERSION}/index.json`, '{"adcp_version":"3.0.26"}'); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /exists with different bytes; refusing overwrite/); + assert.ok( + !f.calls().some((line) => line.includes(`compliance/${VERSION}/index.json`)), + ); +}); + +test("an inconclusive existence check never reads as absent", (t) => { + const f = fixture(t); + const result = f.run("upload", { + HEAD_ERROR: "An error occurred (403) when calling the HeadObject operation: Forbidden", + }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Could not safely determine whether R2 object/); + assert.deepEqual(f.calls(), []); +}); + +for (const [label, overrides] of [ + ["a changed file", { "schemas/core/product.json": '{"title":"Tampered"}\n' }], + ["an extra file", { "schemas/core/extra.json": "{}\n" }], + ["a missing file", { "compliance/notes.txt": null }], +]) { + test(`released trees with ${label} relative to the signed bundle are never published`, (t) => { + const f = fixture(t, { bundleOverrides: overrides }); + const run = f.run("upload"); + assert.notEqual(run.status, 0); + assert.match( + run.stderr, + /signed protocol bundle|does not match the released tree/, + ); + assert.deepEqual(f.bucketKeys(), []); + assert.deepEqual(f.calls(), []); + }); +} + +test("only the bundle build's own timestamp pin may differ, and only in its own file", (t) => { + // A rewritten timestamp in a file the bundle build never pins is a content + // difference, not an exemption. + const f = fixture(t, { + bundleOverrides: { + "compliance/universal/pagination.yaml": "id: pagination\ngenerated_at: x\n", + }, + }); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /differs from the signed protocol bundle/); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("recovery refuses a prerelease, a mismatched checkout, and a stale branch", (t) => { + const prerelease = fixture(t); + const rc = prerelease.run("upload", { VERSION: "3.2.0-rc.3" }); + assert.notEqual(rc.status, 0); + assert.match(rc.stderr, /exact stable semver/); + assert.deepEqual(prerelease.calls(), []); + + const mismatched = fixture(t, { packageVersion: "3.0.26" }); + const wrongCheckout = mismatched.run("upload"); + assert.notEqual(wrongCheckout.status, 0); + assert.match(wrongCheckout.stderr, /checkout holds package version 3\.0\.26/); + assert.deepEqual(mismatched.calls(), []); + + const stale = fixture(t); + stale.write("remote", "f".repeat(40)); + const moved = stale.run("upload"); + assert.notEqual(moved.status, 0); + assert.match(moved.stderr, /Refusing stale tested SHA/); + assert.deepEqual(stale.calls(), []); +}); + +test("the branch fence is rechecked between objects, so a mid-run branch move stops publication", (t) => { + const f = fixture(t); + // Fence calls: one preflight, then three per published object. Moving the + // branch on the fifth lands inside the second object, after the first was + // already written. + const result = f.run("upload", { ADVANCE_AFTER: "5" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Refusing stale tested SHA/); + assert.equal(f.calls().length, 1); + assert.equal(f.bucketKeys().length, 1); +}); + +test("a file whose extension has no published content type stops recovery before any write", (t) => { + const f = fixture(t, { + tree: { ...RELEASED_TREE, "schemas/bundle.tar": "binary\n" }, + bundleOverrides: {}, + }); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /no published content type/); + assert.deepEqual(f.bucketKeys(), []); + assert.deepEqual(f.calls(), []); +}); + +test("a symlinked entry in the released tree is refused", (t) => { + const f = fixture(t); + f.git("rm", "-q", "--cached", `dist/schemas/${VERSION}/core/product.json`); + fs.rmSync(path.join(f.dir, `dist/schemas/${VERSION}/core/product.json`)); + fs.symlinkSync( + "../../../../package.json", + path.join(f.dir, `dist/schemas/${VERSION}/core/product.json`), + ); + f.git("add", "-A"); + f.git("commit", "-qm", "Symlinked schema"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /unsupported Git entry in the released trees/); + assert.deepEqual(f.calls(), []); +}); + +test("published bytes are the committed blobs, not a .gitattributes rewrite", (t) => { + // `git archive` applies the tag's own export-subst filter, which splices the + // commit subject into the bytes it emits. Recovery must read blobs instead. + const f = fixture(t, { + tree: { ...RELEASED_TREE, "schemas/core/product.json": '{"s":"$Format:%s$"}\n' }, + }); + f.write(".gitattributes", `dist/schemas/${VERSION}/core/product.json export-subst\n`); + f.git("add", "-A"); + f.git("commit", "-qm", "SUBSTITUTED-COMMIT-SUBJECT"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.equal(result.status, 0, result.stderr); + const published = fs.readFileSync( + path.join(f.dir, `bucket/schemas/${VERSION}/core/product.json`), + "utf8", + ); + assert.equal(published, '{"s":"$Format:%s$"}\n'); + assert.doesNotMatch(published, /SUBSTITUTED-COMMIT-SUBJECT/); +}); + +test("a tarball that fails its committed checksum stops recovery", (t) => { + const f = fixture(t); + f.write(`dist/protocol/${VERSION}.tgz.sha256`, `${"0".repeat(64)} ${VERSION}.tgz\n`); + f.git("add", "-A"); + f.git("commit", "-qm", "Broken checksum"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /fails its committed checksum/); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("a bundle for another version cannot stand in for this one", (t) => { + // cosign binds the signing workflow, not the version, so the bundle's root + // directory name is the only thing tying the signature to this release. + const f = fixture(t); + const stage = path.join(f.dir, "restage"); + fs.mkdirSync(path.join(stage, "adcp-3.0.26"), { recursive: true }); + fs.writeFileSync(path.join(stage, "adcp-3.0.26", "README.md"), "other\n"); + execFileSync("tar", [ + "-czf", path.join(f.dir, `dist/protocol/${VERSION}.tgz`), + "-C", stage, "adcp-3.0.26", + ]); + const digest = crypto + .createHash("sha256") + .update(fs.readFileSync(path.join(f.dir, `dist/protocol/${VERSION}.tgz`))) + .digest("hex"); + f.write(`dist/protocol/${VERSION}.tgz.sha256`, `${digest} ${VERSION}.tgz\n`); + f.git("add", "-A"); + f.git("commit", "-qm", "Swapped bundle"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, new RegExp(`no adcp-${VERSION} root directory`)); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("a released tree without its index.json is refused", (t) => { + const tree = { ...RELEASED_TREE }; + delete tree["compliance/index.json"]; + const f = fixture(t, { tree }); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /without an index\.json/); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("an existing object with the right bytes but the wrong content type is refused", (t) => { + // No-overwrite means a mistyped object cannot be repaired here, and the CDN + // serves the stored type, so this must stop rather than report "verified". + const f = fixture(t); + f.write( + `bucket/schemas/${VERSION}/index.json`, + RELEASED_TREE["schemas/index.json"], + ); + f.write( + `bucket/schemas/${VERSION}/index.json.meta`, + JSON.stringify({ ContentType: "text/plain; charset=utf-8" }), + ); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /exists with content type text\/plain/); +}); + +test("upload refuses an unusable destination or missing credentials", (t) => { + const f = fixture(t); + for (const [extra, pattern] of [ + [{ BUCKET: "s3://adcp-artifacts" }, /must be a bucket name, not a URL/], + [{ BUCKET: "" }, /Missing destination bucket/], + [{ R2_ENDPOINT: "", R2_ACCOUNT_ID: "" }, /Missing R2 endpoint/], + [{ AWS_SECRET_ACCESS_KEY: "" }, /Missing R2 publication credentials/], + ]) { + const result = f.run("upload", extra); + assert.notEqual(result.status, 0); + assert.match(result.stderr, pattern); + } + assert.deepEqual(f.bucketKeys(), []); + assert.deepEqual(f.calls(), []); +}); + +test("the script refuses an unknown mode and an unusable retry budget", (t) => { + const f = fixture(t); + const mode = f.run("publish"); + assert.equal(mode.status, 2); + assert.match(mode.stderr, /Usage: recover-cdn-version-trees\.sh/); + const budget = f.run("verify", { CDN_VERIFY_ATTEMPTS: "0" }); + assert.notEqual(budget.status, 0); + assert.match(budget.stderr, /CDN_VERIFY_ATTEMPTS must be a positive integer/); +}); + +// --- verify ----------------------------------------------------------------- + +function serveCdn(t, handler) { + const server = http.createServer(handler); + server.listen(0, "127.0.0.1"); + // curl keeps its parallel connections alive, so close() alone would leave + // the test process waiting on them after the assertions are done. + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + return new Promise((resolve) => + server.once("listening", () => + resolve(`http://127.0.0.1:${server.address().port}`), + ), + ); +} + +// An R2 hit through the worker: the released bytes, the immutable policy, and +// R2's single-part ETag, which is the body's hex MD5. +function exactServer(f, options = {}) { + const { + cacheControl = IMMUTABLE, + corrupt = null, + // Models the worker proxying an R2 miss to the Fly origin, which serves + // the same committed tree under the same immutable policy but cannot + // produce R2's ETag. + etagFor = (body) => `"${crypto.createHash("md5").update(body).digest("hex")}"`, + } = options; + // The served set is enumerated once, up front. A request only ever selects + // an entry from it, so no request value reaches the filesystem and an + // unknown or traversing path simply has no entry. + const servedRoot = path.resolve(f.dir, "dist"); + const served = new Map( + listFiles(servedRoot).map((relative) => [ + relative, + path.join(servedRoot, relative), + ]), + ); + return (request, response) => { + const key = decodeURIComponent(request.url.split("?")[0]).replace( + /^\/+/, + "", + ); + const file = served.get(key); + if (file === undefined) { + response.writeHead(404).end(); + return; + } + const body = + key === corrupt ? Buffer.from("tampered\n") : fs.readFileSync(file); + const headers = { "cache-control": cacheControl }; + const etag = etagFor(body, key); + if (etag) headers.etag = etag; + response.writeHead(200, headers); + response.end(request.method === "HEAD" ? undefined : body); + }; +} + +test("verification passes only when the CDN serves the released bytes", async (t) => { + const f = fixture(t); + const base = await serveCdn(t, exactServer(f)); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.equal(result.status, 0, result.stderr + result.stdout); + assert.match(result.stdout, /serves all 8 3\.0\.27 schema and compliance objects byte-exact/); +}); + +test("verification fails when any served object differs", async (t) => { + const f = fixture(t); + const corrupt = `schemas/${VERSION}/core/product.json`; + const base = await serveCdn( + t, + // Serve tampered bytes under the ETag the real object would have, so the + // header checks pass and the byte comparison is the thing that must catch + // it. + exactServer(f, { + corrupt, + etagFor: (body, key) => + `"${crypto + .createHash("md5") + .update( + key === corrupt + ? fs.readFileSync(path.join(f.dir, "dist", key)) + : body, + ) + .digest("hex")}"`, + }), + ); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /differs from the released bytes/); +}); + +test("verification fails when the version is still served under a fallback policy", async (t) => { + // Before recovery the worker resolves the pin to a neighbouring release and + // stamps the revalidate policy. Byte comparison alone can pass when the two + // releases happen to agree, so the policy must be checked too. + const f = fixture(t); + const base = await serveCdn(t, exactServer(f, { cacheControl: REVALIDATE })); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /do not match the recovered 3\.0\.27 objects/); +}); + +test("verification fails when the bytes come from the origin rather than the bucket", async (t) => { + // The case the policy check alone cannot catch: on an R2 miss the worker + // proxies to Fly, which serves the same committed tree AND stamps the same + // immutable policy for an exact pinned hit. Only R2 produces the body's MD5 + // as the ETag, so that is what proves the object is really in the bucket. + const f = fixture(t); + const base = await serveCdn( + t, + exactServer(f, { etagFor: () => 'W/"1a2-18f3c4d5e6f"' }), + ); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not come from the bucket/); +}); + +test("verification fails when a served ETag is absent", async (t) => { + const f = fixture(t); + const base = await serveCdn(t, exactServer(f, { etagFor: () => null })); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /do not match the recovered 3\.0\.27 objects/); +}); + +test("verification retries while the CDN catches up, then passes", async (t) => { + // The worker caches its published-version list for 60s, so a freshly created + // prefix can need a retry before it wins an exact pinned hit. + const f = fixture(t); + // Each attempt uses a distinct cache-bust token, which is the only + // order-independent way to tell the rounds apart under parallel requests. + const busts = new Set(); + const base = await serveCdn(t, (request, response) => { + const bust = new URL(request.url, "http://cdn.invalid").searchParams.get( + "cache_bust", + ); + busts.add(bust); + const isFirstRound = [...busts][0] === bust; + exactServer(f, isFirstRound ? { cacheControl: REVALIDATE } : {})( + request, + response, + ); + }); + const result = await f.runAsync("verify", { + CDN_BASE_URL: base, + CDN_VERIFY_ATTEMPTS: "2", + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /failed on attempt 1; retrying/); + assert.match(result.stdout, /byte-exact \(attempt 2\)/); +}); + +test("verification fails when an object is missing from the CDN", async (t) => { + const f = fixture(t); + const base = await serveCdn(t, (request, response) => { + if (request.url.includes("notes.txt")) { + response.writeHead(404).end(); + return; + } + exactServer(f)(request, response); + }); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.doesNotMatch(result.stdout, /byte-exact/); +}); + +// --- workflow wiring -------------------------------------------------------- + +test("the dispatch keeps its existing protocol-only default", () => { + const inputs = workflow.on.workflow_dispatch.inputs; + assert.equal(inputs.surfaces.type, "choice"); + assert.equal(inputs.surfaces.default, "protocol"); + assert.deepEqual(inputs.surfaces.options, ["protocol", "all"]); + assert.equal(workflow.permissions.contents, "read"); + assert.equal(workflow.env.PUBLICATION_BRANCH, "main"); + // One recovery per version, whatever the surfaces, so two dispatches cannot + // interleave reads and conditional writes on the same objects. + assert.ok(!workflow.concurrency.group.includes("surfaces")); + assert.equal(workflow.concurrency["cancel-in-progress"], false); +}); + +test("tree recovery runs the current branch's scripts, not the recovered tag's", () => { + const preserve = stepNamed("Preserve current workflow publication fence"); + const checkout = recoverySteps.findIndex( + (s) => s.name === "Validate release input and checkout tag", + ); + assert.ok( + recoverySteps.indexOf(preserve) < checkout, + "scripts must be preserved before the tag checkout replaces them", + ); + for (const script of [ + "check-release-state.cjs", + "check-protocol-bundle-parity.cjs", + "recover-cdn-version-trees.sh", + ]) { + assert.ok( + preserve.run.includes(`cp scripts/${script} "$RUNNER_TEMP/${script}"`), + `${script} must be preserved from the current branch`, + ); + assert.ok( + fs.existsSync(path.join(root, "scripts", script)), + `${script} must exist on the current branch`, + ); + } + + for (const [name, mode] of [ + ["Upload only missing immutable schema and compliance objects", "upload"], + ["Verify recovered public CDN trees", "verify"], + ]) { + const step = stepNamed(name); + assert.ok(step, `${name} must exist`); + assert.equal(step.if, "inputs.surfaces == 'all'"); + assert.equal( + step.run.trim(), + `bash "$RUNNER_TEMP/recover-cdn-version-trees.sh" ${mode}`, + ); + assert.equal(step.env.VERSION, "${{ inputs.version }}"); + assert.equal(step.env.FENCE_SCRIPT, "${{ runner.temp }}/check-release-state.cjs"); + assert.equal( + step.env.PARITY_SCRIPT, + "${{ runner.temp }}/check-protocol-bundle-parity.cjs", + ); + // Retry budget and CDN origin stay at the script defaults. + assert.ok(!("CDN_VERIFY_ATTEMPTS" in step.env)); + assert.ok(!("CDN_BASE_URL" in step.env)); + } + + const verify = stepNamed("Verify recovered public CDN trees"); + for (const secret of ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "BUCKET"]) + assert.ok( + !(secret in verify.env), + `${secret} must not be exposed to the read-only verification step`, + ); +}); + +test("every recovery step still fails closed on the current-branch fence", () => { + const protocolUpload = stepNamed("Upload only missing immutable R2 objects"); + assert.match(protocolUpload.run, /check-release-state\.cjs" current/); + const script = fs.readFileSync( + path.join(root, "scripts/recover-cdn-version-trees.sh"), + "utf8", + ); + assert.match(script, /node "\$FENCE_SCRIPT" current/); + assert.match(script, /--if-none-match '\*'/); + assert.doesNotMatch(script, /s3 rm|--delete|rb /); +});