From e8dfb2fe72ca2ae7a49599ece0d8d7bb10d30d8b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 19 Oct 2026 22:47:27 +0000 Subject: [PATCH 1/4] fix(release): recover a release's schema and compliance trees on the CDN schemas/3.0.27, schemas/3.0.28 and both compliance trees never reached the artifact bucket. That does not 404: the worker's pinned-semver fallback resolves the pin to the nearest published release on the line, so those versions answer 200 with 3.0.26 bytes, and the discovery pointers keep v3.0 pinned one release back while every deploy logs the gap. No path could fix it. release.yml publishes only the current checkout's package version, routine deploy excludes every semver path, and live bulk backfill is disabled, so RELEASING.md told that warning's reader to recover releases the repository had no way to recover. Add a surfaces input to recover-protocol-cdn.yml. protocol stays the default and is unchanged; all additionally recovers the immutable version trees. R2 objects are unsigned, so the authority comes from the one signature that exists: build-protocol-tarball.cjs stages the tarball by copying the dist trees verbatim, so proving tree parity against the cosign-verified tarball makes that signature cover every object. The bundle's adcp- root is the version binding, since cosign's identity regexp binds the workflow and not the version. Bytes are read with git cat-file blob rather than git archive, whose output the tag's own .gitattributes can rewrite. Writes are conditional and only where the object is confirmed missing; an existing object must match both bytes and content type. Verification then requires the public origin to return the bytes, the immutable policy and the R2 ETag, because the Fly origin serves the same committed tree under the same policy on a miss and only R2 reports the body's MD5 as its ETag. Closes #8347. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/recover-protocol-cdn.yml | 49 +- RELEASING.md | 96 ++- package.json | 2 +- scripts/check-protocol-bundle-parity.cjs | 196 ++++++ scripts/recover-cdn-version-trees.sh | 396 +++++++++++ tests/recover-cdn-version-trees.test.cjs | 784 +++++++++++++++++++++ 6 files changed, 1517 insertions(+), 6 deletions(-) create mode 100755 scripts/check-protocol-bundle-parity.cjs create mode 100755 scripts/recover-cdn-version-trees.sh create mode 100644 tests/recover-cdn-version-trees.test.cjs diff --git a/.github/workflows/recover-protocol-cdn.yml b/.github/workflows/recover-protocol-cdn.yml index 7cc73a26a49..b289e421c35 100644 --- a/.github/workflows/recover-protocol-cdn.yml +++ b/.github/workflows/recover-protocol-cdn.yml @@ -1,4 +1,4 @@ -name: Recover protocol release on CDN +name: Recover release artifacts on CDN on: workflow_dispatch: @@ -7,7 +7,19 @@ on: description: Exact stable semver to recover (for example, 3.0.22) required: true type: string - + surfaces: + description: >- + protocol = the four signed tuple files only. all = also the immutable + schemas// and compliance// trees. + required: false + default: protocol + type: choice + options: + - protocol + - all + +# One recovery per version at a time, whatever the requested surfaces, so two +# dispatches cannot interleave reads and conditional writes on the same objects. concurrency: group: recover-protocol-cdn-${{ inputs.version }} cancel-in-progress: false @@ -21,7 +33,7 @@ permissions: jobs: recover: - name: Recover immutable protocol tuple + name: Recover immutable release artifacts runs-on: ubuntu-latest steps: - name: Checkout repository history @@ -29,10 +41,15 @@ jobs: with: fetch-depth: 0 + # The tag checked out below carries its own copies of these scripts, and + # an old release predates them entirely. Recovery logic must come from + # the current reviewed branch, never from the artifact being recovered. - name: Preserve current workflow publication fence run: | node scripts/check-release-state.cjs current cp scripts/check-release-state.cjs "$RUNNER_TEMP/check-release-state.cjs" + cp scripts/check-protocol-bundle-parity.cjs "$RUNNER_TEMP/check-protocol-bundle-parity.cjs" + cp scripts/recover-cdn-version-trees.sh "$RUNNER_TEMP/recover-cdn-version-trees.sh" - name: Install cosign uses: sigstore/cosign-installer@v3 @@ -164,3 +181,29 @@ jobs: --certificate-identity-regexp '^https://github\.com/adcontextprotocol/adcp/\.github/workflows/release\.yml@refs/heads/.*$' \ --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ "${VERSION}.tgz" + + # The signed tuple above is the only cryptographic authority available + # for a historical version; R2 objects are unsigned. The script below + # proves the trees it publishes are byte-for-byte the trees inside that + # tarball before it writes anything, so one cosign verification covers + # every object. See RELEASING.md "Recovering a release's CDN artifacts". + - name: Upload only missing immutable schema and compliance objects + if: inputs.surfaces == 'all' + env: + VERSION: ${{ inputs.version }} + FENCE_SCRIPT: ${{ runner.temp }}/check-release-state.cjs + PARITY_SCRIPT: ${{ runner.temp }}/check-protocol-bundle-parity.cjs + BUCKET: ${{ vars.ADCP_ARTIFACT_R2_BUCKET || 'adcp-artifacts' }} + R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY || secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + run: bash "$RUNNER_TEMP/recover-cdn-version-trees.sh" upload + + - name: Verify recovered public CDN trees + if: inputs.surfaces == 'all' + env: + VERSION: ${{ inputs.version }} + FENCE_SCRIPT: ${{ runner.temp }}/check-release-state.cjs + PARITY_SCRIPT: ${{ runner.temp }}/check-protocol-bundle-parity.cjs + run: bash "$RUNNER_TEMP/recover-cdn-version-trees.sh" verify diff --git a/RELEASING.md b/RELEASING.md index c550744c64e..02a1ba84007 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -424,8 +424,100 @@ Live unscoped bulk backfills are disabled. `--dry-run` remains available for inventory; it is not release authorization. Historical CDN recovery must use an existing approved tag/release, a current tested branch fence, unchanged committed bytes, and conditional missing-object creation. The stable-only -`recover-protocol-cdn.yml` recovers four protocol files, not schemas/compliance -and not RC releases. +`recover-protocol-cdn.yml` is the single dispatch for that; it does not +recover RC releases. + +### Recovering a release's CDN artifacts + +A stable release whose objects never reached R2 does not announce itself. The +worker's pinned-semver fallback resolves `/schemas/3.0.28/...` to the newest +published release at-or-below it on the same line, so the request returns 200 +with an *older release's* bytes. Only two signals distinguish that from a real +hit: the response carries `cache-control: public, no-cache, must-revalidate` +instead of the immutable policy an exact directory gets, and +`build-schema-discovery-pointers.cjs` refuses to advertise the release, so the +deploy log carries `::warning title=Schema versions missing from the CDN` +naming the gap and the line's alias stays pinned one release back (#8318, +#8324, #8347). + +Nothing in the routine paths can close that gap. `release.yml` publishes only +the version in the current checkout's `package.json`; `--latest-only` excludes +every semver path; live bulk backfill is disabled. Recovery is a separate, +maintainer-dispatched authority: + +``` +gh workflow run recover-protocol-cdn.yml --ref main \ + -f version= -f surfaces=all +``` + +`surfaces` defaults to `protocol` (the four signed tuple files, the original +scope of this workflow). `all` additionally recovers the immutable +`schemas//` and `compliance//` trees through +`scripts/recover-cdn-version-trees.sh`. The run: + +- checks out tag `v` and requires its `package.json` to hold exactly + that version. The fence, the parity checker and the recovery script are + copied from the dispatched branch before the tag checkout overwrites them, so + recovery logic never comes from the artifact being recovered; +- verifies the committed signed tuple with `cosign verify-blob` against the + `release.yml` identity and runs `check-release-state.cjs published`, which + binds the tag to its GitHub Release and to the committed tuple bytes. Both + happen on the protocol path, which always runs, so tree recovery inherits + them; +- proves that the committed `dist/schemas/` and `dist/compliance/` trees + are byte-for-byte the `schemas/` and `compliance/` trees inside that signed + tarball (`scripts/check-protocol-bundle-parity.cjs`). R2 objects are + unsigned, so this is what makes the release signature cover them. cosign's + identity regexp binds the signing workflow, not the version, so the bundle's + `adcp-/` root directory is the load-bearing version binding. The + only tolerated divergence is the bounded build timestamp + `build-protocol-tarball.cjs` re-pins in `schemas/manifest.json` and + `compliance/index.json` — the one field the signature does not cover, so it + is constrained by shape and size instead. Any other difference, extra file or + missing file fails the run before a single write; +- reads every published byte with `git cat-file blob`, never the working tree + and never `git archive`, whose output is filtered by the tag's own + `.gitattributes` (`export-subst` splices commit text into the bytes). It + refuses any path that is not a regular blob with an unambiguous name and a + known published content type, and re-checks each file's digest immediately + before its write; +- writes conditionally (`If-None-Match: *`) and only where the object is + confirmed missing. An existing object must already match byte-for-byte *and* + carry the right content type; an inconclusive existence check aborts rather + than reading as absent. Nothing is overwritten or deleted, and the root + discovery pointers stay with the deploy path; +- re-checks the current-branch publication fence before and after every object, + so a branch move mid-run stops publication; +- then fetches every recovered object from the public origin and requires the + bytes, the immutable cache policy **and** the `ETag` to match. The ETag is + the decisive one: R2 reports a single-part object's ETag as the body's hex + MD5, so it proves a bucket object with exactly those bytes answered. Bytes + and cache policy alone cannot, because on a miss the worker proxies to the + Fly origin, which serves the same committed tree and stamps the same + immutable policy for an exact pinned hit + (`server/src/schemas-middleware.ts`). + +The dispatch is idempotent and safe to re-run: a complete recovery reports +every object as already verified. + +Operational notes: + +- **It is slow.** Recovery fences the current branch before and after every + object, exactly as release publication does, so ~770 objects means ~2,300 + `git ls-remote` round trips plus the S3 calls. Budget tens of minutes, and + expect a transient failure to abort mid-run; re-dispatch resumes safely. +- **A branch move fails a successful run.** Anything merging to `main` during + the upload trips the fence in the verification step, so the job can go red + after publication actually succeeded. Re-dispatch to confirm. +- **The line alias moves as soon as the first object lands.** The worker + resolves `v3`/`v3.0` from the R2 prefix listing, and a prefix exists after + the first write, so mid-run those aliases already resolve to the version + being recovered while its tree is incomplete. Objects not yet uploaded miss + R2 and fall through to the origin. This is the same progressive-publication + window `release.yml` has for a new release, not a new one, but it is a window + — prefer running recovery when the line is quiet. +- The discovery pointers advance on the next successful `main` deploy that sees + the new prefix, not as part of recovery. ### Explicit recovery of a superseded release merge diff --git a/package.json b/package.json index 150fa903ef5..dc309f8d380 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,7 @@ "test:build-schemas-hoist-by-identity": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-hoist-by-identity.test.cjs", "test:build-schemas-async-response-refs": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-async-response-refs.test.cjs", "test:build-schemas-vendor-metric-uniqueness": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-vendor-metric-uniqueness.test.cjs", - "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs tests/maintenance-release-publication.test.cjs tests/changesets-fence.test.cjs tests/release-supersession.test.cjs tests/skip-withdrawn-release.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs && node --test --test-timeout=30000 tests/cdn-artifacts.test.cjs tests/artifact-cdn-worker.test.cjs tests/schema-discovery-pointers.test.cjs", + "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs tests/maintenance-release-publication.test.cjs tests/changesets-fence.test.cjs tests/release-supersession.test.cjs tests/skip-withdrawn-release.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs && node --test --test-timeout=30000 tests/cdn-artifacts.test.cjs tests/artifact-cdn-worker.test.cjs tests/schema-discovery-pointers.test.cjs && node --test --test-timeout=120000 tests/recover-cdn-version-trees.test.cjs", "promote:rc": "node scripts/promote-release-candidate.mjs", "test:immutable-release-artifacts": "node tests/immutable-release-artifacts.test.cjs", "test:runtime-artifacts": "node --test --test-force-exit --test-timeout=30000 tests/runtime-artifact-pruning.test.cjs", diff --git a/scripts/check-protocol-bundle-parity.cjs b/scripts/check-protocol-bundle-parity.cjs new file mode 100755 index 00000000000..838fbbf4b93 --- /dev/null +++ b/scripts/check-protocol-bundle-parity.cjs @@ -0,0 +1,196 @@ +#!/usr/bin/env node +// Prove that a release's committed schema and compliance trees are exactly the +// trees inside that release's signed protocol tarball. +// +// CDN recovery (RELEASING.md "Recovering a release's CDN artifacts") republishes +// hundreds of individual objects for a historical version. Nothing in R2 is +// signed, so the only cryptographic authority available is the cosign signature +// over `dist/protocol/.tgz`. `build-protocol-tarball.cjs` stages that +// tarball by copying `dist/schemas/` and `dist/compliance/` +// verbatim, so proving tree parity makes that one signature transitively cover +// every object recovery is about to write — every byte except the value of one +// bounded timestamp field in the two files named below, which the bundle build +// re-pins and which is therefore constrained by shape and size rather than by +// the signature. +// +// Usage: +// node check-protocol-bundle-parity.cjs +// +// released-root directory holding dist/schemas/ and +// dist/compliance/, materialized from the tag's +// Git blobs +// bundle-root the extracted `adcp-/` directory from the tarball +// +// Exits non-zero, with no partial success, on any divergence. + +const fs = require("node:fs"); +const path = require("node:path"); + +const STABLE_SEMVER = /^\d+\.\d+\.\d+$/; +// Bounded fractional seconds. The published bytes for the two files below are +// the committed ones, so this field's value is the only thing the release +// signature does not cover; an unbounded fraction would let a release PR +// commit a multi-megabyte "timestamp" that passes parity and is then +// unfixable under the no-overwrite rule. +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/; +// Belt and braces on the same bound, in bytes rather than shape. +const MAX_PINNED_DELTA_BYTES = 64; + +// The only divergence `build-protocol-tarball.cjs` is allowed to introduce. +// `pinGeneratedAt` rewrites the staged copy's build timestamp, which is written +// by a later build step than the one that wrote the committed tree. Its sibling +// `pinPublishedVersion` only runs when the bundle is republished under a +// different version, which a release build never does, so nothing else may +// differ. Keyed by surface so a compliance file named manifest.json, or a +// schema file named index.json, gets no exemption. +const PINNED_BUILD_TIMESTAMP_FILES = new Map([ + ["schemas", "manifest.json"], + ["compliance", "index.json"], +]); + +function fail(message) { + throw new Error(message); +} + +// Reject anything that is not a plain file or directory, so a symlink cannot +// smuggle bytes from outside the compared tree into the comparison or the +// upload that follows it. +function walk(root, surface) { + const files = []; + const visit = (relative) => { + const absolute = path.join(root, relative || "."); + const metadata = fs.lstatSync(absolute); + if (metadata.isSymbolicLink()) + fail(`Symlinked ${surface} entry: ${relative}`); + if (metadata.isDirectory()) { + for (const name of fs.readdirSync(absolute).sort()) + visit(relative ? path.join(relative, name) : name); + return; + } + if (!metadata.isFile()) fail(`Unsupported ${surface} entry: ${relative}`); + files.push(relative); + }; + visit(""); + if (files.length === 0) fail(`Empty ${surface} tree: ${root}`); + return files; +} + +function parsePinnedDocument(bytes, label) { + let document; + try { + document = JSON.parse(bytes.toString("utf8")); + } catch { + return fail(`Pinned build-timestamp file is not JSON: ${label}`); + } + if ( + !document || + typeof document !== "object" || + Array.isArray(document) || + typeof document.generated_at !== "string" || + !ISO_TIMESTAMP.test(document.generated_at) + ) + fail(`Pinned build-timestamp file has no ISO generated_at: ${label}`); + return document; +} + +// Byte equality after substituting exactly the one value the bundle build is +// allowed to rewrite. Deep JSON equality would also accept reordered keys and +// reformatted bytes; this does not. +function matchesExceptPinnedTimestamp(releasedBytes, bundleBytes, label) { + if (Math.abs(releasedBytes.length - bundleBytes.length) > MAX_PINNED_DELTA_BYTES) + fail(`Pinned build-timestamp file differs by more than a timestamp: ${label}`); + const released = parsePinnedDocument(releasedBytes, label); + const bundle = parsePinnedDocument(bundleBytes, label); + const needle = Buffer.from( + `"generated_at": ${JSON.stringify(bundle.generated_at)}`, + "utf8", + ); + const replacement = Buffer.from( + `"generated_at": ${JSON.stringify(released.generated_at)}`, + "utf8", + ); + const first = bundleBytes.indexOf(needle); + if (first === -1) return false; + if (bundleBytes.indexOf(needle, first + needle.length) !== -1) + fail(`Ambiguous generated_at occurrences in ${label}`); + const normalized = Buffer.concat([ + bundleBytes.subarray(0, first), + replacement, + bundleBytes.subarray(first + needle.length), + ]); + return normalized.equals(releasedBytes); +} + +function compareSurface(surface, releasedRoot, bundleRoot) { + for (const [label, root] of [ + [`released ${surface}`, releasedRoot], + [`bundled ${surface}`, bundleRoot], + ]) + if (!fs.existsSync(root)) fail(`Missing ${label} tree: ${root}`); + + const released = walk(releasedRoot, `released ${surface}`); + const bundle = walk(bundleRoot, `bundled ${surface}`); + const bundled = new Set(bundle); + const expected = new Set(released); + const extra = bundle.filter((file) => !expected.has(file)); + const missing = released.filter((file) => !bundled.has(file)); + if (missing.length || extra.length) + fail( + `Bundled ${surface} tree does not match the released tree` + + (missing.length ? `; missing ${missing.slice(0, 5).join(", ")}` : "") + + (extra.length ? `; unexpected ${extra.slice(0, 5).join(", ")}` : ""), + ); + + const pinned = PINNED_BUILD_TIMESTAMP_FILES.get(surface); + for (const file of released) { + const releasedBytes = fs.readFileSync(path.join(releasedRoot, file)); + const bundleBytes = fs.readFileSync(path.join(bundleRoot, file)); + if (releasedBytes.equals(bundleBytes)) continue; + const label = `${surface}/${file}`; + if ( + file === pinned && + matchesExceptPinnedTimestamp(releasedBytes, bundleBytes, label) + ) + continue; + fail(`Released ${label} differs from the signed protocol bundle`); + } + return released.length; +} + +function checkBundleParity(version, releasedRoot, bundleRoot) { + if (!STABLE_SEMVER.test(version || "")) + fail("Bundle parity requires an exact stable semver version."); + const counts = {}; + for (const surface of ["schemas", "compliance"]) { + counts[surface] = compareSurface( + surface, + path.join(releasedRoot, "dist", surface, version), + path.join(bundleRoot, surface), + ); + } + return counts; +} + +if (require.main === module) { + const [version, releasedRoot, bundleRoot] = process.argv.slice(2); + if (!version || !releasedRoot || !bundleRoot) { + console.error( + "Usage: check-protocol-bundle-parity.cjs ", + ); + process.exitCode = 2; + } else { + try { + const counts = checkBundleParity(version, releasedRoot, bundleRoot); + console.log( + `Signed protocol bundle ${version} covers ${counts.schemas} schema and ${counts.compliance} compliance files.`, + ); + } catch (error) { + console.error( + `::error::${error.message}\nRecovery stopped: the released trees are not the signed bundle's trees, so the release signature does not cover them. See RELEASING.md.`, + ); + process.exitCode = 1; + } + } +} + +module.exports = { checkBundleParity }; diff --git a/scripts/recover-cdn-version-trees.sh b/scripts/recover-cdn-version-trees.sh new file mode 100755 index 00000000000..27c5bf35952 --- /dev/null +++ b/scripts/recover-cdn-version-trees.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +# Recover a released stable version's immutable schema and compliance trees on +# the artifact CDN, or verify that the public CDN serves them byte-exact. +# +# Why this exists: a release whose `schemas//` and +# `compliance//` objects never reached R2 does not 404. The CDN +# worker's pinned-semver fallback resolves the request to the nearest published +# release on the line, so a consumer pinning the missing version silently gets +# an older release's bytes, and the schema discovery pointers keep that line +# pinned one release back (#8318, #8324, #8347). `release.yml` cannot republish +# a historical version (`check-release-state.cjs committed` requires the local, +# current-tested and original committed package versions to be equal), routine +# deployment excludes every semver path, and live bulk backfill is disabled. +# +# Authority model (RELEASING.md "Publication authority and ordering"): +# +# - The caller checks out tag `v$VERSION`, verifies the signed protocol tuple +# with cosign bound to `release.yml`, and runs `check-release-state.cjs +# published` to bind that tag to its GitHub Release and to the committed +# tuple bytes. This script re-checks the tarball checksum, then proves that +# the trees it is about to publish are byte-for-byte the trees inside that +# signed tarball, so the release signature transitively covers every object +# it writes. +# - Every byte comes from `git cat-file blob`, never the working tree and never +# `git archive` (which applies the tag's own `.gitattributes`, so +# `export-subst` would splice commit text into published bytes). +# - This script, the fence and the parity checker are supplied by the caller +# from the current reviewed branch, never from the checked-out tag. +# - Writes are conditional (`If-None-Match: *`). An object that already exists +# must already match byte-for-byte; nothing is ever overwritten or deleted. +# - The current-branch publication fence runs before and after every write. +# +# Usage: recover-cdn-version-trees.sh +# +# Environment: +# VERSION exact stable semver (required) +# FENCE_SCRIPT path to the preserved check-release-state.cjs +# PARITY_SCRIPT path to the preserved check-protocol-bundle-parity.cjs +# BUCKET destination R2 bucket (upload only) +# R2_ENDPOINT or R2_ACCOUNT_ID (upload only) +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY (upload only) +# CDN_BASE_URL public origin to verify (default adcontextprotocol.org) +# CDN_VERIFY_ATTEMPTS verification attempts before failing (default 5) + +set -euo pipefail + +mode="${1:-}" +if [[ "$mode" != upload && "$mode" != verify ]]; then + echo "::error::Usage: recover-cdn-version-trees.sh " >&2 + exit 2 +fi + +die() { + echo "::error::$1" >&2 + echo "Recovery stopped. See RELEASING.md \"Recovering a release's CDN artifacts\"." >&2 + exit 1 +} + +VERSION="${VERSION:-}" +# Stable only: prereleases are never recovered, matching recover-protocol-cdn.yml. +[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || + die "VERSION must be an exact stable semver (for example, 3.0.27)" +version_re="${VERSION//./\\.}" + +FENCE_SCRIPT="${FENCE_SCRIPT:-}" +PARITY_SCRIPT="${PARITY_SCRIPT:-}" +[[ -f "$FENCE_SCRIPT" ]] || die "FENCE_SCRIPT must point at the preserved publication fence" +[[ -f "$PARITY_SCRIPT" ]] || die "PARITY_SCRIPT must point at the preserved bundle parity checker" + +cdn_base_url="${CDN_BASE_URL:-https://adcontextprotocol.org}" +attempts="${CDN_VERIFY_ATTEMPTS:-5}" +[[ "$attempts" =~ ^[1-9][0-9]*$ ]] || die "CDN_VERIFY_ATTEMPTS must be a positive integer" +immutable_cache="public, max-age=31536000, immutable" + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | cut -d ' ' -f1 + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$1" | cut -d ' ' -f1 + else + return 1 + fi +} + +# R2 reports a single-part object's ETag as the hex MD5 of its bytes, and +# `aws s3api put-object` is always single part. That makes the served ETag a +# direct statement about the object in the bucket, which is what verification +# below needs. MD5 is used here only as R2's object identifier, never as a +# security property: the bytes themselves are compared too. +md5_of() { + if command -v md5sum >/dev/null 2>&1; then + md5sum "$1" | cut -d ' ' -f1 + elif command -v md5 >/dev/null 2>&1; then + md5 -q "$1" + else + return 1 + fi +} + +work="" +cleanup() { + if [[ -n "$work" ]]; then rm -rf -- "$work"; fi +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +work="$(mktemp -d "${TMPDIR:-/tmp}/adcp-cdn-recovery.XXXXXX")" +chmod 700 "$work" + +# The fence proves the dispatch is still current for the supported release +# branch. It is the same gate release publication uses, and the only one this +# recovery shares with it. +node "$FENCE_SCRIPT" current /dev/null || + die "missing ${tuple}${suffix} in the checked-out release" +done + +# --------------------------------------------------------------------------- +# Enumerate the released trees from the tag's Git objects. +# --------------------------------------------------------------------------- + +schemas_root="dist/schemas/${VERSION}" +compliance_root="dist/compliance/${VERSION}" + +# A path's extension decides the content type the CDN will serve it with. An +# unmapped extension fails the whole run rather than publishing a tree with a +# silently mistyped hole in it, which is the same class of bug as the missing +# directory this recovers. +content_type_for() { + case "$1" in + *.jsonl) printf 'application/x-ndjson; charset=utf-8' ;; + *.json) printf 'application/json; charset=utf-8' ;; + *.yaml | *.yml) printf 'application/yaml; charset=utf-8' ;; + *.md | *.mdx) printf 'text/markdown; charset=utf-8' ;; + *.txt) printf 'text/plain; charset=utf-8' ;; + *) printf '' ;; + esac +} + +# Validate while the enumeration is still NUL-delimited. A path containing a +# newline would otherwise split into two plausible-looking lines downstream. +tree_list="$work/tree.list" +: > "$tree_list" +while IFS= read -r -d '' row; do + meta="${row%%$'\t'*}" + file="${row#*$'\t'}" + read -r mode_bits object_type _oid <<< "$meta" + # Regular blobs only: a symlink, submodule or subtree entry here would let + # publication read bytes the release never committed at that path. + [[ "$object_type" == blob && ( "$mode_bits" == 100644 || "$mode_bits" == 100755 ) ]] || + die "unsupported Git entry in the released trees: ${file}" + key="${file#dist/}" + # Keys become R2 keys and public URLs. Hold them to an unambiguous charset so + # neither needs escaping and neither can be reinterpreted. + [[ "$key" =~ ^(schemas|compliance)/${version_re}/[A-Za-z0-9][A-Za-z0-9._/-]*$ && "$key" != *..* ]] || + die "unsupported released artifact path: ${file}" + [[ -n "$(content_type_for "$key")" ]] || + die "no published content type for ${key}; recovery would leave the tree incomplete" + printf '%s\n' "$file" >> "$tree_list" +done < <(git ls-tree -r -z HEAD -- "$schemas_root" "$compliance_root") + +LC_ALL=C sort -o "$tree_list" "$tree_list" +[[ -s "$tree_list" ]] || die "release ${VERSION} commits no schema or compliance tree" +[[ "$(LC_ALL=C sort -u "$tree_list" | wc -l)" == "$(wc -l < "$tree_list")" ]] || + die "ambiguous duplicate paths in the released trees" + +for root in "$schemas_root" "$compliance_root"; do + awk -v prefix="${root}/" ' + index($0, prefix) == 1 { tree = 1 } + $0 == prefix "index.json" { manifest = 1 } + END { exit tree ? (manifest ? 0 : 2) : 1 } + ' "$tree_list" && continue + case "$?" in + 1) die "release ${VERSION} commits no ${root}" ;; + *) die "release ${VERSION} commits ${root} without an index.json" ;; + esac +done + +# Materialize each blob exactly as committed. `git archive` would be one +# command, but it runs the tag's own `.gitattributes` filters (`export-subst` +# splices the commit subject into the output, `export-ignore` drops files), so +# its bytes are not the blob's bytes. `git cat-file blob` is attribute-immune, +# which is why backfill-cdn-artifacts.sh uses it too. +released="$work/released" +while IFS= read -r file; do + mkdir -p "${released}/$(dirname "$file")" + git cat-file blob "HEAD:${file}" > "${released}/${file}" +done < "$tree_list" +[[ "$(find "$released" -type f | wc -l)" == "$(wc -l < "$tree_list")" ]] || + die "materialized release trees do not match the committed file set" + +# --------------------------------------------------------------------------- +# Bind the trees to the release signature through the signed tarball. +# --------------------------------------------------------------------------- + +tuple_dir="$work/tuple" +bundle="$work/bundle" +mkdir -p "$tuple_dir" "$bundle" +for suffix in "" .sha256; do + git cat-file blob "HEAD:${tuple}${suffix}" > "${tuple_dir}/${VERSION}.tgz${suffix}" +done +expected_digest="$(cut -d ' ' -f1 < "${tuple_dir}/${VERSION}.tgz.sha256")" +[[ "$(sha256_of "${tuple_dir}/${VERSION}.tgz")" == "$expected_digest" ]] || + die "committed protocol tarball ${VERSION}.tgz fails its committed checksum" +tar -xzf "${tuple_dir}/${VERSION}.tgz" -C "$bundle" --no-same-owner --no-same-permissions +# cosign's --certificate-identity-regexp binds the signing workflow, not the +# version: a renamed copy of another release's signed tuple would verify. The +# bundle's root directory name is the only thing that ties the signature to +# this version, so it is load-bearing, not a sanity check. +[[ -d "${bundle}/adcp-${VERSION}" ]] || + die "signed protocol bundle has no adcp-${VERSION} root directory" +node "$PARITY_SCRIPT" "$VERSION" "$released" "${bundle}/adcp-${VERSION}" "$digest_list" +while IFS= read -r file; do + printf '%s %s %s\n' \ + "$(sha256_of "${released}/${file}")" "$(md5_of "${released}/${file}")" "$file" >> "$digest_list" +done < "$tree_list" + +total="$(wc -l < "$tree_list" | tr -d ' ')" + +if [[ "$mode" == upload ]]; then + BUCKET="${BUCKET:-}" + endpoint="${R2_ENDPOINT:-}" + if [[ -z "$endpoint" && -n "${R2_ACCOUNT_ID:-}" ]]; then + endpoint="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" + fi + [[ -n "$BUCKET" ]] || die "Missing destination bucket" + [[ "$BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,62}$ ]] || + die "BUCKET must be a bucket name, not a URL: ${BUCKET}" + [[ -n "$endpoint" ]] || die "Missing R2 endpoint" + [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" ]] || + die "Missing R2 publication credentials" + export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-auto}" + + existing_body="$work/existing" + existing_type="$work/existing-type" + head_error="$work/head-error" + created=0 + verified=0 + + while read -r digest _md5 file <&3; do + key="${file#dist/}" + body_path="${released}/${file}" + content_type="$(content_type_for "$key")" + [[ "$(sha256_of "$body_path")" == "$digest" ]] || + die "publication body for ${key} changed after the signature proof" + + node "$FENCE_SCRIPT" current "$existing_type" 2>"$head_error"; then + aws s3 cp "s3://${BUCKET}/${key}" "$existing_body" \ + --endpoint-url "$endpoint" --region "$AWS_DEFAULT_REGION" \ + --no-cli-pager --no-progress --only-show-errors &2 + die "Could not safely determine whether R2 object ${key} exists" + fi + node "$FENCE_SCRIPT" current /dev/null + created=$((created + 1)) + fi + node "$FENCE_SCRIPT" current "$expected" +while read -r _digest md5 file; do + printf '%s 200 "%s" %s\n' "${file#dist/}" "$md5" "$immutable_cache" >> "$expected" +done < "$digest_list" +LC_ALL=C sort -o "$expected" "$expected" + +verify_once() { + local attempt="$1" + local bust file key line url rest + bust="${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-0}-${attempt}-$(date +%s)" + rm -rf "$download_dir" + mkdir -p "$download_dir" + : > "$config" + while IFS= read -r file; do + key="${file#dist/}" + mkdir -p "${download_dir}/$(dirname "$key")" + printf 'fail\nlocation\nurl = "%s"\noutput = "%s"\n' \ + "${cdn_base_url}/${key}?cache_bust=${bust}" "${download_dir}/${key}" >> "$config" + done < "$tree_list" + + # Bound every transfer. Without this a server that accepts the connection and + # never answers hangs the job forever and the retry loop below never runs. + if ! curl --silent --show-error --fail-early \ + --parallel --parallel-immediate --parallel-max 16 \ + --connect-timeout 20 --max-time 300 --speed-limit 1024 --speed-time 60 \ + --write-out '%{url_effective} %{http_code} %header{etag} %header{cache-control}\n' \ + --config "$config" > "$report"; then + return 1 + fi + + # --parallel completes out of order, so correlate on the URL rather than + # comparing line by line. + : > "$observed" + while IFS= read -r line; do + url="${line%% *}" + rest="${line#* }" + key="${url#"${cdn_base_url}/"}" + key="${key%%\?*}" + printf '%s %s\n' "$key" "$rest" >> "$observed" + done < "$report" + LC_ALL=C sort -o "$observed" "$observed" + + if ! diff -q "$expected" "$observed" >/dev/null; then + echo "CDN responses do not match the recovered ${VERSION} objects:" >&2 + diff "$expected" "$observed" | head -n 10 >&2 + echo "A missing or mismatched ETag means the bytes did not come from the bucket." >&2 + return 1 + fi + + while IFS= read -r file; do + key="${file#dist/}" + if ! cmp -s "${released}/${file}" "${download_dir}/${key}"; then + echo "CDN object ${key} differs from the released bytes." >&2 + return 1 + fi + done < "$tree_list" + return 0 +} + +# The worker caches its published-version list for VERSION_CACHE_TTL_MS = 60s +# (workers/artifact-cdn/src/index.js), so a freshly created prefix can take +# that long to win an exact pinned hit. The cumulative backoff below reaches +# 60s at attempt 4; do not shrink it below that. +for attempt in $(seq 1 "$attempts"); do + if verify_once "$attempt"; then + echo "Public CDN serves all ${total} ${VERSION} schema and compliance objects byte-exact (attempt ${attempt})." + exit 0 + fi + if [[ "$attempt" != "$attempts" ]]; then + echo "CDN verification failed on attempt ${attempt}; retrying." + sleep $((attempt * 10)) + fi +done + +die "public CDN does not serve the recovered ${VERSION} trees byte-exact" diff --git a/tests/recover-cdn-version-trees.test.cjs b/tests/recover-cdn-version-trees.test.cjs new file mode 100644 index 00000000000..0e5022f398d --- /dev/null +++ b/tests/recover-cdn-version-trees.test.cjs @@ -0,0 +1,784 @@ +// Recovery of a released version's immutable schema and compliance trees on +// the artifact CDN (#8347). The gap these cover: a missing version directory +// does not 404, the worker silently serves a neighbouring release, so a +// recovery that publishes the wrong bytes, overwrites an object, or merely +// uploads without checking what the CDN then serves is indistinguishable from +// the bug it is meant to fix. +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const http = require("node:http"); +const os = require("node:os"); +const path = require("node:path"); +const { execFileSync, spawn, spawnSync } = require("node:child_process"); +const YAML = require("yaml"); + +const root = path.resolve(__dirname, ".."); +const realGit = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); +const workflowPath = path.join( + root, + ".github/workflows/recover-protocol-cdn.yml", +); +const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")); +const recoverySteps = workflow.jobs.recover.steps; +const stepNamed = (name) => recoverySteps.find((s) => s.name === name); + +const VERSION = "3.0.27"; +const IMMUTABLE = "public, max-age=31536000, immutable"; +const REVALIDATE = "public, no-cache, must-revalidate"; + +// One representative file per published content type, plus a nested path. +const RELEASED_TREE = { + "schemas/index.json": '{\n "adcp_version": "3.0.27"\n}\n', + "schemas/manifest.json": + '{\n "published_version": "3.0.27",\n "file_count": 3,\n "generated_at": "2026-09-30T14:37:10.432Z"\n}\n', + "schemas/core/product.json": '{\n "title": "Product"\n}\n', + "compliance/index.json": + '{\n "adcp_version": "3.0.27",\n "generated_at": "2026-09-30T14:37:12.795Z"\n}\n', + "compliance/universal/pagination.yaml": "id: pagination\n", + "compliance/README.md": "# Compliance\n", + "compliance/test-kits/rows.jsonl": '{"row":1}\n', + "compliance/notes.txt": "notes\n", +}; + +function fixture(t, options = {}) { + const { + version = VERSION, + tree = RELEASED_TREE, + // Divergence injected into the staged bundle only, so the committed trees + // stop matching the signed tarball. + bundleOverrides = {}, + packageVersion = version, + } = options; + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "recover-cdn-trees-")); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + + const write = (file, value) => { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), value); + }; + const git = (...args) => + execFileSync(realGit, args, { + cwd: dir, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + + write("package.json", JSON.stringify({ version: packageVersion })); + for (const [relative, body] of Object.entries(tree)) { + const [surface, ...rest] = relative.split("/"); + write(`dist/${surface}/${version}/${rest.join("/")}`, body); + } + + // Stage and build the signed tarball exactly the way build-protocol-tarball + // does: the bundle's schemas/ and compliance/ are copies of the released + // trees, with only the build timestamps re-pinned. + const bundleRoot = `adcp-${version}`; + for (const [relative, body] of Object.entries(tree)) { + write(`stage/${bundleRoot}/${relative}`, body); + } + // Mirror build-protocol-tarball.cjs: the staged copies carry a later build + // timestamp than the committed trees. + for (const [relative, from] of [ + ["schemas/manifest.json", "2026-09-30T14:37:10.432Z"], + ["compliance/index.json", "2026-09-30T14:37:12.795Z"], + ]) + if (tree[relative]) + write( + `stage/${bundleRoot}/${relative}`, + tree[relative].replace(from, "2026-09-30T14:37:20.000Z"), + ); + write(`stage/${bundleRoot}/README.md`, "# AdCP Protocol Bundle\n"); + for (const [relative, body] of Object.entries(bundleOverrides)) { + if (body === null) fs.rmSync(path.join(dir, "stage", bundleRoot, relative)); + else write(`stage/${bundleRoot}/${relative}`, body); + } + + fs.mkdirSync(path.join(dir, "dist/protocol"), { recursive: true }); + const tarball = path.join(dir, `dist/protocol/${version}.tgz`); + execFileSync("tar", ["-czf", tarball, "-C", path.join(dir, "stage"), bundleRoot]); + const digest = crypto + .createHash("sha256") + .update(fs.readFileSync(tarball)) + .digest("hex"); + write(`dist/protocol/${version}.tgz.sha256`, `${digest} ${version}.tgz\n`); + write(`dist/protocol/${version}.tgz.sig`, "signature\n"); + write(`dist/protocol/${version}.tgz.crt`, "certificate\n"); + fs.rmSync(path.join(dir, "stage"), { recursive: true, force: true }); + + git("init", "-q"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.invalid"); + git("add", "."); + git("commit", "-qm", "Release"); + const head = git("rev-parse", "HEAD"); + write("remote", head); + write("calls", ""); + fs.mkdirSync(path.join(dir, "bucket"), { recursive: true }); + + // Recovery runs the preserved copies from the current branch, never the + // tag's. Mirror that: the scripts live outside the checkout. + for (const script of [ + "check-release-state.cjs", + "check-protocol-bundle-parity.cjs", + "recover-cdn-version-trees.sh", + ]) + write(`temp/${script}`, fs.readFileSync(path.join(root, "scripts", script))); + + const executable = (name, code) => { + write(`bin/${name}`, `#!/usr/bin/env node\n${code}`); + fs.chmodSync(path.join(dir, "bin", name), 0o755); + }; + const common = `const fs=require('node:fs'),path=require('node:path');const args=process.argv.slice(2);const read=f=>fs.readFileSync(f,'utf8');const log=s=>fs.appendFileSync('calls',s+'\\n');`; + executable( + "git", + common + + ` + if(args[0]==='ls-remote') { + if(process.env.REMOTE_ERROR) process.exit(1); + const advance=Number(process.env.ADVANCE_AFTER||'0'); + const count=(fs.existsSync('fence-count')?Number(read('fence-count')):0)+1; + fs.writeFileSync('fence-count',String(count)); + // Move the branch under the run, before this very fence answers. + if(advance&&count===advance) fs.writeFileSync('remote','f'.repeat(40)); + const ref=args.find(a=>a.startsWith('refs/heads/')); + console.log(read('remote')+'\\t'+ref); process.exit(0); + } + const r=require('node:child_process').spawnSync(${JSON.stringify(realGit)},args,{stdio:'inherit'});process.exit(r.status??1); + `, + ); + executable( + "aws", + common + + ` + const value=k=>args[args.indexOf(k)+1]; + const key=args.includes('--key')?value('--key'):'';const object=path.join('bucket',key); + const metaPath=object+'.meta'; + if(args[1]==='head-object') { + if(process.env.HEAD_ERROR) { console.error(process.env.HEAD_ERROR);process.exit(1); } + if(!fs.existsSync(object)) {console.error('An error occurred (404) when calling the HeadObject operation: Not Found');process.exit(1);} + const meta=fs.existsSync(metaPath)?JSON.parse(read(metaPath)):{ContentType:process.env.EXISTING_CONTENT_TYPE||'application/json; charset=utf-8'}; + if(args.includes('--query')&&value('--query')==='ContentType') console.log(meta.ContentType); + process.exit(0); + } + if(args[1]==='put-object') { + if(value('--if-none-match')!=='*') throw Error('unconditional write'); + if(fs.existsSync(object)) {console.error('PreconditionFailed');process.exit(1);} + fs.mkdirSync(path.dirname(object),{recursive:true});fs.copyFileSync(value('--body'),object); + fs.writeFileSync(metaPath,JSON.stringify({ContentType:value('--content-type'),CacheControl:value('--cache-control')})); + log('put '+key+' | '+value('--content-type')+' | '+value('--cache-control')); + process.exit(0); + } + if(args[0]==='s3'&&args[1]==='cp'&&args[2].startsWith('s3://')) { + fs.copyFileSync(path.join('bucket',args[2].split('/').slice(3).join('/')),args[3]); + process.exit(0); + } + throw Error('unexpected aws '+args.join(' ')); + `, + ); + + const env = { + PATH: `${dir}/bin:${process.env.PATH}`, + HOME: dir, + TESTED_SHA: head, + GITHUB_SHA: head, + PUBLICATION_BRANCH: "main", + GITHUB_REF_NAME: "main", + VERSION: version, + FENCE_SCRIPT: path.join(dir, "temp/check-release-state.cjs"), + PARITY_SCRIPT: path.join(dir, "temp/check-protocol-bundle-parity.cjs"), + BUCKET: "test", + R2_ENDPOINT: "https://r2.invalid", + AWS_ACCESS_KEY_ID: "test", + AWS_SECRET_ACCESS_KEY: "test", + CDN_VERIFY_ATTEMPTS: "1", + }; + + const argv = (mode) => [ + "--noprofile", + "--norc", + path.join(dir, "temp/recover-cdn-version-trees.sh"), + mode, + ]; + const run = (mode, extra = {}) => + spawnSync("bash", argv(mode), { + cwd: dir, + env: { ...env, ...extra }, + encoding: "utf8", + }); + // The verification tests answer the script's own HTTP requests from this + // process, so they cannot block its event loop on spawnSync. + const runAsync = (mode, extra = {}) => + new Promise((resolve, reject) => { + const child = spawn("bash", argv(mode), { + cwd: dir, + env: { ...env, ...extra }, + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (status) => resolve({ status, stdout, stderr })); + }); + + return { + dir, + head, + version, + write, + git, + run, + runAsync, + bucketKeys: () => + fs.existsSync(path.join(dir, "bucket")) + ? listFiles(path.join(dir, "bucket")) + .filter((key) => !key.endsWith(".meta")) + .sort() + : [], + calls: () => + fs + .readFileSync(path.join(dir, "calls"), "utf8") + .split("\n") + .filter(Boolean), + }; +} + +function listFiles(base, prefix = "") { + const out = []; + for (const entry of fs.readdirSync(base, { withFileTypes: true })) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isDirectory()) + out.push(...listFiles(path.join(base, entry.name), relative)); + else out.push(relative); + } + return out; +} + +const expectedKeys = Object.keys(RELEASED_TREE) + .map((relative) => { + const [surface, ...rest] = relative.split("/"); + return `${surface}/${VERSION}/${rest.join("/")}`; + }) + .sort(); + +// --- upload ----------------------------------------------------------------- + +test("recovery publishes every missing version object conditionally and immutably", (t) => { + const f = fixture(t); + const result = f.run("upload"); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(f.bucketKeys(), expectedKeys); + assert.match(result.stdout, /created 8 missing object\(s\), verified 0/); + + const published = new Map( + f.calls().map((line) => { + const [key, contentType, cacheControl] = line + .replace(/^put /, "") + .split(" | "); + return [key, { contentType, cacheControl }]; + }), + ); + assert.deepEqual([...published.keys()].sort(), expectedKeys); + for (const entry of published.values()) + assert.equal(entry.cacheControl, IMMUTABLE); + assert.equal( + published.get(`schemas/${VERSION}/index.json`).contentType, + "application/json; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/universal/pagination.yaml`).contentType, + "application/yaml; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/README.md`).contentType, + "text/markdown; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/test-kits/rows.jsonl`).contentType, + "application/x-ndjson; charset=utf-8", + ); + assert.equal( + published.get(`compliance/${VERSION}/notes.txt`).contentType, + "text/plain; charset=utf-8", + ); +}); + +test("an already published object is verified, never rewritten", (t) => { + const f = fixture(t); + f.write( + `bucket/schemas/${VERSION}/index.json`, + RELEASED_TREE["schemas/index.json"], + ); + const result = f.run("upload"); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /created 7 missing object\(s\), verified 1/); + assert.ok( + !f.calls().some((line) => line.includes(`schemas/${VERSION}/index.json`)), + ); +}); + +test("a published object with different bytes stops recovery instead of overwriting", (t) => { + const f = fixture(t); + f.write(`bucket/compliance/${VERSION}/index.json`, '{"adcp_version":"3.0.26"}'); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /exists with different bytes; refusing overwrite/); + assert.ok( + !f.calls().some((line) => line.includes(`compliance/${VERSION}/index.json`)), + ); +}); + +test("an inconclusive existence check never reads as absent", (t) => { + const f = fixture(t); + const result = f.run("upload", { + HEAD_ERROR: "An error occurred (403) when calling the HeadObject operation: Forbidden", + }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Could not safely determine whether R2 object/); + assert.deepEqual(f.calls(), []); +}); + +for (const [label, overrides] of [ + ["a changed file", { "schemas/core/product.json": '{"title":"Tampered"}\n' }], + ["an extra file", { "schemas/core/extra.json": "{}\n" }], + ["a missing file", { "compliance/notes.txt": null }], +]) { + test(`released trees with ${label} relative to the signed bundle are never published`, (t) => { + const f = fixture(t, { bundleOverrides: overrides }); + const run = f.run("upload"); + assert.notEqual(run.status, 0); + assert.match( + run.stderr, + /signed protocol bundle|does not match the released tree/, + ); + assert.deepEqual(f.bucketKeys(), []); + assert.deepEqual(f.calls(), []); + }); +} + +test("only the bundle build's own timestamp pin may differ, and only in its own file", (t) => { + // A rewritten timestamp in a file the bundle build never pins is a content + // difference, not an exemption. + const f = fixture(t, { + bundleOverrides: { + "compliance/universal/pagination.yaml": "id: pagination\ngenerated_at: x\n", + }, + }); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /differs from the signed protocol bundle/); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("recovery refuses a prerelease, a mismatched checkout, and a stale branch", (t) => { + const prerelease = fixture(t); + const rc = prerelease.run("upload", { VERSION: "3.2.0-rc.3" }); + assert.notEqual(rc.status, 0); + assert.match(rc.stderr, /exact stable semver/); + assert.deepEqual(prerelease.calls(), []); + + const mismatched = fixture(t, { packageVersion: "3.0.26" }); + const wrongCheckout = mismatched.run("upload"); + assert.notEqual(wrongCheckout.status, 0); + assert.match(wrongCheckout.stderr, /checkout holds package version 3\.0\.26/); + assert.deepEqual(mismatched.calls(), []); + + const stale = fixture(t); + stale.write("remote", "f".repeat(40)); + const moved = stale.run("upload"); + assert.notEqual(moved.status, 0); + assert.match(moved.stderr, /Refusing stale tested SHA/); + assert.deepEqual(stale.calls(), []); +}); + +test("the branch fence is rechecked between objects, so a mid-run branch move stops publication", (t) => { + const f = fixture(t); + // Fence calls: one preflight, then three per published object. Moving the + // branch on the fifth lands inside the second object, after the first was + // already written. + const result = f.run("upload", { ADVANCE_AFTER: "5" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Refusing stale tested SHA/); + assert.equal(f.calls().length, 1); + assert.equal(f.bucketKeys().length, 1); +}); + +test("a file whose extension has no published content type stops recovery before any write", (t) => { + const f = fixture(t, { + tree: { ...RELEASED_TREE, "schemas/bundle.tar": "binary\n" }, + bundleOverrides: {}, + }); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /no published content type/); + assert.deepEqual(f.bucketKeys(), []); + assert.deepEqual(f.calls(), []); +}); + +test("a symlinked entry in the released tree is refused", (t) => { + const f = fixture(t); + f.git("rm", "-q", "--cached", `dist/schemas/${VERSION}/core/product.json`); + fs.rmSync(path.join(f.dir, `dist/schemas/${VERSION}/core/product.json`)); + fs.symlinkSync( + "../../../../package.json", + path.join(f.dir, `dist/schemas/${VERSION}/core/product.json`), + ); + f.git("add", "-A"); + f.git("commit", "-qm", "Symlinked schema"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /unsupported Git entry in the released trees/); + assert.deepEqual(f.calls(), []); +}); + +test("published bytes are the committed blobs, not a .gitattributes rewrite", (t) => { + // `git archive` applies the tag's own export-subst filter, which splices the + // commit subject into the bytes it emits. Recovery must read blobs instead. + const f = fixture(t, { + tree: { ...RELEASED_TREE, "schemas/core/product.json": '{"s":"$Format:%s$"}\n' }, + }); + f.write(".gitattributes", `dist/schemas/${VERSION}/core/product.json export-subst\n`); + f.git("add", "-A"); + f.git("commit", "-qm", "SUBSTITUTED-COMMIT-SUBJECT"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.equal(result.status, 0, result.stderr); + const published = fs.readFileSync( + path.join(f.dir, `bucket/schemas/${VERSION}/core/product.json`), + "utf8", + ); + assert.equal(published, '{"s":"$Format:%s$"}\n'); + assert.doesNotMatch(published, /SUBSTITUTED-COMMIT-SUBJECT/); +}); + +test("a tarball that fails its committed checksum stops recovery", (t) => { + const f = fixture(t); + f.write(`dist/protocol/${VERSION}.tgz.sha256`, `${"0".repeat(64)} ${VERSION}.tgz\n`); + f.git("add", "-A"); + f.git("commit", "-qm", "Broken checksum"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /fails its committed checksum/); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("a bundle for another version cannot stand in for this one", (t) => { + // cosign binds the signing workflow, not the version, so the bundle's root + // directory name is the only thing tying the signature to this release. + const f = fixture(t); + const stage = path.join(f.dir, "restage"); + fs.mkdirSync(path.join(stage, "adcp-3.0.26"), { recursive: true }); + fs.writeFileSync(path.join(stage, "adcp-3.0.26", "README.md"), "other\n"); + execFileSync("tar", [ + "-czf", path.join(f.dir, `dist/protocol/${VERSION}.tgz`), + "-C", stage, "adcp-3.0.26", + ]); + const digest = crypto + .createHash("sha256") + .update(fs.readFileSync(path.join(f.dir, `dist/protocol/${VERSION}.tgz`))) + .digest("hex"); + f.write(`dist/protocol/${VERSION}.tgz.sha256`, `${digest} ${VERSION}.tgz\n`); + f.git("add", "-A"); + f.git("commit", "-qm", "Swapped bundle"); + f.write("remote", f.git("rev-parse", "HEAD")); + const result = f.run("upload", { TESTED_SHA: f.git("rev-parse", "HEAD") }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, new RegExp(`no adcp-${VERSION} root directory`)); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("a released tree without its index.json is refused", (t) => { + const tree = { ...RELEASED_TREE }; + delete tree["compliance/index.json"]; + const f = fixture(t, { tree }); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /without an index\.json/); + assert.deepEqual(f.bucketKeys(), []); +}); + +test("an existing object with the right bytes but the wrong content type is refused", (t) => { + // No-overwrite means a mistyped object cannot be repaired here, and the CDN + // serves the stored type, so this must stop rather than report "verified". + const f = fixture(t); + f.write( + `bucket/schemas/${VERSION}/index.json`, + RELEASED_TREE["schemas/index.json"], + ); + f.write( + `bucket/schemas/${VERSION}/index.json.meta`, + JSON.stringify({ ContentType: "text/plain; charset=utf-8" }), + ); + const result = f.run("upload"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /exists with content type text\/plain/); +}); + +test("upload refuses an unusable destination or missing credentials", (t) => { + const f = fixture(t); + for (const [extra, pattern] of [ + [{ BUCKET: "s3://adcp-artifacts" }, /must be a bucket name, not a URL/], + [{ BUCKET: "" }, /Missing destination bucket/], + [{ R2_ENDPOINT: "", R2_ACCOUNT_ID: "" }, /Missing R2 endpoint/], + [{ AWS_SECRET_ACCESS_KEY: "" }, /Missing R2 publication credentials/], + ]) { + const result = f.run("upload", extra); + assert.notEqual(result.status, 0); + assert.match(result.stderr, pattern); + } + assert.deepEqual(f.bucketKeys(), []); + assert.deepEqual(f.calls(), []); +}); + +test("the script refuses an unknown mode and an unusable retry budget", (t) => { + const f = fixture(t); + const mode = f.run("publish"); + assert.equal(mode.status, 2); + assert.match(mode.stderr, /Usage: recover-cdn-version-trees\.sh/); + const budget = f.run("verify", { CDN_VERIFY_ATTEMPTS: "0" }); + assert.notEqual(budget.status, 0); + assert.match(budget.stderr, /CDN_VERIFY_ATTEMPTS must be a positive integer/); +}); + +// --- verify ----------------------------------------------------------------- + +function serveCdn(t, handler) { + const server = http.createServer(handler); + server.listen(0, "127.0.0.1"); + // curl keeps its parallel connections alive, so close() alone would leave + // the test process waiting on them after the assertions are done. + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + return new Promise((resolve) => + server.once("listening", () => + resolve(`http://127.0.0.1:${server.address().port}`), + ), + ); +} + +// An R2 hit through the worker: the released bytes, the immutable policy, and +// R2's single-part ETag, which is the body's hex MD5. +function exactServer(f, options = {}) { + const { + cacheControl = IMMUTABLE, + corrupt = null, + // Models the worker proxying an R2 miss to the Fly origin, which serves + // the same committed tree under the same immutable policy but cannot + // produce R2's ETag. + etagFor = (body) => `"${crypto.createHash("md5").update(body).digest("hex")}"`, + } = options; + return (request, response) => { + const key = decodeURIComponent(request.url.split("?")[0]).slice(1); + const file = path.join(f.dir, "dist", key); + if (!fs.existsSync(file)) { + response.writeHead(404).end(); + return; + } + const body = + key === corrupt ? Buffer.from("tampered\n") : fs.readFileSync(file); + const headers = { "cache-control": cacheControl }; + const etag = etagFor(body, key); + if (etag) headers.etag = etag; + response.writeHead(200, headers); + response.end(request.method === "HEAD" ? undefined : body); + }; +} + +test("verification passes only when the CDN serves the released bytes", async (t) => { + const f = fixture(t); + const base = await serveCdn(t, exactServer(f)); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.equal(result.status, 0, result.stderr + result.stdout); + assert.match(result.stdout, /serves all 8 3\.0\.27 schema and compliance objects byte-exact/); +}); + +test("verification fails when any served object differs", async (t) => { + const f = fixture(t); + const corrupt = `schemas/${VERSION}/core/product.json`; + const base = await serveCdn( + t, + // Serve tampered bytes under the ETag the real object would have, so the + // header checks pass and the byte comparison is the thing that must catch + // it. + exactServer(f, { + corrupt, + etagFor: (body, key) => + `"${crypto + .createHash("md5") + .update( + key === corrupt + ? fs.readFileSync(path.join(f.dir, "dist", key)) + : body, + ) + .digest("hex")}"`, + }), + ); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /differs from the released bytes/); +}); + +test("verification fails when the version is still served under a fallback policy", async (t) => { + // Before recovery the worker resolves the pin to a neighbouring release and + // stamps the revalidate policy. Byte comparison alone can pass when the two + // releases happen to agree, so the policy must be checked too. + const f = fixture(t); + const base = await serveCdn(t, exactServer(f, { cacheControl: REVALIDATE })); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /do not match the recovered 3\.0\.27 objects/); +}); + +test("verification fails when the bytes come from the origin rather than the bucket", async (t) => { + // The case the policy check alone cannot catch: on an R2 miss the worker + // proxies to Fly, which serves the same committed tree AND stamps the same + // immutable policy for an exact pinned hit. Only R2 produces the body's MD5 + // as the ETag, so that is what proves the object is really in the bucket. + const f = fixture(t); + const base = await serveCdn( + t, + exactServer(f, { etagFor: () => 'W/"1a2-18f3c4d5e6f"' }), + ); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not come from the bucket/); +}); + +test("verification fails when a served ETag is absent", async (t) => { + const f = fixture(t); + const base = await serveCdn(t, exactServer(f, { etagFor: () => null })); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /do not match the recovered 3\.0\.27 objects/); +}); + +test("verification retries while the CDN catches up, then passes", async (t) => { + // The worker caches its published-version list for 60s, so a freshly created + // prefix can need a retry before it wins an exact pinned hit. + const f = fixture(t); + // Each attempt uses a distinct cache-bust token, which is the only + // order-independent way to tell the rounds apart under parallel requests. + const busts = new Set(); + const base = await serveCdn(t, (request, response) => { + const bust = new URL(request.url, "http://cdn.invalid").searchParams.get( + "cache_bust", + ); + busts.add(bust); + const isFirstRound = [...busts][0] === bust; + exactServer(f, isFirstRound ? { cacheControl: REVALIDATE } : {})( + request, + response, + ); + }); + const result = await f.runAsync("verify", { + CDN_BASE_URL: base, + CDN_VERIFY_ATTEMPTS: "2", + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /failed on attempt 1; retrying/); + assert.match(result.stdout, /byte-exact \(attempt 2\)/); +}); + +test("verification fails when an object is missing from the CDN", async (t) => { + const f = fixture(t); + const base = await serveCdn(t, (request, response) => { + if (request.url.includes("notes.txt")) { + response.writeHead(404).end(); + return; + } + exactServer(f)(request, response); + }); + const result = await f.runAsync("verify", { CDN_BASE_URL: base }); + assert.notEqual(result.status, 0); + assert.doesNotMatch(result.stdout, /byte-exact/); +}); + +// --- workflow wiring -------------------------------------------------------- + +test("the dispatch keeps its existing protocol-only default", () => { + const inputs = workflow.on.workflow_dispatch.inputs; + assert.equal(inputs.surfaces.type, "choice"); + assert.equal(inputs.surfaces.default, "protocol"); + assert.deepEqual(inputs.surfaces.options, ["protocol", "all"]); + assert.equal(workflow.permissions.contents, "read"); + assert.equal(workflow.env.PUBLICATION_BRANCH, "main"); + // One recovery per version, whatever the surfaces, so two dispatches cannot + // interleave reads and conditional writes on the same objects. + assert.ok(!workflow.concurrency.group.includes("surfaces")); + assert.equal(workflow.concurrency["cancel-in-progress"], false); +}); + +test("tree recovery runs the current branch's scripts, not the recovered tag's", () => { + const preserve = stepNamed("Preserve current workflow publication fence"); + const checkout = recoverySteps.findIndex( + (s) => s.name === "Validate release input and checkout tag", + ); + assert.ok( + recoverySteps.indexOf(preserve) < checkout, + "scripts must be preserved before the tag checkout replaces them", + ); + for (const script of [ + "check-release-state.cjs", + "check-protocol-bundle-parity.cjs", + "recover-cdn-version-trees.sh", + ]) { + assert.ok( + preserve.run.includes(`cp scripts/${script} "$RUNNER_TEMP/${script}"`), + `${script} must be preserved from the current branch`, + ); + assert.ok( + fs.existsSync(path.join(root, "scripts", script)), + `${script} must exist on the current branch`, + ); + } + + for (const [name, mode] of [ + ["Upload only missing immutable schema and compliance objects", "upload"], + ["Verify recovered public CDN trees", "verify"], + ]) { + const step = stepNamed(name); + assert.ok(step, `${name} must exist`); + assert.equal(step.if, "inputs.surfaces == 'all'"); + assert.equal( + step.run.trim(), + `bash "$RUNNER_TEMP/recover-cdn-version-trees.sh" ${mode}`, + ); + assert.equal(step.env.VERSION, "${{ inputs.version }}"); + assert.equal(step.env.FENCE_SCRIPT, "${{ runner.temp }}/check-release-state.cjs"); + assert.equal( + step.env.PARITY_SCRIPT, + "${{ runner.temp }}/check-protocol-bundle-parity.cjs", + ); + // Retry budget and CDN origin stay at the script defaults. + assert.ok(!("CDN_VERIFY_ATTEMPTS" in step.env)); + assert.ok(!("CDN_BASE_URL" in step.env)); + } + + const verify = stepNamed("Verify recovered public CDN trees"); + for (const secret of ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "BUCKET"]) + assert.ok( + !(secret in verify.env), + `${secret} must not be exposed to the read-only verification step`, + ); +}); + +test("every recovery step still fails closed on the current-branch fence", () => { + const protocolUpload = stepNamed("Upload only missing immutable R2 objects"); + assert.match(protocolUpload.run, /check-release-state\.cjs" current/); + const script = fs.readFileSync( + path.join(root, "scripts/recover-cdn-version-trees.sh"), + "utf8", + ); + assert.match(script, /node "\$FENCE_SCRIPT" current/); + assert.match(script, /--if-none-match '\*'/); + assert.doesNotMatch(script, /s3 rm|--delete|rb /); +}); From c6fde477a3b7769bb3c2efaa6b5a633c7da212cd Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 19 Oct 2026 23:21:07 +0000 Subject: [PATCH 2/4] test(release): keep the CDN fixture server inside its served tree CodeQL flagged the recovery verification fixture: it joined a request path onto the fixture directory and read it, so a traversing request would have escaped the served tree. Resolve the path and require containment, which is also what the real origin does. Co-Authored-By: Claude Opus 5 (1M context) --- tests/recover-cdn-version-trees.test.cjs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/recover-cdn-version-trees.test.cjs b/tests/recover-cdn-version-trees.test.cjs index 0e5022f398d..c40cff01217 100644 --- a/tests/recover-cdn-version-trees.test.cjs +++ b/tests/recover-cdn-version-trees.test.cjs @@ -576,10 +576,23 @@ function exactServer(f, options = {}) { // produce R2's ETag. etagFor = (body) => `"${crypto.createHash("md5").update(body).digest("hex")}"`, } = options; + const servedRoot = path.resolve(f.dir, "dist"); return (request, response) => { - const key = decodeURIComponent(request.url.split("?")[0]).slice(1); - const file = path.join(f.dir, "dist", key); - if (!fs.existsSync(file)) { + const key = decodeURIComponent(request.url.split("?")[0]).replace( + /^\/+/, + "", + ); + // Serve only from inside the fixture's released tree, so a traversing + // request gets a 404 rather than reading the rest of the sandbox. + const file = path.resolve(servedRoot, key); + if ( + file !== servedRoot && + !file.startsWith(servedRoot + path.sep) + ) { + response.writeHead(404).end(); + return; + } + if (!fs.existsSync(file) || !fs.statSync(file).isFile()) { response.writeHead(404).end(); return; } From 1ce56db7b0126c5f2d0116d04e3f35eabe1b2ead Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 19 Oct 2026 23:38:43 +0000 Subject: [PATCH 3/4] test(release): serve the CDN fixture from an enumerated file set The containment check still let a request value reach the filesystem, and paired existsSync/readFileSync added a race. Enumerate the served tree once and let a request only select an entry, so an unknown or traversing path has no entry and nothing derives a path from the request. Co-Authored-By: Claude Opus 5 (1M context) --- tests/recover-cdn-version-trees.test.cjs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/recover-cdn-version-trees.test.cjs b/tests/recover-cdn-version-trees.test.cjs index c40cff01217..bb0738e2706 100644 --- a/tests/recover-cdn-version-trees.test.cjs +++ b/tests/recover-cdn-version-trees.test.cjs @@ -576,23 +576,23 @@ function exactServer(f, options = {}) { // produce R2's ETag. etagFor = (body) => `"${crypto.createHash("md5").update(body).digest("hex")}"`, } = options; + // The served set is enumerated once, up front. A request only ever selects + // an entry from it, so no request value reaches the filesystem and an + // unknown or traversing path simply has no entry. const servedRoot = path.resolve(f.dir, "dist"); + const served = new Map( + listFiles(servedRoot).map((relative) => [ + relative, + path.join(servedRoot, relative), + ]), + ); return (request, response) => { const key = decodeURIComponent(request.url.split("?")[0]).replace( /^\/+/, "", ); - // Serve only from inside the fixture's released tree, so a traversing - // request gets a 404 rather than reading the rest of the sandbox. - const file = path.resolve(servedRoot, key); - if ( - file !== servedRoot && - !file.startsWith(servedRoot + path.sep) - ) { - response.writeHead(404).end(); - return; - } - if (!fs.existsSync(file) || !fs.statSync(file).isFile()) { + const file = served.get(key); + if (file === undefined) { response.writeHead(404).end(); return; } From 6a4cf179174ba621bdbdb3f53987008d8abed346 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Tue, 20 Oct 2026 00:16:58 +0000 Subject: [PATCH 4/4] fix(release): bound the recovery upload's S3 calls Ladon's review noted the upload loop relies on AWS CLI default timeouts while the verification fetches are explicitly bounded. The loop makes three calls per object across hundreds of objects, so one stalled socket would hang publication instead of failing it. Bound them the same way. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/recover-cdn-version-trees.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/recover-cdn-version-trees.sh b/scripts/recover-cdn-version-trees.sh index 27c5bf35952..030a4c99422 100755 --- a/scripts/recover-cdn-version-trees.sh +++ b/scripts/recover-cdn-version-trees.sh @@ -244,6 +244,10 @@ if [[ "$mode" == upload ]]; then [[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" ]] || die "Missing R2 publication credentials" export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-auto}" + # Bound every S3 call the way the verification fetches are bounded. The run + # makes three calls per object over a long loop; a single stalled socket on + # CLI defaults would hang the publication rather than fail it. + aws_timeouts=(--cli-connect-timeout 20 --cli-read-timeout 120) existing_body="$work/existing" existing_type="$work/existing-type" @@ -262,11 +266,11 @@ if [[ "$mode" == upload ]]; then rm -f "$existing_body" "$existing_type" "$head_error" if aws s3api head-object --bucket "$BUCKET" --key "$key" \ --endpoint-url "$endpoint" --region "$AWS_DEFAULT_REGION" \ - --no-cli-pager --query ContentType --output text \ + --no-cli-pager --query ContentType --output text "${aws_timeouts[@]}" \ "$existing_type" 2>"$head_error"; then aws s3 cp "s3://${BUCKET}/${key}" "$existing_body" \ --endpoint-url "$endpoint" --region "$AWS_DEFAULT_REGION" \ - --no-cli-pager --no-progress --only-show-errors /dev/null + --cache-control "$immutable_cache" --content-type "$content_type" \ + "${aws_timeouts[@]}" /dev/null created=$((created + 1)) fi node "$FENCE_SCRIPT" current