From 9795347bfcf033b2c0beabef04ffc60422973ba4 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 13:29:54 +0000 Subject: [PATCH 1/9] fix(release): fence publication and recover superseded releases --- .github/workflows/deploy.yml | 12 +- .github/workflows/recover-protocol-cdn.yml | 24 +- .github/workflows/release.yml | 170 ++++--- RELEASING.md | 124 ++++- scripts/backfill-cdn-artifacts.sh | 122 ++++- scripts/check-release-state.cjs | 127 ++++++ tests/release-publication-ordering.test.cjs | 451 +++++++++++++++++++ tests/release-workflow-immutability.test.cjs | 8 +- workers/artifact-cdn/README.md | 9 +- 9 files changed, 973 insertions(+), 74 deletions(-) create mode 100644 scripts/check-release-state.cjs create mode 100644 tests/release-publication-ordering.test.cjs diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e25d6e82d0c..cf922bfa02d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -243,6 +243,14 @@ jobs: # alone can come from the previous release and must never turn an aborted # release command into a green deployment. The convergence gate below # remains mandatory to prove every machine received one image and config. + # Fly also serves dist: do not expose a committed candidate via the app. + - name: Require committed version to have a complete release + env: + GH_TOKEN: ${{ github.token }} + TESTED_SHA: ${{ github.event.workflow_run.head_sha }} + PUBLICATION_BRANCH: main + run: node scripts/check-release-state.cjs pending + - name: Deploy id: deploy env: @@ -495,6 +503,8 @@ jobs: - name: Publish latest artifacts to R2 env: + TESTED_SHA: ${{ github.event.workflow_run.head_sha }} + PUBLICATION_BRANCH: main ADCP_ARTIFACT_R2_BUCKET: ${{ vars.ADCP_ARTIFACT_R2_BUCKET || 'adcp-artifacts' }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} @@ -505,7 +515,7 @@ jobs: echo "::error::Missing R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, or R2_SECRET_ACCESS_KEY secret." exit 1 fi - npm run backfill:cdn-artifacts -- --bucket "${ADCP_ARTIFACT_R2_BUCKET}" --build-latest --quiet + npm run backfill:cdn-artifacts -- --bucket "${ADCP_ARTIFACT_R2_BUCKET}" --build-latest --latest-only --quiet # On any failure (hard or fallback-with-degraded-gates), snapshot Fly # state so the next-turn investigator doesn't have to fetch it diff --git a/.github/workflows/recover-protocol-cdn.yml b/.github/workflows/recover-protocol-cdn.yml index 57ce1304e73..7cc73a26a49 100644 --- a/.github/workflows/recover-protocol-cdn.yml +++ b/.github/workflows/recover-protocol-cdn.yml @@ -12,6 +12,10 @@ concurrency: group: recover-protocol-cdn-${{ inputs.version }} cancel-in-progress: false +env: + TESTED_SHA: ${{ github.sha }} + PUBLICATION_BRANCH: main + permissions: contents: read @@ -25,6 +29,11 @@ jobs: with: fetch-depth: 0 + - name: Preserve current workflow publication fence + run: | + node scripts/check-release-state.cjs current + cp scripts/check-release-state.cjs "$RUNNER_TEMP/check-release-state.cjs" + - name: Install cosign uses: sigstore/cosign-installer@v3 with: @@ -71,6 +80,7 @@ jobs: - name: Upload only missing immutable R2 objects env: VERSION: ${{ inputs.version }} + GH_TOKEN: ${{ github.token }} BUCKET: ${{ vars.ADCP_ARTIFACT_R2_BUCKET || 'adcp-artifacts' }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} @@ -83,6 +93,7 @@ jobs: exit 1 fi + node "$RUNNER_TEMP/check-release-state.cjs" published "$VERSION" endpoint="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" upload_immutable() { @@ -118,14 +129,11 @@ jobs: exit 1 fi - aws s3 cp "${source}" "s3://${BUCKET}/${key}" \ - --endpoint-url "${endpoint}" \ - --region auto \ - --no-progress \ - --only-show-errors \ - --no-guess-mime-type \ - --cache-control 'public, max-age=31536000, immutable' \ - --content-type "${content_type}" + node "$RUNNER_TEMP/check-release-state.cjs" current + aws s3api put-object --bucket "$BUCKET" --key "$key" --body "$source" \ + --if-none-match '*' --endpoint-url "$endpoint" --region auto --no-cli-pager \ + --cache-control 'public, max-age=31536000, immutable' --content-type "$content_type" + node "$RUNNER_TEMP/check-release-state.cjs" current echo "Uploaded missing immutable object ${key}" } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6438be06e78..e4d42c544c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,23 @@ on: - '3.1.x' - '3.0.x' -concurrency: ${{ github.workflow }}-${{ github.ref }} + workflow_dispatch: + inputs: + release_commit: + description: Original approved release merge SHA to recover on the current branch + required: true + type: string + +# Keep pending release commits; freshness fences still reject obsolete runs. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + queue: max + +env: + TESTED_SHA: ${{ github.sha }} + PUBLICATION_BRANCH: ${{ github.ref_name }} + RELEASE_SHA: ${{ inputs.release_commit || github.sha }} jobs: verify-release: @@ -23,6 +39,9 @@ jobs: with: fetch-depth: 0 + - name: Require current release branch + run: node scripts/check-release-state.cjs current + - name: Setup Node.js uses: actions/setup-node@v7 with: @@ -34,10 +53,14 @@ jobs: run: | set -euo pipefail VERSION=$(node -p "require('./package.json').version") - TAG="v${VERSION}" echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - changed_files="$(git show --format= --name-only --no-renames "${GITHUB_SHA}")" + if [ "${GITHUB_EVENT_NAME}" = "workflow_dispatch" ]; then + node scripts/check-release-state.cjs recovery "$RELEASE_SHA" + echo "relevant=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + changed_files="$(git show --first-parent --format= --name-only --no-renames "${GITHUB_SHA}")" if grep -Eq '^(\.changeset/|package(-lock)?[.]json$|dist/(schemas|compliance|protocol)/|static/(schemas|compliance)/source/|server/src/training-agent/|scripts/(build-schemas[.]cjs|build-compliance[.]cjs|build-protocol-tarball[.]cjs|sign-protocol-tarball[.]sh|run-storyboards-([^/]+[.]sh|isolated[.]mjs))|[.]github/workflows/(release|training-agent-storyboards|validate-schema-bundle)[.]yml$)' <<< "${changed_files}"; then echo "relevant=true" >> "$GITHUB_OUTPUT" echo "Release-relevant files changed." @@ -47,6 +70,17 @@ jobs: echo "relevant=false" >> "$GITHUB_OUTPUT" echo "No release-relevant changes detected; skipping release job body." + - name: Detect stranded committed release + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + changed_files="$(git show --first-parent --format= --name-only --no-renames "$GITHUB_SHA")" + VERSION=$(node -p "require('./package.json').version") + if [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] && ! grep -Eq "^dist/(schemas|compliance)/${VERSION}/|^dist/protocol/${VERSION}[.]" <<< "$changed_files"; then + node scripts/check-release-state.cjs pending + fi + - name: Install Dependencies if: steps.release-relevance.outputs.relevant == 'true' run: npm ci @@ -105,6 +139,9 @@ jobs: fetch-depth: 0 token: ${{ steps.app-token.outputs.token }} + - name: Require current release branch + run: node scripts/check-release-state.cjs current + - name: Setup Node.js uses: actions/setup-node@v7 with: @@ -128,10 +165,13 @@ jobs: run: | set -euo pipefail VERSION=$(node -p "require('./package.json').version") - TAG="v${VERSION}" echo "version=${VERSION}" >> "$GITHUB_OUTPUT" - changed_files="$(git show --format= --name-only --no-renames "${GITHUB_SHA}")" + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + node scripts/check-release-state.cjs recovery "$RELEASE_SHA" + fi + node scripts/check-release-state.cjs current + changed_files="$(git show --first-parent --format= --name-only --no-renames "${RELEASE_SHA}")" if grep -Eq "^dist/(schemas|compliance)/${VERSION}/|^dist/protocol/${VERSION}[.]" <<< "${changed_files}"; then echo "has_release_artifacts=true" >> "$GITHUB_OUTPUT" echo "Detected committed release artifact changes for ${VERSION}." @@ -152,19 +192,20 @@ jobs: pulls="$(gh api \ -H 'Accept: application/vnd.github+json' \ - "/repos/${GITHUB_REPOSITORY}/commits/${GITHUB_SHA}/pulls")" + "/repos/${GITHUB_REPOSITORY}/commits/${RELEASE_SHA}/pulls")" pr_number="$(jq -r \ --arg base "${GITHUB_REF_NAME}" \ '[.[] | select(.merged_at != null and .base.ref == $base)] | sort_by(.merged_at) | last | .number // empty' \ <<< "${pulls}")" if [ -z "${pr_number}" ]; then - echo "::error::Release artifact commit ${GITHUB_SHA} is not associated with a merged PR targeting ${GITHUB_REF_NAME}." + echo "::error::Release artifact commit ${RELEASE_SHA} is not associated with a merged PR targeting ${GITHUB_REF_NAME}." exit 1 fi pr="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}")" head_sha="$(jq -r '.head.sha' <<< "${pr}")" + test "$(jq -r '.merge_commit_sha' <<< "$pr")" = "$RELEASE_SHA" author="$(jq -r '.user.login' <<< "${pr}")" reviews="$(gh api --paginate "/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/reviews" | jq -s 'add')" approved_count="$(jq \ @@ -184,6 +225,37 @@ jobs: echo "Release PR #${pr_number} has ${approved_count} human approval(s) on final head ${head_sha}." + - name: Fence Changesets mutations + if: steps.release-artifacts.outputs.has_release_artifacts != 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + node scripts/check-release-state.cjs pending + # Changesets builds for minutes before pushing. Its pinned git-CLI + # path honors pre-push and propagates a failed push before PR writes. + mkdir -p "$RUNNER_TEMP/release-hooks" + cp scripts/check-release-state.cjs "$RUNNER_TEMP/release-hooks/check.cjs" + # shellcheck disable=SC2016 + printf '#!/bin/sh\nnode "$RUNNER_TEMP/release-hooks/check.cjs" current\n' > "$RUNNER_TEMP/release-hooks/pre-push" + chmod +x "$RUNNER_TEMP/release-hooks/pre-push" + git config core.hooksPath "$RUNNER_TEMP/release-hooks" + # Also recheck after a long git push, so Changesets stops before its + # subsequent GitHub PR API request if main moved during the push. + command -v git > "$RUNNER_TEMP/release-hooks/git-path" + cat > "$RUNNER_TEMP/release-hooks/git" <<'SH' + #!/bin/sh + real_git=$(cat "$RUNNER_TEMP/release-hooks/git-path") + if [ "$1" = push ]; then + "$real_git" "$@" || exit $? + node "$RUNNER_TEMP/release-hooks/check.cjs" current + else + exec "$real_git" "$@" + fi + SH + chmod +x "$RUNNER_TEMP/release-hooks/git" + echo "$RUNNER_TEMP/release-hooks" >> "$GITHUB_PATH" + - name: Create Release Pull Request or Tag Release if: steps.release-artifacts.outputs.has_release_artifacts != 'true' id: changesets @@ -197,16 +269,15 @@ jobs: with: github-token: ${{ steps.app-token.outputs.token }} version-script: npm run version - publish-script: npx --no-install changeset git-tag commit-message: 'Version Packages' pr-title: 'Version Packages' - create-github-releases: true + create-github-releases: false push-with-git-cli: true env: HUSKY: '0' - - name: Sign local protocol tarball if needed - if: steps.changesets.outputs.published == 'true' || steps.release-artifacts.outputs.has_release_artifacts == 'true' + - name: Require committed signed protocol tarball + if: steps.release-artifacts.outputs.has_release_artifacts == 'true' run: | set -euo pipefail VERSION="${{ steps.release-artifacts.outputs.version }}" @@ -215,14 +286,11 @@ jobs: echo "::error::Tarball not found at ${TARBALL}. Did npm run version fail upstream?" exit 1 fi - if [ ! -f "${TARBALL}.sig" ] || [ ! -f "${TARBALL}.crt" ]; then - npm run sign:protocol-tarball - fi test -f "${TARBALL}.sig" test -f "${TARBALL}.crt" - name: Verify local protocol tarball before publication - if: steps.changesets.outputs.published == 'true' || steps.release-artifacts.outputs.has_release_artifacts == 'true' + if: steps.release-artifacts.outputs.has_release_artifacts == 'true' run: | set -euo pipefail VERSION="${{ steps.release-artifacts.outputs.version }}" @@ -236,23 +304,10 @@ jobs: --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ "${TARBALL}" - # Attach the protocol tarball + signature sidecars to the GitHub - # Release. For packages with real publish output, changesets/action may - # create the GitHub Release first. For this private repo, Version - # Packages commits can already contain the release artifacts while - # changesets/action reports "All changesets are empty"; create the - # release here if needed. - # - # Tarballs and sidecars (.sha256, .sig, .crt) are written to - # dist/protocol/${VERSION}.tgz{,.sha256,.sig,.crt} by `npm run version` - # in the Version Packages PR (build-protocol-tarball + - # sign-protocol-tarball). - # Without this step the artifacts ship to the repo's dist/ tree but - # are never attached as Release assets, leaving adopters who pin via - # release URL with a 404. v3.0.0 had assets only because they were - # uploaded manually; v3.0.1, v3.0.2, v3.0.3 all shipped empty. + # Stage all four committed assets in a draft, then publish the release + # before R2. Failed staging stays a draft for explicit recovery. - name: Upload protocol tarball to GitHub Release - if: steps.changesets.outputs.published == 'true' || steps.release-artifacts.outputs.has_release_artifacts == 'true' + if: steps.release-artifacts.outputs.has_release_artifacts == 'true' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | @@ -264,51 +319,68 @@ jobs: echo "::error::Tarball not found at ${TARBALL}. Did npm run version fail upstream?" exit 1 fi + node scripts/check-release-state.cjs current + remote_tag="$(git ls-remote origin "refs/tags/$TAG" "refs/tags/$TAG^{}" | tail -1 | cut -f1)" + if [ -n "$remote_tag" ] && [ "$remote_tag" != "$RELEASE_SHA" ]; then + echo '::error::Existing release tag targets a different commit.' + exit 1 + fi if ! gh release view "${TAG}" >/dev/null 2>&1; then release_args=() if [[ "${VERSION}" == *-* ]]; then release_args+=(--prerelease) fi - gh release create "${TAG}" \ - --target "${GITHUB_SHA}" \ + node scripts/check-release-state.cjs current + gh release create "${TAG}" --draft \ + --target "${RELEASE_SHA}" \ --title "${TAG}" \ --notes "AdCP ${VERSION}" \ "${release_args[@]}" fi + release_state="$(gh release view "$TAG" --json isDraft,targetCommitish)" + if [ "$(jq -r '.isDraft' <<< "$release_state")" = true ] && [ "$(jq -r '.targetCommitish' <<< "$release_state")" != "$RELEASE_SHA" ]; then + echo '::error::Existing draft targets a different release commit.' + exit 1 + fi + upload_asset_if_missing() { local path="$1" local name name="$(basename "${path}")" local remote_name - remote_name="$(gh release view "${TAG}" --json assets --jq ".assets[] | select(.name == \"${name}\") | .name" || true)" - local remote_digest - remote_digest="$(gh release view "${TAG}" --json assets --jq ".assets[] | select(.name == \"${name}\") | .digest" || true)" - + remote_name="$(gh release view "${TAG}" --json assets --jq ".assets[] | select(.name == \"${name}\") | .name")" if [ -n "${remote_name}" ]; then - if [[ "${remote_digest}" == sha256:* && "${name}" != *.sig && "${name}" != *.crt ]]; then - local local_digest - local_digest="sha256:$(shasum -a 256 "${path}" | awk '{print $1}')" - if [ "${local_digest}" != "${remote_digest}" ]; then - echo "::error::Release asset ${name} already exists on ${TAG} with digest ${remote_digest}, but local digest is ${local_digest}. Refusing to overwrite immutable release asset." - exit 1 - fi + local existing + existing="$(mktemp -d)" + gh release download "$TAG" --pattern "$name" --dir "$existing" + if ! cmp -s "$path" "$existing/$name"; then + echo "::error::Release asset $name differs; refusing immutable overwrite." + rm -rf "$existing" + exit 1 fi + rm -rf "$existing" echo "Release asset ${name} already exists on ${TAG}; leaving it untouched." return 0 fi + node scripts/check-release-state.cjs current gh release upload "${TAG}" "${path}" + node scripts/check-release-state.cjs current } upload_asset_if_missing "${TARBALL}" upload_asset_if_missing "${TARBALL}.sha256" upload_asset_if_missing "${TARBALL}.sig" upload_asset_if_missing "${TARBALL}.crt" + node scripts/check-release-state.cjs current + gh release edit "$TAG" --draft=false + node scripts/check-release-state.cjs published "$VERSION" - name: Publish release artifacts to R2 - if: steps.changesets.outputs.published == 'true' || steps.release-artifacts.outputs.has_release_artifacts == 'true' + if: steps.release-artifacts.outputs.has_release_artifacts == 'true' env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} ADCP_ARTIFACT_R2_BUCKET: ${{ vars.ADCP_ARTIFACT_R2_BUCKET || 'adcp-artifacts' }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID || secrets.CLOUDFLARE_ACCOUNT_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID || secrets.AWS_ACCESS_KEY_ID }} @@ -319,14 +391,10 @@ jobs: echo "::error::Missing R2_ACCOUNT_ID, R2_ACCESS_KEY_ID, or R2_SECRET_ACCESS_KEY secret." exit 1 fi - latest_args=() - if [ "${GITHUB_REF_NAME}" != "main" ]; then - latest_args=(--skip-latest) - fi - npm run backfill:cdn-artifacts -- --bucket "${ADCP_ARTIFACT_R2_BUCKET}" --quiet "${latest_args[@]}" + npm run backfill:cdn-artifacts -- --bucket "${ADCP_ARTIFACT_R2_BUCKET}" --quiet --version "${{ steps.release-artifacts.outputs.version }}" --skip-latest - name: Verify published protocol tarball from CDN - if: steps.changesets.outputs.published == 'true' || steps.release-artifacts.outputs.has_release_artifacts == 'true' + if: steps.release-artifacts.outputs.has_release_artifacts == 'true' run: | set -euo pipefail VERSION="${{ steps.release-artifacts.outputs.version }}" diff --git a/RELEASING.md b/RELEASING.md index 6f0702ba08e..952e01a3e54 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -194,9 +194,129 @@ re-enter another tag before that stable versioning step. ## Recovery -If the automated release fails after the Version Packages merge, fix and -rerun the workflow when safe. Manual tag/release recovery is the last resort: +If the automated release fails after the Version Packages merge, use the +explicit current-head recovery procedure below. Manual tag/release recovery +is the last resort: the tag must target the Version Packages merge, and the same four signed assets must be uploaded without overwriting an existing artifact with a different digest. See `.agents/shortcuts/cut-major.md` for the fallback commands. + +### Publication authority and ordering + +Routine app deployment is authorized to rebuild and upload only +`schemas/latest/**`, `compliance/latest/**`, and `protocol/latest.tgz*`. +It uses `backfill-cdn-artifacts.sh --latest-only`; root version indexes and +all semver paths are excluded. The app itself serves committed `dist`, so a +committed package version without a published GitHub Release and its four +assets also blocks app deployment. This can delay app-only fixes while a +release needs recovery. + +Only the release workflow publishes a new immutable version. It verifies the +current tested branch, the original merged release PR's final-head human +approval, and the committed signed tuple. It stages the four GitHub assets in +a draft, publishes the tag/release targeting the original release merge, then +uploads that version alone with `--version VERSION --skip-latest`. Changesets +only prepares release PRs; it has no independent tag/publish command. Missing +signatures must be fixed through review, never regenerated during publication. + +Release runs use `queue: max` without in-progress cancellation. GitHub permits +100 pending runs; this is not an unlimited or guaranteed delivery queue. Old +runs fail their current-branch fence. A later push with a stranded committed +version fails explicitly, including an app-only push, and must use the recovery +procedure below before Changesets advances the package again. The git pre-push +hook rechecks freshness after Changesets' long version/build/sign operation, +before its branch push. A wrapper checks again after the push, before the +subsequent PR write. + +Freshness is checked again before external writes and after uploads. GitHub +refs, GitHub Releases, and R2 have no shared transaction: a branch can move +between a check and a request. The pre-push hook narrows the Changesets race to +its push/PR requests; it does not lock main. An in-flight request can finish +after main moves. Such a failure requires inspection and explicit recovery; +never treat a previous verification or a queued run as current authority. + +GitHub assets are staged privately in a draft before release publication. R2 +publication is **progressive, with per-object atomicity only**: conditional +`If-None-Match: *` writes create missing objects, and existing objects must +match byte-for-byte, including signatures and certificates. A failed upload +may leave a publicly visible subset after the approved tag/release. It must +not overwrite, delete, advertise whole-bundle atomicity, or claim CDN +completion. Consumers needing one complete artifact should use the signed +GitHub tarball and verify its checksum/signature; CDN completion additionally +requires a full committed-path byte audit, including JSONL after the separate +extension-filter fix. Mutable `latest` uploads remain progressive as well. + +Live unscoped bulk backfills are disabled. `--dry-run` remains available for +inventory; it is not release authorization. Historical CDN recovery must use +an existing approved tag/release, a current tested branch fence, unchanged +committed bytes, and conditional missing-object creation. The stable-only +`recover-protocol-cdn.yml` recovers four protocol files, not schemas/compliance +and not RC releases. + +### Explicit recovery of a superseded release merge + +Do not rerun the obsolete push: a rerun retains its old SHA and workflow. +After the corrected workflow is merged, an authorized maintainer can dispatch +`release.yml` **on current main** (or the applicable current maintenance branch) +with `release_commit` set to the original approved release merge. The dispatch +re-runs the existing verification jobs on current main, verifies that the +release merge is an ancestor, requires the same package version and unchanged +versioned artifacts, rechecks original final-head human approval, and fences +every publication phase. It never silently repairs from an unrelated push. +A new main commit during the attempt requires another explicit dispatch and +verification of the new head. If a later package version has already landed, +stop for a separately reviewed historical recovery; this path refuses it. + +For the 2026-09-14 `3.2.0-rc.3` incident, the recovery target is +`71f9cd5414454e94ccfef87ce25777ead6fad228` (#7485), whose final PR head +`e7d42cbe5d7b3d05c3fcdcaa6f568b08ba131439` received human approval at +12:34:18 UTC. Release run `34844149713` was cancelled with zero jobs. Deploy +`34845112730` exposed its R2 objects before a tag/GitHub Release; the independent +incident audit found 2,092 matching objects of 2,093 committed paths. The JSONL +workstream owns the omitted path/filter and full enumeration verification. +This version is externally exposed; absence of a GitHub Release is not +permission to edit or retire its bytes. + +The stale `eb3cbd605fc11be6e398c5b83f4e53d332a8e365` release run +`34842463221` subsequently created #7508 at +`102916bb28c156b1710f031aa28e4655bcf71cdc`, tree +`fd299f52f6bf1a4427ef85f890832d1e5713aade`. Its sole parent/merge-base +is eb3cbd, two commits behind then-current af1. The direct delta against +current main changes exposed rc.3 artifacts and omits #7504; its three-dot PR +display obscures that omission. The coordinator marked it Draft, disabled +auto-merge, and posted a +[HOLD](https://github.com/adcontextprotocol/adcp/pull/7508#issuecomment-5664735087). +It is not a recovery candidate. Do not merge, close, replace, or otherwise +mutate it without coordinating with that owner; retain the original approved +71f release artifacts. The fencing regression tests cover this stale-parent +shape, missing later changeset, and conflicting rc.3 bytes. + +The exact recovery sequence, requiring separate maintainer authorization, is: + +1. Review and merge the ordering fix and JSONL filter fix through normal human + review. Coordinate #7502/#7503's immutability guard and documentation claims; + this patch changes no release content. Account for every existing release + and deploy run using the old workflow, including `34842463221` and + `34847610745`, before enabling recovery. New workflow code does not fence + runs that already loaded the old code. Maintainers must quiesce those old + writers and reserve a quiet publication window. +2. Recheck current main, original approval, ancestry, exact package version, + the full committed rc.3 tree, checksum, Sigstore identity, tag and release + state, and every existing R2 object's bytes. Any conflicting tag, release + asset, or R2 bytes is a stop; do not overwrite, delete, regenerate, roll + back, or retag. Preserve already exposed bytes and record discrepancies. +3. Dispatch `release.yml` on that current main with `release_commit` equal to + `71f9cd5414454e94ccfef87ce25777ead6fad228`. Let all original verification and + human-approval checks run. It stages/reuses the identical four assets, + publishes `v3.2.0-rc.3` at **71f**, and conditionally fills missing rc.3 R2 + objects. It does not tag the later tested main or publish `latest`. +4. Independently verify the tag target, prerelease flag, all four GitHub assets, + full 2,093-path CDN inventory and byte hashes (including JSONL), checksum and + Sigstore identity. Check deployment and docs/SDK readiness separately. If + anything fails or main advances, record the partial state and repeat only + through explicit recovery from the next verified current head. + +These are recovery instructions, not a record of operations performed. The +ordering-fix work must not publish, delete, roll back, tag, release, deploy, +rerun/cancel workflows, mutate R2/CDN, or change branch rules. diff --git a/scripts/backfill-cdn-artifacts.sh b/scripts/backfill-cdn-artifacts.sh index 424f1e4e397..3d1ca06ad80 100755 --- a/scripts/backfill-cdn-artifacts.sh +++ b/scripts/backfill-cdn-artifacts.sh @@ -19,6 +19,10 @@ Options: --aws-dry-run Execute AWS with --dryrun. Very verbose: AWS prints every object it would upload. --build-latest Rebuild dist/*/latest and dist/protocol/latest.tgz first. + --latest-only Upload only mutable development artifacts (routine deploy). + --version VERSION Publish only this already-approved, tagged GitHub release. + Existing objects must match; missing objects are created + conditionally. Requires --skip-latest. --skip-latest Upload only versioned artifacts; do not update mutable schemas/latest, compliance/latest, or protocol/latest. --apply-cors Apply public read CORS config to the bucket via Wrangler. @@ -45,6 +49,8 @@ dry_run=0 aws_dry_run=0 build_latest=0 skip_latest=0 +latest_only=0 +release_version="" apply_cors=0 quiet=0 @@ -89,6 +95,14 @@ while [[ $# -gt 0 ]]; do build_latest=1 shift ;; + --latest-only) + latest_only=1 + shift + ;; + --version) + release_version="$(require_option_value "$1" "${2:-}")" + shift 2 + ;; --skip-latest) skip_latest=1 shift @@ -117,6 +131,20 @@ while [[ $# -gt 0 ]]; do esac done +# Live bulk backfills could publish unapproved versions. Select one authority. +if [[ "$latest_only" -eq 1 && ( "$skip_latest" -eq 1 || -n "$release_version" ) ]]; then + echo "--latest-only cannot be combined with versioned publication." >&2 + exit 2 +fi +if [[ -n "$release_version" && ( ! "$release_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ || "$skip_latest" -ne 1 || "$build_latest" -eq 1 ) ]]; then + echo "--version requires exact semver, --skip-latest, and committed artifacts." >&2 + exit 2 +fi +if [[ "$dry_run" -eq 0 && "$latest_only" -eq 0 && -z "$release_version" ]]; then + echo "Live bulk backfill is disabled; use --latest-only or --version VERSION --skip-latest. See RELEASING.md." >&2 + exit 2 +fi + if [[ -n "$env_file" ]]; then if [[ ! -f "$env_file" ]]; then echo "Env file not found: $env_file" >&2 @@ -186,6 +214,14 @@ Run with --build-latest if this checkout has not already run the artifact build. WARN fi +fence_script="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check-release-state.cjs" +if [[ "$dry_run" -eq 0 ]]; then + node "$fence_script" current + if [[ -n "$release_version" ]]; then + node "$fence_script" published "$release_version" + fi +fi + planned_count="$(find dist/schemas dist/compliance dist/protocol -type f ! -path '*/.staging/*' | wc -l | tr -d ' ')" planned_bytes="$(du -sk dist/schemas dist/compliance dist/protocol | awk '{sum += $1} END {printf "%.0f", sum * 1024}')" @@ -215,6 +251,7 @@ apply_bucket_cors() { ] } JSON + node "$fence_script" current wrangler r2 bucket cors set "$bucket" --file "$cors_file" --force rm -f "$cors_file" } @@ -226,7 +263,58 @@ run_cmd() { if [[ "$dry_run" -eq 1 && "$aws_dry_run" -eq 0 ]]; then return 0 fi + if [[ "$dry_run" -eq 0 ]]; then node "$fence_script" current; fi "$@" + if [[ "$dry_run" -eq 0 ]]; then node "$fence_script" current; fi +} + +# R2 writes are atomic per object, not per bundle. Compare every existing +# object byte-for-byte; conditional creation closes the head/put overwrite race. +put_immutable() { + local source="$1" key="$2" cache_control="$3" content_type="$4" + local existing head_error + existing="$(mktemp)" + head_error="$(mktemp)" + node "$fence_script" current + if aws s3api head-object --bucket "$bucket" --key "$key" \ + --endpoint-url "$endpoint" --region "$AWS_DEFAULT_REGION" --no-cli-pager >/dev/null 2>"$head_error"; then + aws s3 cp "s3://${bucket}/${key}" "$existing" "${common_aws_args[@]}" + if ! cmp -s "$source" "$existing"; then + echo "Immutable object differs: $key; refusing overwrite." >&2 + rm -f "$existing" "$head_error" + exit 1 + fi + else + if ! grep -Eq 'An error occurred \(404\)|\(NoSuchKey\)' "$head_error"; then + cat "$head_error" >&2 + rm -f "$existing" "$head_error" + exit 1 + fi + node "$fence_script" current + aws s3api put-object --bucket "$bucket" --key "$key" --body "$source" \ + --if-none-match '*' --endpoint-url "$endpoint" --region "$AWS_DEFAULT_REGION" \ + --no-cli-pager --cache-control "$cache_control" --content-type "$content_type" >/dev/null + fi + rm -f "$existing" "$head_error" + node "$fence_script" current +} + +publish_filtered() { + local root="$1" prefix="$2" cache_control="$3" content_type="$4" + shift 4 + local file relative pattern + while IFS= read -r -d '' file; do + [[ -f "$file" && ! -L "$file" ]] || { echo "Missing or symlinked artifact: $file" >&2; exit 1; } + relative="${file#"$root"/}" + for pattern in "$@"; do + # Deliberately use the same extension globs as the AWS filters. + # shellcheck disable=SC2053 + if [[ "$relative" == $pattern ]]; then + put_immutable "$file" "$prefix/$relative" "$cache_control" "$content_type" + break + fi + done + done < <(git ls-files -z -- "$root") } sync_filtered() { @@ -236,6 +324,10 @@ sync_filtered() { local content_type="$4" shift 4 local patterns=("$@") + if [[ -n "$release_version" && "$dry_run" -eq 0 ]]; then + publish_filtered "$source_dir" "$dest_prefix" "$cache_control" "$content_type" "${patterns[@]}" + return + fi local args=( s3 sync "$source_dir" @@ -265,6 +357,10 @@ cp_filtered_recursive() { local content_type="$4" shift 4 local patterns=("$@") + if [[ -n "$release_version" && "$dry_run" -eq 0 ]]; then + publish_filtered "$source_dir" "$dest_prefix" "$cache_control" "$content_type" "${patterns[@]}" + return + fi local args=( s3 cp "$source_dir" @@ -292,6 +388,10 @@ cp_file() { local dest_key="$2" local cache_control="$3" local content_type="$4" + if [[ -n "$release_version" && "$dry_run" -eq 0 ]]; then + put_immutable "$file" "$dest_key" "$cache_control" "$content_type" + return + fi run_cmd aws s3 cp "$file" "s3://${bucket}/${dest_key}" \ "${common_aws_args[@]}" \ --no-guess-mime-type \ @@ -344,21 +444,23 @@ fi immutable_cache="public, max-age=31536000, immutable" revalidate_cache="public, no-cache, must-revalidate" +if [[ "$latest_only" -eq 0 ]]; then for schema_dir in dist/schemas/*; do if [[ ! -d "$schema_dir" ]]; then continue fi schema_version="$(basename "$schema_dir")" - if [[ "$schema_version" == "latest" ]]; then + if [[ "$schema_version" == "latest" || ( -n "$release_version" && "$schema_version" != "$release_version" ) ]]; then continue fi sync_schema_tree "$schema_dir" "schemas/$schema_version" "$immutable_cache" done +fi if [[ "$skip_latest" -eq 0 ]]; then - if [[ -f dist/schemas/index.json ]]; then + if [[ "$latest_only" -eq 0 && -f dist/schemas/index.json ]]; then cp_file dist/schemas/index.json schemas/index.json "$revalidate_cache" "application/json; charset=utf-8" fi - if [[ -f dist/schemas/latest.json ]]; then + if [[ "$latest_only" -eq 0 && -f dist/schemas/latest.json ]]; then cp_file dist/schemas/latest.json schemas/latest.json "$revalidate_cache" "application/json; charset=utf-8" fi if [[ -d dist/schemas/latest ]]; then @@ -366,16 +468,28 @@ if [[ "$skip_latest" -eq 0 ]]; then fi fi -exclude_latest=1 sync_compliance_tree dist/compliance compliance "$immutable_cache" +if [[ -n "$release_version" ]]; then + sync_compliance_tree "dist/compliance/$release_version" "compliance/$release_version" "$immutable_cache" +elif [[ "$latest_only" -eq 0 ]]; then + exclude_latest=1 sync_compliance_tree dist/compliance compliance "$immutable_cache" +fi if [[ "$skip_latest" -eq 0 && -d dist/compliance/latest ]]; then cp_compliance_tree dist/compliance/latest compliance/latest "$revalidate_cache" fi +if [[ -n "$release_version" ]]; then + cp_file "dist/protocol/$release_version.tgz" "protocol/$release_version.tgz" "$immutable_cache" "application/gzip" + cp_file "dist/protocol/$release_version.tgz.sha256" "protocol/$release_version.tgz.sha256" "$immutable_cache" "text/plain; charset=utf-8" + cp_file "dist/protocol/$release_version.tgz.sig" "protocol/$release_version.tgz.sig" "$immutable_cache" "application/octet-stream" + cp_file "dist/protocol/$release_version.tgz.crt" "protocol/$release_version.tgz.crt" "$immutable_cache" "application/x-pem-file" +elif [[ "$latest_only" -eq 0 ]]; then exclude_latest=1 sync_filtered dist/protocol protocol "$immutable_cache" "application/gzip" "*.tgz" exclude_latest=1 cp_filtered_recursive dist/protocol protocol "$revalidate_cache" "text/plain; charset=utf-8" "*.sha256" exclude_latest=1 cp_filtered_recursive dist/protocol protocol "$revalidate_cache" "application/octet-stream" "*.sig" exclude_latest=1 cp_filtered_recursive dist/protocol protocol "$revalidate_cache" "application/x-pem-file" "*.crt" +fi + if [[ "$skip_latest" -eq 0 && -f dist/protocol/latest.tgz ]]; then cp_file dist/protocol/latest.tgz protocol/latest.tgz "$revalidate_cache" "application/gzip" fi diff --git a/scripts/check-release-state.cjs b/scripts/check-release-state.cjs new file mode 100644 index 00000000000..14367f8c221 --- /dev/null +++ b/scripts/check-release-state.cjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node +// Release/deploy fences. Remote failures are errors, never evidence of absence. +const { execFileSync } = require("node:child_process"); +const fs = require("node:fs"); + +const run = (command, args) => + execFileSync(command, args, { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +const git = (...args) => run("git", args); +const shaPattern = /^[a-f0-9]{40}$/; + +function current() { + const expected = process.env.TESTED_SHA || process.env.GITHUB_SHA; + const branch = process.env.PUBLICATION_BRANCH || process.env.GITHUB_REF_NAME; + if ( + !shaPattern.test(expected || "") || + !["main", "3.1.x", "3.0.x"].includes(branch) + ) { + throw new Error( + "Publication requires an exact tested SHA and a supported release branch.", + ); + } + const remote = git( + "ls-remote", + "--exit-code", + "origin", + `refs/heads/${branch}`, + ); + if (remote !== `${expected}\trefs/heads/${branch}`) { + throw new Error( + `Refusing stale tested SHA ${expected}; origin/${branch} is ${remote || "unavailable"}. Use the current branch and the explicit recovery procedure in RELEASING.md.`, + ); + } + return expected; +} + +function published( + version = JSON.parse(fs.readFileSync("package.json")).version, + releaseSha, +) { + if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(version)) + throw new Error("Invalid release version."); + const tag = `v${version}`; + const refs = git( + "ls-remote", + "--exit-code", + "origin", + `refs/tags/${tag}`, + `refs/tags/${tag}^{}`, + ) + .split("\n") + .map((line) => line.split("\t")); + const target = + refs.find(([, ref]) => ref.endsWith("^{}"))?.[0] || refs[0]?.[0]; + if (!shaPattern.test(target || "") || (releaseSha && target !== releaseSha)) { + throw new Error( + `Tag ${tag} does not identify the approved release commit ${releaseSha || ""}.`, + ); + } + const paths = [ + `dist/schemas/${version}`, + `dist/compliance/${version}`, + ...["", ".sha256", ".sig", ".crt"].map( + (s) => `dist/protocol/${version}.tgz${s}`, + ), + ]; + if (git("diff", "--name-only", target, "--", ...paths)) + throw new Error("Local release artifacts differ from the published tag."); + const release = JSON.parse( + run("gh", ["release", "view", tag, "--json", "tagName,isDraft,assets"]), + ); + const names = release.assets.map((asset) => asset.name); + if ( + release.tagName !== tag || + release.isDraft || + !["", ".sha256", ".sig", ".crt"].every((suffix) => + names.includes(`${version}.tgz${suffix}`), + ) + ) { + throw new Error( + `Release ${tag} is not published with the complete signed tuple.`, + ); + } +} + +function recovery(source) { + const tested = current(); + if (!shaPattern.test(source || "")) + throw new Error("Recovery requires the original release merge SHA."); + git("merge-base", "--is-ancestor", source, tested); + const version = JSON.parse(git("show", `${source}:package.json`)).version; + if (version !== JSON.parse(fs.readFileSync("package.json")).version) + throw new Error("Recovery must precede the next version bump."); + const paths = [ + `dist/schemas/${version}`, + `dist/compliance/${version}`, + ...["", ".sha256", ".sig", ".crt"].map( + (s) => `dist/protocol/${version}.tgz${s}`, + ), + ]; + for (const path of paths) git("cat-file", "-e", `${source}:${path}`); + if (git("diff", "--name-only", source, tested, "--", ...paths)) + throw new Error( + "Recovery artifacts differ from the original approved commit.", + ); +} + +try { + const [mode, argument] = process.argv.slice(2); + if (mode === "current") current(); + else if (mode === "published") { + current(); + published(argument, process.env.RELEASE_SHA); + } else if (mode === "pending") { + current(); + const version = JSON.parse(fs.readFileSync("package.json")).version; + if (fs.existsSync(`dist/protocol/${version}.tgz`)) published(version); + } else if (mode === "recovery") recovery(argument); + else throw new Error("Expected current, published, pending, or recovery."); +} catch (error) { + console.error( + `::error::${error.message}\nPublication stopped. A missing/incomplete release must be recovered explicitly from its original approved merge on current tested main; see RELEASING.md. Do not rerun an obsolete workflow.`, + ); + process.exitCode = 1; +} diff --git a/tests/release-publication-ordering.test.cjs b/tests/release-publication-ordering.test.cjs new file mode 100644 index 00000000000..f9b05b033f2 --- /dev/null +++ b/tests/release-publication-ordering.test.cjs @@ -0,0 +1,451 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { execFileSync, spawnSync } = require("node:child_process"); +const YAML = require("yaml"); +const root = path.resolve(__dirname, ".."); +const release = YAML.parse( + fs.readFileSync(path.join(root, ".github/workflows/release.yml"), "utf8"), +); +const deploy = YAML.parse( + fs.readFileSync(path.join(root, ".github/workflows/deploy.yml"), "utf8"), +); +const steps = release.jobs.release.steps; +const step = (name) => steps.find((s) => s.name === name); +const version = "3.2.0-rc.3"; +const realGit = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); + +function fixture(t) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "publication-order-")); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const write = (file, value) => { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), value); + }; + const git = (...args) => + execFileSync(realGit, args, { + cwd: dir, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + write("package.json", JSON.stringify({ version })); + for (const area of ["schemas", "compliance"]) { + write(`dist/${area}/${version}/index.json`, '{"release":true}'); + write(`dist/${area}/latest/index.json`, '{"development":true}'); + } + for (const suffix of ["", ".sha256", ".sig", ".crt"]) + write(`dist/protocol/${version}.tgz${suffix}`, `signed${suffix}`); + write("dist/protocol/latest.tgz", "development"); + for (const script of ["check-release-state.cjs", "backfill-cdn-artifacts.sh"]) + write( + `scripts/${script}`, + fs.readFileSync(path.join(root, "scripts", script)), + ); + git("init", "-q"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.invalid"); + git("add", "."); + git("commit", "-qm", "Approved release"); + const source = git("rev-parse", "HEAD"); + write(".changeset/next.md", "later changeset"); + git("add", "."); + git("commit", "-qm", "Later changeset"); + const head = git("rev-parse", "HEAD"); + write("remote", head); + write("tag", source); + write("calls", ""); + write( + "release.json", + JSON.stringify({ + tagName: `v${version}`, + targetCommitish: source, + isDraft: false, + assets: ["", ".sha256", ".sig", ".crt"].map((s) => ({ + name: `${version}.tgz${s}`, + })), + }), + ); + const executable = (name, code) => { + write(`bin/${name}`, `#!/usr/bin/env node\n${code}`); + fs.chmodSync(path.join(dir, "bin", name), 0o755); + }; + const common = `const fs=require('node:fs'),path=require('node:path');const args=process.argv.slice(2);const read=f=>fs.readFileSync(f,'utf8');const log=s=>fs.appendFileSync('calls',s+'\\n');const change=()=>fs.writeFileSync('remote','f'.repeat(40));`; + executable( + "git", + common + + ` + if(args[0]==='ls-remote') { + if(process.env.REMOTE_ERROR) process.exit(1); + const ref=args.find(a=>a.startsWith('refs/heads/')); + if(ref) { console.log(read('remote')+'\\t'+ref); process.exit(0); } + const tag=read('tag'); if(!tag) process.exit(args.includes('--exit-code')?2:0); + console.log(tag+'\\trefs/tags/v${version}'); process.exit(0); + } + const r=require('node:child_process').spawnSync(${JSON.stringify(realGit)},args,{stdio:'inherit'});process.exit(r.status??1); + `, + ); + executable( + "gh", + common + + ` + const release=()=>JSON.parse(read('release.json')); + const save=r=>fs.writeFileSync('release.json',JSON.stringify(r)); + if(args[0]==='api') { + const url=args.find(a=>a.startsWith('/repos/')); + if(url.includes('/commits/')) console.log(JSON.stringify([{number:1,merged_at:'2026-09-14',base:{ref:'main'}}])); + else if(url.endsWith('/reviews')) console.log(process.env.REVIEWS||'[]'); + else console.log(JSON.stringify({head:{sha:'a'.repeat(40)},merge_commit_sha:process.env.RELEASE_SHA,user:{login:'author'}})); + } else if(args[1]==='view') { + if(!fs.existsSync('release.json')) process.exit(1); + const r=release(); + if(args.includes('--jq')) { + const q=args[args.indexOf('--jq')+1];const name=q.match(/name == "([^"]+)"/)[1];console.log(r.assets.find(a=>a.name===name)?.name||''); + } else console.log(JSON.stringify(r)); + } else if(args[1]==='create') { + log('github draft'); save({tagName:args[2],targetCommitish:process.env.RELEASE_SHA,isDraft:true,assets:[]}); + } else if(args[1]==='upload') { + const r=release(); if(!r.isDraft) throw Error('upload was publicly visible before staging completed'); + const name=path.basename(args[3]);r.assets.push({name});save(r);log('github upload '+name); + if(process.env.ADVANCE_GITHUB) change(); + } else if(args[1]==='download') { + const name=args[args.indexOf('--pattern')+1],dir=args[args.indexOf('--dir')+1]; + fs.copyFileSync('dist/protocol/'+name,path.join(dir,name)); + } else if(args[1]==='edit') { + const r=release();if(r.assets.length!==4) throw Error('incomplete release');r.isDraft=false;save(r);fs.writeFileSync('tag',process.env.RELEASE_SHA);log('github publish'); + } else throw Error('unexpected gh '+args); + `, + ); + executable( + "aws", + common + + ` + const value=k=>args[args.indexOf(k)+1]; + const key=args.includes('--key')?value('--key'):'';const object=path.join('bucket',key); + if(args[1]==='head-object') { + if(process.env.HEAD_ERROR) { console.error('Forbidden (403)');process.exit(1); } + if(!fs.existsSync(object)) {console.error('An error occurred (404) when calling HeadObject');process.exit(1);} + } else if(args[1]==='put-object') { + if(value('--if-none-match')!=='*') throw Error('unconditional write'); + if(fs.existsSync(object)||process.env.PUT_RACE) {console.error('PreconditionFailed');process.exit(1);} + fs.mkdirSync(path.dirname(object),{recursive:true});fs.copyFileSync(value('--body'),object);log('r2 put '+key); + if(process.env.ADVANCE_AWS) change(); + } else if(args[0]==='s3'&&args[1]==='cp'&&args[2].startsWith('s3://')) { + fs.copyFileSync(path.join('bucket',args[2].split('/').slice(3).join('/')),args[3]); + } else if(args[0]==='s3') { + log('r2 mutable '+args[3]);if(process.env.ADVANCE_AWS) change(); + } else throw Error('unexpected aws '+args); + `, + ); + executable("npm", common + `if(process.env.ADVANCE_BUILD) change();`); + const env = { + PATH: `${dir}/bin:${process.env.PATH}`, + TESTED_SHA: head, + GITHUB_SHA: head, + PUBLICATION_BRANCH: "main", + GITHUB_REF_NAME: "main", + GITHUB_REPOSITORY: "adcontextprotocol/adcp", + RELEASE_SHA: source, + R2_ENDPOINT: "https://r2.invalid", + AWS_ACCESS_KEY_ID: "test", + AWS_SECRET_ACCESS_KEY: "test", + RUNNER_TEMP: path.join(dir, "temp"), + GITHUB_PATH: path.join(dir, "github-path"), + }; + const run = (command, args, extra = {}) => + spawnSync( + command, + command === "bash" ? ["--noprofile", "--norc", ...args] : args, + { cwd: dir, env: { ...env, ...extra }, encoding: "utf8" }, + ); + const script = (mode, extra) => + run("node", ["scripts/check-release-state.cjs", mode], extra); + const upload = (options, extra) => + run( + "bash", + ["scripts/backfill-cdn-artifacts.sh", "--bucket", "test", ...options], + extra, + ); + return { + dir, + source, + head, + write, + git, + run, + script, + upload, + calls: () => fs.readFileSync(path.join(dir, "calls"), "utf8"), + }; +} +const immutable = ["--version", version, "--skip-latest"]; +const approval = (extra) => + JSON.stringify([ + { + user: { type: "User", login: "reviewer" }, + state: "APPROVED", + commit_id: "a".repeat(40), + submitted_at: "2026-09-14", + ...extra, + }, + ]); + +test("app-only deploy publishes only development keys with no human release approval", (t) => { + const f = fixture(t); + const result = f.upload(["--latest-only"]); + assert.equal(result.status, 0, result.stderr); + assert.ok(f.calls().includes("schemas/latest")); + assert.ok(!f.calls().includes(version)); + assert.ok( + f + .calls() + .trim() + .split("\n") + .every((line) => /\/latest(?:\/|\.|$)/.test(line)), + ); +}); +for (const scenario of [ + "before upload", + "during long build", + "during upload", + "remote lookup failure", +]) + test(`fails closed when main changes ${scenario}`, (t) => { + const f = fixture(t); + if (scenario === "before upload") f.write("remote", "f".repeat(40)); + const env = + scenario === "during long build" + ? { ADVANCE_BUILD: "1" } + : scenario === "during upload" + ? { ADVANCE_AWS: "1" } + : scenario === "remote lookup failure" + ? { REMOTE_ERROR: "1" } + : {}; + const r = f.upload(["--latest-only", "--build-latest"], env); + assert.notEqual(r.status, 0); + assert.equal( + f.calls().trim().split("\n").filter(Boolean).length, + scenario === "during upload" ? 1 : 0, + ); + }); +test("main advances during a long app deploy after its successful pre-deploy gate", (t) => { + const f = fixture(t); + assert.equal(f.script("pending").status, 0); + // Simulate a completed long app deployment: its tested SHA is now obsolete. + f.write("remote", "f".repeat(40)); + assert.notEqual(f.upload(["--latest-only"]).status, 0); + assert.equal(f.calls(), ""); +}); +test("old run cannot update Changesets after verification/versioning or enter publication", (t) => { + const f = fixture(t); + const install = f.run("bash", ["-c", step("Fence Changesets mutations").run]); + assert.equal(install.status, 0, install.stderr); + f.write("remote", "f".repeat(40)); + const hook = f.run("sh", [path.join(f.dir, "temp/release-hooks/pre-push")]); + assert.notEqual(hook.status, 0); + assert.notEqual(f.script("current").status, 0); + assert.equal(f.calls(), ""); +}); +test("queued committed release survives later pushes; lost/manual cancellations require explicit recovery", (t) => { + assert.deepEqual(release.concurrency, { + group: "${{ github.workflow }}-${{ github.ref }}", + "cancel-in-progress": false, + queue: "max", + }); + const f = fixture(t); + f.write("tag", ""); + assert.notEqual(f.script("pending").status, 0); + assert.notEqual(f.upload(immutable).status, 0); + assert.equal(f.calls(), ""); + assert.equal( + f.run("node", ["scripts/check-release-state.cjs", "recovery", f.source]) + .status, + 0, + ); + f.write(`dist/schemas/${version}/index.json`, "changed"); + f.git("add", "."); + f.git("commit", "-qm", "changed artifact"); + const head = f.git("rev-parse", "HEAD"); + f.write("remote", head); + assert.notEqual( + f.run("node", ["scripts/check-release-state.cjs", "recovery", f.source], { + TESTED_SHA: head, + }).status, + 0, + ); +}); +for (const [name, reviews] of [ + ["none", "[]"], + ["bot", approval({ user: { type: "Bot", login: "bot" } })], + ["stale", approval({ commit_id: "b".repeat(40) })], + ["dismissed", approval({ state: "DISMISSED" })], +]) + test(`no ${name} approval can publish`, (t) => { + const f = fixture(t); + const r = f.run( + "bash", + [ + "-c", + step("Require human approval for committed release artifacts").run, + ], + { REVIEWS: reviews }, + ); + assert.notEqual(r.status, 0); + assert.equal(f.calls(), ""); + }); +test("approved publication stages complete GitHub tuple before release and scoped conditional R2 writes", (t) => { + const f = fixture(t); + f.write("tag", ""); + fs.unlinkSync(path.join(f.dir, "release.json")); + const r = f.run( + "bash", + [ + "-c", + step("Require human approval for committed release artifacts").run + + "\n" + + step("Upload protocol tarball to GitHub Release").run.replaceAll( + "${{ steps.release-artifacts.outputs.version }}", + version, + ), + ], + { REVIEWS: approval() }, + ); + assert.equal(r.status, 0, r.stderr); + const upload = f.upload(immutable); + assert.equal(upload.status, 0, upload.stderr); + const calls = f.calls().trim().split("\n"); + assert.equal(calls[0], "github draft"); + assert.equal(calls[5], "github publish"); + assert.equal(calls.filter((c) => c.startsWith("r2 put ")).length, 6); + assert.ok(calls.slice(6).every((c) => c.includes(version))); + assert.equal(f.upload(immutable).status, 0, "identical retries skip objects"); + assert.equal(f.calls(), calls.join("\n") + "\n"); +}); +for (const [name, env] of [ + ["drift during put", { ADVANCE_AWS: "1" }], + ["conditional create race", { PUT_RACE: "1" }], + ["head uncertainty", { HEAD_ERROR: "1" }], +]) + test(`immutable publication stops on ${name}`, (t) => { + const f = fixture(t); + assert.notEqual(f.upload(immutable, env).status, 0); + assert.equal( + f.calls().trim().split("\n").filter(Boolean).length, + env.ADVANCE_AWS ? 1 : 0, + ); + }); +test("existing different bytes cannot be overwritten", (t) => { + const f = fixture(t); + f.write(`bucket/schemas/${version}/index.json`, "different"); + assert.notEqual(f.upload(immutable).status, 0); + assert.equal(f.calls(), ""); +}); +test("main advancing during GitHub staging leaves a draft and publishes no R2 objects", (t) => { + const f = fixture(t); + f.write("tag", ""); + fs.unlinkSync(path.join(f.dir, "release.json")); + const r = f.run( + "bash", + [ + "-c", + step("Upload protocol tarball to GitHub Release").run.replaceAll( + "${{ steps.release-artifacts.outputs.version }}", + version, + ), + ], + { ADVANCE_GITHUB: "1" }, + ); + assert.notEqual(r.status, 0); + assert.equal( + JSON.parse(fs.readFileSync(path.join(f.dir, "release.json"))).isDraft, + true, + ); + assert.ok(!f.calls().includes("github publish")); +}); +test("workflow wiring binds tested SHA, approval, exact recovery, and publication order", () => { + const ds = deploy.jobs.deploy.steps; + assert.ok( + ds.findIndex( + (s) => s.name === "Require committed version to have a complete release", + ) < ds.findIndex((s) => s.name === "Deploy"), + ); + const upload = ds.find((s) => s.name === "Publish latest artifacts to R2"); + assert.match(upload.run, /--latest-only/); + assert.equal( + upload.env.TESTED_SHA, + "${{ github.event.workflow_run.head_sha }}", + ); + assert.ok( + steps.indexOf( + step("Require human approval for committed release artifacts"), + ) < steps.indexOf(step("Upload protocol tarball to GitHub Release")), + ); + assert.ok( + steps.indexOf(step("Upload protocol tarball to GitHub Release")) < + steps.indexOf(step("Publish release artifacts to R2")), + ); + assert.equal( + step("Create Release Pull Request or Tag Release").with["publish-script"], + undefined, + ); + assert.equal( + step("Create Release Pull Request or Tag Release").with[ + "create-github-releases" + ], + false, + ); + for (const name of [ + "Upload protocol tarball to GitHub Release", + "Publish release artifacts to R2", + ]) + assert.equal( + step(name).if, + "steps.release-artifacts.outputs.has_release_artifacts == 'true'", + ); + assert.match( + step("Publish release artifacts to R2").run, + /--version .* --skip-latest/, + ); + assert.match( + step("Detect committed release artifacts").run, + /check-release-state.cjs recovery/, + ); +}); + +test("#7508: stale sole-parent release omits a later changeset and rewrites rc.3", (t) => { + // Incident: 102916bb28c156b1710f031aa28e4655bcf71cdc / tree + // fd299f52f6bf1a4427ef85f890832d1e5713aade had sole parent eb3cbd, + // two commits behind af1. This fixture reproduces that graph and delta. + const f = fixture(t); + const oldInput = f.source; + f.write("app-only.txt", "new main behavior"); + f.git("add", "app-only.txt"); + f.git("commit", "-qm", "Second main advance"); + const current = f.git("rev-parse", "HEAD"); + f.write("remote", current); + f.git("checkout", "-qb", "stale-release", oldInput); + f.write( + `dist/schemas/${version}/index.json`, + "different generated rc.3 bytes", + ); + f.git("add", `dist/schemas/${version}/index.json`); + f.git("commit", "-qm", "Stale generated release"); + const stale = f.git("rev-parse", "HEAD"); + assert.equal(f.git("rev-parse", "HEAD^"), oldInput); + assert.equal(f.git("merge-base", stale, current), oldInput); + assert.equal(f.git("rev-list", "--count", `${oldInput}..${current}`), "2"); + const delta = f.git("diff", "--name-status", current, stale); + assert.match(delta, /M\s+dist\/schemas\/3\.2\.0-rc\.3\/index.json/); + assert.match(delta, /D\s+\.changeset\/next.md/); + const staleEnvironment = { TESTED_SHA: oldInput, GITHUB_SHA: oldInput }; + assert.notEqual(f.script("current", staleEnvironment).status, 0); + assert.notEqual(f.upload(immutable, staleEnvironment).status, 0); + assert.notEqual( + f.run("node", ["scripts/check-release-state.cjs", "recovery", stale], { + TESTED_SHA: current, + }).status, + 0, + ); + assert.equal(f.calls(), ""); +}); diff --git a/tests/release-workflow-immutability.test.cjs b/tests/release-workflow-immutability.test.cjs index bcf15869d1d..ca8acbcdd52 100644 --- a/tests/release-workflow-immutability.test.cjs +++ b/tests/release-workflow-immutability.test.cjs @@ -241,7 +241,7 @@ assert( ); assert( - approvalGate.includes('/commits/${GITHUB_SHA}/pulls') && + approvalGate.includes('/commits/${RELEASE_SHA}/pulls') && approvalGate.includes('.base.ref == $base') && approvalGate.includes('.merged_at != null'), 'The approval gate must resolve the merged PR associated with the release commit and branch.' @@ -281,10 +281,9 @@ assert.deepStrictEqual( with: { 'github-token': '${{ steps.app-token.outputs.token }}', 'version-script': 'npm run version', - 'publish-script': 'npx --no-install changeset git-tag', 'commit-message': 'Version Packages', 'pr-title': 'Version Packages', - 'create-github-releases': true, + 'create-github-releases': false, 'push-with-git-cli': true, }, env: { @@ -359,3 +358,6 @@ for (const { line, source } of forwardMergeWorkflows) { } console.log('Release and forward-merge workflow checks passed.'); + +// Keep the behavioral publication regressions in the existing release gate. +require('./release-publication-ordering.test.cjs'); diff --git a/workers/artifact-cdn/README.md b/workers/artifact-cdn/README.md index c5416272d8b..b4e1241df8d 100644 --- a/workers/artifact-cdn/README.md +++ b/workers/artifact-cdn/README.md @@ -28,17 +28,16 @@ workflows keep the bucket fresh: Automation after these are set: -- `release.yml` uploads the newly published versioned artifacts after a real - Changesets publish. Main-line releases also update mutable `latest`; release - branches such as `3.0.x` use `--skip-latest` so they cannot move global - `latest` backward. +- `release.yml` uploads only the approved version after publishing its signed + GitHub Release, using `--version VERSION --skip-latest`. See `RELEASING.md` + for freshness fences, progressive visibility, and explicit recovery. - `deploy.yml` rebuilds and uploads mutable `latest` artifacts after the Fly deploy, machine-image check, tenant smoke, and console cleanup all pass. 1. Refresh mutable artifacts in R2. ```sh - npm run backfill:cdn-artifacts -- --bucket adcp-artifacts --quiet + npm run backfill:cdn-artifacts -- --bucket adcp-artifacts --latest-only --quiet ``` 2. Verify the shadow Worker against current production. From f99f7698b64f6c7bd59067ede06894711fcda845 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 13:39:52 +0000 Subject: [PATCH 2/9] fix(release): validate complete publication authority --- .github/workflows/release.yml | 9 +- RELEASING.md | 10 +- scripts/check-release-state.cjs | 48 ++++++- tests/release-publication-ordering.test.cjs | 131 +++++++++++++++++++- 4 files changed, 189 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4d42c544c1..c6654e2b618 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -338,12 +338,19 @@ jobs: "${release_args[@]}" fi - release_state="$(gh release view "$TAG" --json isDraft,targetCommitish)" + release_state="$(gh release view "$TAG" --json isDraft,isPrerelease,targetCommitish)" if [ "$(jq -r '.isDraft' <<< "$release_state")" = true ] && [ "$(jq -r '.targetCommitish' <<< "$release_state")" != "$RELEASE_SHA" ]; then echo '::error::Existing draft targets a different release commit.' exit 1 fi + expected_prerelease=false + if [[ "$VERSION" == *-* ]]; then expected_prerelease=true; fi + if [ "$(jq -r '.isPrerelease' <<< "$release_state")" != "$expected_prerelease" ]; then + echo '::error::Existing release prerelease status conflicts with the version.' + exit 1 + fi + upload_asset_if_missing() { local path="$1" local name diff --git a/RELEASING.md b/RELEASING.md index 952e01a3e54..6bd03415271 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -273,8 +273,10 @@ For the 2026-09-14 `3.2.0-rc.3` incident, the recovery target is `e7d42cbe5d7b3d05c3fcdcaa6f568b08ba131439` received human approval at 12:34:18 UTC. Release run `34844149713` was cancelled with zero jobs. Deploy `34845112730` exposed its R2 objects before a tag/GitHub Release; the independent -incident audit found 2,092 matching objects of 2,093 committed paths. The JSONL -workstream owns the omitted path/filter and full enumeration verification. +incident audit found 2,092 matching schema/compliance objects of 2,093 paths +(1,612 schemas and 481 compliance files). Four protocol files bring the full +release inventory to 2,097. The JSONL workstream owns the omitted path/filter +and schema/compliance enumeration verification. This version is externally exposed; absence of a GitHub Release is not permission to edit or retire its bytes. @@ -312,8 +314,8 @@ The exact recovery sequence, requiring separate maintainer authorization, is: publishes `v3.2.0-rc.3` at **71f**, and conditionally fills missing rc.3 R2 objects. It does not tag the later tested main or publish `latest`. 4. Independently verify the tag target, prerelease flag, all four GitHub assets, - full 2,093-path CDN inventory and byte hashes (including JSONL), checksum and - Sigstore identity. Check deployment and docs/SDK readiness separately. If + all 2,093 schema/compliance paths (including JSONL) plus the four protocol + files, byte hashes, checksum and Sigstore identity (2,097 files total). Check deployment and docs/SDK readiness separately. If anything fails or main advances, record the partial state and repeat only through explicit recovery from the next verified current head. diff --git a/scripts/check-release-state.cjs b/scripts/check-release-state.cjs index 14367f8c221..fd8c202c727 100644 --- a/scripts/check-release-state.cjs +++ b/scripts/check-release-state.cjs @@ -2,6 +2,8 @@ // Release/deploy fences. Remote failures are errors, never evidence of absence. const { execFileSync } = require("node:child_process"); const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); const run = (command, args) => execFileSync(command, args, { @@ -69,12 +71,19 @@ function published( if (git("diff", "--name-only", target, "--", ...paths)) throw new Error("Local release artifacts differ from the published tag."); const release = JSON.parse( - run("gh", ["release", "view", tag, "--json", "tagName,isDraft,assets"]), + run("gh", [ + "release", + "view", + tag, + "--json", + "tagName,isDraft,isPrerelease,assets", + ]), ); const names = release.assets.map((asset) => asset.name); if ( release.tagName !== tag || release.isDraft || + release.isPrerelease !== version.includes("-") || !["", ".sha256", ".sig", ".crt"].every((suffix) => names.includes(`${version}.tgz${suffix}`), ) @@ -83,6 +92,34 @@ function published( `Release ${tag} is not published with the complete signed tuple.`, ); } + // Names/digests alone do not prove that recovery preserves the published + // tuple. Compare all four assets, including signature and certificate bytes. + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "adcp-release-")); + try { + for (const suffix of ["", ".sha256", ".sig", ".crt"]) { + const name = `${version}.tgz${suffix}`; + git("cat-file", "-e", `${target}:dist/protocol/${name}`); + run("gh", [ + "release", + "download", + tag, + "--pattern", + name, + "--dir", + temporary, + ]); + try { + run("cmp", ["-s", `dist/protocol/${name}`, path.join(temporary, name)]); + } catch { + throw new Error( + `GitHub release asset ${name} differs from the tagged local tuple.`, + ); + } + } + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } + current(); } function recovery(source) { @@ -116,7 +153,14 @@ try { } else if (mode === "pending") { current(); const version = JSON.parse(fs.readFileSync("package.json")).version; - if (fs.existsSync(`dist/protocol/${version}.tgz`)) published(version); + const surfaces = [ + `dist/schemas/${version}`, + `dist/compliance/${version}`, + ...["", ".sha256", ".sig", ".crt"].map( + (suffix) => `dist/protocol/${version}.tgz${suffix}`, + ), + ]; + if (surfaces.some((surface) => fs.existsSync(surface))) published(version); } else if (mode === "recovery") recovery(argument); else throw new Error("Expected current, published, pending, or recovery."); } catch (error) { diff --git a/tests/release-publication-ordering.test.cjs b/tests/release-publication-ordering.test.cjs index f9b05b033f2..11b489e00fc 100644 --- a/tests/release-publication-ordering.test.cjs +++ b/tests/release-publication-ordering.test.cjs @@ -17,7 +17,8 @@ const step = (name) => steps.find((s) => s.name === name); const version = "3.2.0-rc.3"; const realGit = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); -function fixture(t) { +function fixture(t, fixtureVersion = version) { + const version = fixtureVersion; const dir = fs.mkdtempSync(path.join(os.tmpdir(), "publication-order-")); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); const write = (file, value) => { @@ -62,6 +63,7 @@ function fixture(t) { tagName: `v${version}`, targetCommitish: source, isDraft: false, + isPrerelease: version.includes("-"), assets: ["", ".sha256", ".sig", ".crt"].map((s) => ({ name: `${version}.tgz${s}`, })), @@ -76,6 +78,7 @@ function fixture(t) { "git", common + ` + if(args[0]==='push' && process.env.ADVANCE_PUSH) { log('branch push');change();process.exit(0); } if(args[0]==='ls-remote') { if(process.env.REMOTE_ERROR) process.exit(1); const ref=args.find(a=>a.startsWith('refs/heads/')); @@ -104,7 +107,7 @@ function fixture(t) { const q=args[args.indexOf('--jq')+1];const name=q.match(/name == "([^"]+)"/)[1];console.log(r.assets.find(a=>a.name===name)?.name||''); } else console.log(JSON.stringify(r)); } else if(args[1]==='create') { - log('github draft'); save({tagName:args[2],targetCommitish:process.env.RELEASE_SHA,isDraft:true,assets:[]}); + log('github draft'); save({tagName:args[2],targetCommitish:process.env.RELEASE_SHA,isDraft:true,isPrerelease:args.includes('--prerelease'),assets:[]}); } else if(args[1]==='upload') { const r=release(); if(!r.isDraft) throw Error('upload was publicly visible before staging completed'); const name=path.basename(args[3]);r.assets.push({name});save(r);log('github upload '+name); @@ -112,6 +115,7 @@ function fixture(t) { } else if(args[1]==='download') { const name=args[args.indexOf('--pattern')+1],dir=args[args.indexOf('--dir')+1]; fs.copyFileSync('dist/protocol/'+name,path.join(dir,name)); + if(process.env.DIFFERING_GITHUB_ASSET===name) fs.appendFileSync(path.join(dir,name),'different'); } else if(args[1]==='edit') { const r=release();if(r.assets.length!==4) throw Error('incomplete release');r.isDraft=false;save(r);fs.writeFileSync('tag',process.env.RELEASE_SHA);log('github publish'); } else throw Error('unexpected gh '+args); @@ -449,3 +453,126 @@ test("#7508: stale sole-parent release omits a later changeset and rewrites rc.3 ); assert.equal(f.calls(), ""); }); + +for (const surface of ["schemas", "compliance", "partial-protocol"]) { + test(`the app gate rejects an unapproved ${surface}-only committed release`, (t) => { + const f = fixture(t); + for (const area of ["schemas", "compliance", "protocol"]) { + fs.rmSync(path.join(f.dir, "dist", area), { recursive: true }); + } + const remaining = + surface === "partial-protocol" + ? `dist/protocol/${version}.tgz.sig` + : `dist/${surface}/${version}/index.json`; + f.write(remaining, "partial candidate"); + f.write("tag", ""); + const result = f.script("pending"); + assert.notEqual(result.status, 0); + assert.equal(f.calls(), ""); + }); +} + +for (const candidate of ["3.2.0-rc.3", "3.1.23"]) { + for (const draft of [true, false]) { + test(`rejects wrong prerelease metadata for ${candidate} (draft=${draft})`, (t) => { + const f = fixture(t, candidate); + const record = JSON.parse( + fs.readFileSync(path.join(f.dir, "release.json")), + ); + record.isPrerelease = !candidate.includes("-"); + record.isDraft = draft; + f.write("release.json", JSON.stringify(record)); + const result = f.run("bash", [ + "-c", + step("Upload protocol tarball to GitHub Release").run.replaceAll( + "${{ steps.release-artifacts.outputs.version }}", + candidate, + ), + ]); + assert.notEqual(result.status, 0); + assert.notEqual( + f.upload(["--version", candidate, "--skip-latest"]).status, + 0, + ); + assert.equal(f.calls(), ""); + }); + } +} + +for (const suffix of ["", ".sha256", ".sig", ".crt"]) { + test(`direct/stable R2 recovery rejects different GitHub ${suffix || "tarball"} bytes`, (t) => { + const f = fixture(t, "3.1.23"); + const result = f.upload(["--version", "3.1.23", "--skip-latest"], { + DIFFERING_GITHUB_ASSET: `3.1.23.tgz${suffix}`, + }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /differs from the tagged local tuple/); + assert.equal(f.calls(), ""); + }); +} + +test("a successful long Changesets push followed by drift stops before the PR API call", (t) => { + const f = fixture(t); + const installed = f.run("bash", [ + "-c", + step("Fence Changesets mutations").run, + ]); + assert.equal(installed.status, 0, installed.stderr); + // GITHUB_PATH is applied to the next action exactly as Actions applies it. + const wrapperDirectory = fs + .readFileSync(path.join(f.dir, "github-path"), "utf8") + .trim(); + const result = f.run( + "bash", + [ + "-c", + 'set -e\ngit push origin HEAD:changeset-release/main --force\nprintf "PR API write\\n" >> calls', + ], + { + PATH: `${wrapperDirectory}:${path.join(f.dir, "bin")}:${process.env.PATH}`, + ADVANCE_PUSH: "1", + }, + ); + assert.notEqual(result.status, 0); + assert.equal(f.calls(), "branch push\n"); +}); + +for (const mismatch of [false, true]) { + test(`stable recovery workflow validates GitHub bytes before R2 (mismatch=${mismatch})`, (t) => { + const f = fixture(t, "3.1.23"); + const workflow = YAML.parse( + fs.readFileSync( + path.join(root, ".github/workflows/recover-protocol-cdn.yml"), + "utf8", + ), + ); + f.write( + "temp/check-release-state.cjs", + fs.readFileSync(path.join(f.dir, "scripts/check-release-state.cjs")), + ); + f.git("checkout", "--detach", f.source); + const upload = workflow.jobs.recover.steps.find( + (s) => s.name === "Upload only missing immutable R2 objects", + ); + const result = f.run("bash", ["-c", upload.run], { + VERSION: "3.1.23", + BUCKET: "test", + R2_ACCOUNT_ID: "test", + ...(mismatch ? { DIFFERING_GITHUB_ASSET: "3.1.23.tgz.sig" } : {}), + }); + if (mismatch) { + assert.notEqual(result.status, 0); + assert.equal(f.calls(), ""); + } else { + assert.equal(result.status, 0, result.stderr); + assert.equal(f.calls().trim().split("\n").length, 4); + assert.ok( + f + .calls() + .trim() + .split("\n") + .every((line) => line.startsWith("r2 put protocol/3.1.23.tgz")), + ); + } + }); +} From 36140697ee0421859568f42b8bb867cf6ff3762b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 13:47:09 +0000 Subject: [PATCH 3/9] fix(ci): classify release orchestration as operational --- scripts/check-changeset-protocol-scope.cjs | 3 ++- tests/changeset-protocol-scope.test.cjs | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/check-changeset-protocol-scope.cjs b/scripts/check-changeset-protocol-scope.cjs index e3b387809f9..f89dedcff1a 100644 --- a/scripts/check-changeset-protocol-scope.cjs +++ b/scripts/check-changeset-protocol-scope.cjs @@ -27,7 +27,8 @@ const PROTOCOL_SCOPED_PATHS = [ /^dist\/protocol\/[^/]+[.]tgz(?:[.](?:sha256|sig|crt))?$/, /^scripts\/(?:build-schemas|build-compliance|build-protocol-tarball|sign-protocol-tarball|update-schema-versions|verify-version-sync|patch-3-0-compat-bundle)[.](?:cjs|mjs|sh)$/, /^scripts\/run-storyboards-(?:[^/]+[.]sh|isolated[.]mjs)$/, - /^[.]github\/workflows\/(?:release|training-agent-storyboards)[.]yml$/, + // Publication orchestration is operational; protocol generators above remain scoped. + /^[.]github\/workflows\/training-agent-storyboards[.]yml$/, ]; const CHANGESET_POLICY_CODE_PATHS = new Set([ diff --git a/tests/changeset-protocol-scope.test.cjs b/tests/changeset-protocol-scope.test.cjs index b19f08fd104..26d0e756125 100644 --- a/tests/changeset-protocol-scope.test.cjs +++ b/tests/changeset-protocol-scope.test.cjs @@ -75,6 +75,27 @@ assert.strictEqual(isProtocolScopedPath('docs/registry/index.mdx'), false); assert.strictEqual(isProtocolScopedPath('scripts/run-storyboards-isolated.mjs'), true); assert.strictEqual(isProtocolScopedPath('server/src/billing/subscription-sync.ts'), false); assert.strictEqual(isProtocolScopedPath('.changeset/billing-fix.md'), false); +assert.strictEqual(isProtocolScopedPath('.github/workflows/release.yml'), false); +assert.strictEqual(isProtocolScopedPath('.github/workflows/training-agent-storyboards.yml'), true); +assert.strictEqual(isProtocolScopedPath('scripts/build-protocol-tarball.cjs'), true); +assert.strictEqual( + hasProtocolScopedChanges([ + { status: 'M', paths: ['.github/workflows/release.yml'] }, + { status: 'M', paths: ['scripts/backfill-cdn-artifacts.sh'] }, + { status: 'A', paths: ['scripts/check-release-state.cjs'] }, + ]), + false, + 'Release publication controls do not schedule a protocol release' +); +assert.strictEqual( + hasProtocolScopedChanges([ + { status: 'M', paths: ['.github/workflows/release.yml'] }, + { status: 'M', paths: ['static/schemas/source/core/product.json'] }, + ]), + true, + 'A publication workflow change must not exempt accompanying protocol content' +); + assert.strictEqual( hasProtocolScopedChanges([{ status: 'M', paths: ['server/src/billing/subscription-sync.ts'] }]), false, From 1d599c43a0ffab8aa18aceedec4f3d924154d69e Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 14:01:02 +0000 Subject: [PATCH 4/9] fix(test): register publication ordering suite directly --- package.json | 2 +- tests/release-workflow-immutability.test.cjs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/package.json b/package.json index be6bd6c2f15..a32c3d7672b 100644 --- a/package.json +++ b/package.json @@ -121,7 +121,7 @@ "test:build-schemas-hoist-marked": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-hoist-marked.test.cjs", "test:build-schemas-async-response-refs": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-async-response-refs.test.cjs", "test:build-schemas-vendor-metric-uniqueness": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-vendor-metric-uniqueness.test.cjs", - "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs", + "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs", "promote:rc": "node scripts/promote-release-candidate.mjs", "test:immutable-release-artifacts": "node tests/immutable-release-artifacts.test.cjs", "test:patch-3-0-compat-bundle": "node --test --test-force-exit --test-timeout=30000 tests/patch-3-0-compat-bundle.test.cjs", diff --git a/tests/release-workflow-immutability.test.cjs b/tests/release-workflow-immutability.test.cjs index ca8acbcdd52..f181b0debb6 100644 --- a/tests/release-workflow-immutability.test.cjs +++ b/tests/release-workflow-immutability.test.cjs @@ -358,6 +358,3 @@ for (const { line, source } of forwardMergeWorkflows) { } console.log('Release and forward-merge workflow checks passed.'); - -// Keep the behavioral publication regressions in the existing release gate. -require('./release-publication-ordering.test.cjs'); From 01125c15c0fed4af01aaf22752f5aa5c36aad46d Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 17:34:49 +0000 Subject: [PATCH 5/9] test(release): prove historical artifacts stay outside routine deploy --- RELEASING.md | 9 ++++++ tests/release-publication-ordering.test.cjs | 31 +++++++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 6bd03415271..12f3374bb24 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -204,6 +204,15 @@ commands. ### Publication authority and ordering +For the 2026-09-14 incident, the publication fence in +[#7510](https://github.com/adcontextprotocol/adcp/pull/7510) is a hard +predecessor of the JSONL filter fix in +[#7509](https://github.com/adcontextprotocol/adcp/pull/7509). The legacy deploy +bulk upload would otherwise publish historical JSONL without recovery approval. +Land the human-approved fence first; then rebase and requalify the JSONL fix on +that main, including combined tests and human review. Neither merge authorizes +historical backfill or incident recovery. + Routine app deployment is authorized to rebuild and upload only `schemas/latest/**`, `compliance/latest/**`, and `protocol/latest.tgz*`. It uses `backfill-cdn-artifacts.sh --latest-only`; root version indexes and diff --git a/tests/release-publication-ordering.test.cjs b/tests/release-publication-ordering.test.cjs index 11b489e00fc..2155a542859 100644 --- a/tests/release-publication-ordering.test.cjs +++ b/tests/release-publication-ordering.test.cjs @@ -15,9 +15,10 @@ const deploy = YAML.parse( const steps = release.jobs.release.steps; const step = (name) => steps.find((s) => s.name === name); const version = "3.2.0-rc.3"; +const rowsPath = "test-vectors/reporting-reconciliation/rows.jsonl"; const realGit = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); -function fixture(t, fixtureVersion = version) { +function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { const version = fixtureVersion; const dir = fs.mkdtempSync(path.join(os.tmpdir(), "publication-order-")); t.after(() => fs.rmSync(dir, { recursive: true, force: true })); @@ -38,6 +39,16 @@ function fixture(t, fixtureVersion = version) { } for (const suffix of ["", ".sha256", ".sig", ".crt"]) write(`dist/protocol/${version}.tgz${suffix}`, `signed${suffix}`); + if (includeReleaseHistory) { + for (let rc = 0; rc <= 3; rc++) { + const historical = `3.2.0-rc.${rc}`; + write(`dist/schemas/${historical}/index.json`, '{"release":true}'); + write(`dist/compliance/${historical}/${rowsPath}`, '{"immutable":true}\n'); + for (const suffix of ["", ".sha256", ".sig", ".crt"]) + write(`dist/protocol/${historical}.tgz${suffix}`, `signed${suffix}`); + } + write(`dist/compliance/latest/${rowsPath}`, '{"development":true}\n'); + } write("dist/protocol/latest.tgz", "development"); for (const script of ["check-release-state.cjs", "backfill-cdn-artifacts.sh"]) write( @@ -195,12 +206,22 @@ const approval = (extra) => }, ]); -test("app-only deploy publishes only development keys with no human release approval", (t) => { - const f = fixture(t); - const result = f.upload(["--latest-only"]); +test("app-only deploy cannot backfill immutable JSONL or historical rc.0–rc.3 artifacts", (t) => { + const f = fixture(t, version, true); + for (let rc = 0; rc <= 3; rc++) { + const file = `dist/compliance/3.2.0-rc.${rc}/${rowsPath}`; + assert.equal(f.git("ls-files", file), file); + } + // Main's pre-fence deploy invocation must fail before any upload. + const bulk = f.upload(["--build-latest"]); + assert.notEqual(bulk.status, 0); + assert.match(bulk.stderr, /Live bulk backfill is disabled/); + assert.equal(f.calls(), ""); + // The destination boundary holds independently of #7509's JSONL filter. + const result = f.upload(["--latest-only", "--build-latest"]); assert.equal(result.status, 0, result.stderr); assert.ok(f.calls().includes("schemas/latest")); - assert.ok(!f.calls().includes(version)); + assert.doesNotMatch(f.calls(), /3\.2\.0-rc\.[0-3]|r2 put /); assert.ok( f .calls() From ecccbac7d85df60451d3f2df5d902e34e4293d3e Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 18:23:11 +0000 Subject: [PATCH 6/9] fix(release): require maintainer authority and quarantine leased updates --- .github/workflows/release.yml | 108 ++- RELEASING.md | 110 ++- package.json | 2 +- scripts/check-release-state.cjs | 186 +++- scripts/fence-changesets.cjs | 485 +++++++++++ tests/changesets-fence.test.cjs | 813 ++++++++++++++++++ .../changesets-action-v2.1.2/github-push.ts | 18 + tests/release-publication-ordering.test.cjs | 458 +++++++++- tests/release-workflow-immutability.test.cjs | 23 +- 9 files changed, 2072 insertions(+), 131 deletions(-) create mode 100644 scripts/fence-changesets.cjs create mode 100644 tests/changesets-fence.test.cjs create mode 100644 tests/fixtures/changesets-action-v2.1.2/github-push.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6654e2b618..19b9d28f9f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -180,75 +180,56 @@ jobs: echo "No committed release artifacts detected for ${VERSION}." fi - # Branch rules can be bypassed by repository administrators and GitHub - # Apps. Before publishing committed artifacts, independently verify that - # the merged release PR received a human approval on its final head SHA. + # Public human reviews are not publication authority. Recheck current + # collaborator access and the reviewed head's exact merge provenance, + # independently of branch-rule bypass or non-strict required checks. - name: Require human approval for committed release artifacts if: steps.release-artifacts.outputs.has_release_artifacts == 'true' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: | - set -euo pipefail - - pulls="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - "/repos/${GITHUB_REPOSITORY}/commits/${RELEASE_SHA}/pulls")" - pr_number="$(jq -r \ - --arg base "${GITHUB_REF_NAME}" \ - '[.[] | select(.merged_at != null and .base.ref == $base)] | sort_by(.merged_at) | last | .number // empty' \ - <<< "${pulls}")" - - if [ -z "${pr_number}" ]; then - echo "::error::Release artifact commit ${RELEASE_SHA} is not associated with a merged PR targeting ${GITHUB_REF_NAME}." - exit 1 - fi - - pr="$(gh api "/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}")" - head_sha="$(jq -r '.head.sha' <<< "${pr}")" - test "$(jq -r '.merge_commit_sha' <<< "$pr")" = "$RELEASE_SHA" - author="$(jq -r '.user.login' <<< "${pr}")" - reviews="$(gh api --paginate "/repos/${GITHUB_REPOSITORY}/pulls/${pr_number}/reviews" | jq -s 'add')" - approved_count="$(jq \ - --arg head "${head_sha}" \ - '[.[] | select(.user.type == "User")] - | sort_by(.user.login, .submitted_at) - | group_by(.user.login) - | map(last) - | map(select(.state == "APPROVED" and .commit_id == $head)) - | length' \ - <<< "${reviews}")" - - if [ "${approved_count}" -lt 1 ]; then - echo "::error::Release PR #${pr_number} by ${author} has no human approval on final head ${head_sha}. Refusing to publish artifacts." - exit 1 - fi + run: node scripts/check-release-state.cjs approval - echo "Release PR #${pr_number} has ${approved_count} human approval(s) on final head ${head_sha}." + - name: Mint Changesets mutation token + id: changesets-token + if: steps.release-artifacts.outputs.has_release_artifacts == 'false' + uses: actions/create-github-app-token@v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: adcontextprotocol + repositories: adcp - name: Fence Changesets mutations if: steps.release-artifacts.outputs.has_release_artifacts != 'true' + timeout-minutes: 10 env: - GH_TOKEN: ${{ steps.app-token.outputs.token }} + GH_TOKEN: ${{ steps.changesets-token.outputs.token }} run: | set -euo pipefail - node scripts/check-release-state.cjs pending # Changesets builds for minutes before pushing. Its pinned git-CLI # path honors pre-push and propagates a failed push before PR writes. mkdir -p "$RUNNER_TEMP/release-hooks" cp scripts/check-release-state.cjs "$RUNNER_TEMP/release-hooks/check.cjs" + cp scripts/fence-changesets.cjs "$RUNNER_TEMP/release-hooks/fence.cjs" + command -v git > "$RUNNER_TEMP/release-hooks/git-path" + node "$RUNNER_TEMP/release-hooks/fence.cjs" prepare # shellcheck disable=SC2016 printf '#!/bin/sh\nnode "$RUNNER_TEMP/release-hooks/check.cjs" current\n' > "$RUNNER_TEMP/release-hooks/pre-push" chmod +x "$RUNNER_TEMP/release-hooks/pre-push" git config core.hooksPath "$RUNNER_TEMP/release-hooks" - # Also recheck after a long git push, so Changesets stops before its - # subsequent GitHub PR API request if main moved during the push. - command -v git > "$RUNNER_TEMP/release-hooks/git-path" + # Only the pinned action's push shape may mutate the captured ref. + # The helper leases it, then restores/quarantines on drift or error. cat > "$RUNNER_TEMP/release-hooks/git" <<'SH' #!/bin/sh real_git=$(cat "$RUNNER_TEMP/release-hooks/git-path") + # Permit only the pinned producer's two read-only identity probes. + # Other global options must not route around the leased push path. + if [ "$#" -eq 4 ] && [ "$1" = -c ] && [ "$2" = user.useConfigOnly=true ] && [ "$3" = var ]; then + case "$4" in GIT_AUTHOR_IDENT|GIT_COMMITTER_IDENT) exec "$real_git" "$@";; esac + fi + case "$1" in -*) echo 'Unsupported Git global options in release action' >&2; exit 1;; esac if [ "$1" = push ]; then - "$real_git" "$@" || exit $? - node "$RUNNER_TEMP/release-hooks/check.cjs" current + exec node "$RUNNER_TEMP/release-hooks/fence.cjs" push "$@" else exec "$real_git" "$@" fi @@ -259,6 +240,7 @@ jobs: - name: Create Release Pull Request or Tag Release if: steps.release-artifacts.outputs.has_release_artifacts != 'true' id: changesets + timeout-minutes: 40 # v2 understands the Changesets v3 `.changeset/pre/` archive. This is # v2.1.2, pinned to the contract verified at: # https://github.com/changesets/action/blob/ae32849d5ba541f9ae29e40e22a623bc13562f51/action.yml @@ -267,14 +249,44 @@ jobs: # compliance artifacts exceed GitHub's API payload limit. uses: changesets/action@ae32849d5ba541f9ae29e40e22a623bc13562f51 # v2.1.2 with: - github-token: ${{ steps.app-token.outputs.token }} + github-token: ${{ steps.changesets-token.outputs.token }} version-script: npm run version commit-message: 'Version Packages' pr-title: 'Version Packages' + pr-draft: always create-github-releases: false push-with-git-cli: true env: HUSKY: '0' + GH_TOKEN: ${{ steps.changesets-token.outputs.token }} + + - name: Mint Changesets reconciliation token + id: reconciliation-token + if: always() && steps.release-artifacts.outputs.has_release_artifacts == 'false' && steps.changesets-token.outcome == 'success' + uses: actions/create-github-app-token@v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: adcontextprotocol + repositories: adcp + + - name: Reconcile Changesets transaction + if: always() && steps.release-artifacts.outputs.has_release_artifacts == 'false' + env: + GH_TOKEN: ${{ steps.reconciliation-token.outputs.token }} + CHANGESETS_OUTCOME: ${{ steps.changesets.outcome }} + run: | + if [ -f "$RUNNER_TEMP/release-hooks/state.json" ]; then + node "$RUNNER_TEMP/release-hooks/fence.cjs" finalize + fi + + - name: Retain Changesets reconciliation evidence + if: always() && steps.release-artifacts.outputs.has_release_artifacts == 'false' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: changesets-transaction-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/release-hooks/state.json + if-no-files-found: ignore - name: Require committed signed protocol tarball if: steps.release-artifacts.outputs.has_release_artifacts == 'true' @@ -380,6 +392,8 @@ jobs: upload_asset_if_missing "${TARBALL}.sha256" upload_asset_if_missing "${TARBALL}.sig" upload_asset_if_missing "${TARBALL}.crt" + gh release view "$TAG" --json assets | jq -e --arg v "$VERSION" \ + '([.assets[].name] | sort) == (["", ".sha256", ".sig", ".crt"] | map($v + ".tgz" + .) | sort)' >/dev/null node scripts/check-release-state.cjs current gh release edit "$TAG" --draft=false node scripts/check-release-state.cjs published "$VERSION" diff --git a/RELEASING.md b/RELEASING.md index 12f3374bb24..94e91aa4abc 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -206,11 +206,12 @@ commands. For the 2026-09-14 incident, the publication fence in [#7510](https://github.com/adcontextprotocol/adcp/pull/7510) is a hard -predecessor of the JSONL filter fix in -[#7509](https://github.com/adcontextprotocol/adcp/pull/7509). The legacy deploy -bulk upload would otherwise publish historical JSONL without recovery approval. -Land the human-approved fence first; then rebase and requalify the JSONL fix on -that main, including combined tests and human review. Neither merge authorizes +predecessor of both the JSONL filter fix in +[#7509](https://github.com/adcontextprotocol/adcp/pull/7509) and the curated +3.1.22/3.1.23 artifacts in [#7507](https://github.com/adcontextprotocol/adcp/pull/7507). +The legacy deploy bulk upload would otherwise expose those historical artifacts +without recovery approval. Land the human-approved fence first; then rebase and +requalify each dependent PR on that main, including combined tests and human review. Neither merge authorizes historical backfill or incident recovery. Routine app deployment is authorized to rebuild and upload only @@ -222,8 +223,13 @@ assets also blocks app deployment. This can delay app-only fixes while a release needs recovery. Only the release workflow publishes a new immutable version. It verifies the -current tested branch, the original merged release PR's final-head human -approval, and the committed signed tuple. It stages the four GitHub assets in +current tested branch, the original merged release PR's final-head maintainer +approval, and the committed signed tuple. Approval must come from a non-author +human with current repository write, maintain, or admin permission. Every +candidate approver is checked through GitHub's collaborator-permission API; +missing, failed, ambiguous, or unsupported permission responses stop publication. +Public review ability and author association are not sufficient authority. +It stages the four GitHub assets in a draft, publishes the tag/release targeting the original release merge, then uploads that version alone with `--version VERSION --skip-latest`. Changesets only prepares release PRs; it has no independent tag/publish command. Missing @@ -233,10 +239,47 @@ Release runs use `queue: max` without in-progress cancellation. GitHub permits 100 pending runs; this is not an unlimited or guaranteed delivery queue. Old runs fail their current-branch fence. A later push with a stranded committed version fails explicitly, including an app-only push, and must use the recovery -procedure below before Changesets advances the package again. The git pre-push -hook rechecks freshness after Changesets' long version/build/sign operation, -before its branch push. A wrapper checks again after the push, before the -subsequent PR write. +procedure below before Changesets advances the package again. + +Before Changesets runs, its fence resolves exactly zero or one open Version +Packages PR and captures the prior branch OID (or confirmed absence). An existing +PR must match that ref; it is synchronously marked Draft, verified, and given a +HOLD comment before generation. An orphan ref, missing ref for an existing PR, +ambiguous lookup, or failed quarantine stops the run. The pinned action's single +push shape is replaced with an explicit lease on the captured OID. Fetch/push +URLs must identify this repository; writes use the captured URL and reject URL +rewrites. The HOLD comment's identity, issue, and body are read back before any +push. API/read commands have a 60-second bound; pushes have a 10-minute bound, +with process-group termination and read-back reconciliation on timeout. Freshness is +checked before and after that push, and again after the action's PR API calls. + +Drift, push errors (including lost acknowledgements), or an incomplete action +trigger restoration of the captured ref with a second lease expecting only the +recorded new OID. Only a ref created by this transaction from confirmed absence +may be deleted. Restoration is read back and verified; a lost lease preserves +the third-party ref. Existing PRs stay Draft with HOLD/reconciliation evidence. +API outages fail closed and retain the transaction's prior/new/observed OIDs in +the Actions log, job summary, and attempted artifact upload; they are not reported +as verified quarantine. A retry only reconciles recorded state, never repeats a +push. Successful changes remain Draft (`pr-draft: always`) for a later human +Ready transition and exact-head review. + +Changesets receives a fresh App token after setup, with preparation and action +bounds of 10 and 40 minutes. An `always()` step mints a separate reconciliation +token. Git recovery uses that token through temporary command-scoped headers, +never an expired checkout credential. Failed refresh leaves explicit unverified +reconciliation evidence; it does not fall back to stale credentials. + +An App-token push synchronously changes an already-open release PR, even if +the post-push check fails. Publication therefore also checks immutable Git +provenance: the approved final head must be one commit based on the release +merge's exact first parent, and its entire tree must equal the released tree. +A two-parent merge must name that approved head as its second parent. A stale +or altered merge is quarantined even with trusted approval and bypassed or +non-strict branch checks. Regenerate the release PR from current verified main +and obtain new final-head maintainer review before merging. Do not transplant +old approvals, replay an old workflow, or treat an existing PR as unchanged +just because the later PR API call was skipped. Freshness is checked again before external writes and after uploads. GitHub refs, GitHub Releases, and R2 have no shared transaction: a branch can move @@ -245,7 +288,8 @@ its push/PR requests; it does not lock main. An in-flight request can finish after main moves. Such a failure requires inspection and explicit recovery; never treat a previous verification or a queued run as current authority. -GitHub assets are staged privately in a draft before release publication. R2 +GitHub assets are staged privately in a draft before release publication. Exactly +the four signed tuple assets are required; extras and duplicates are rejected. R2 publication is **progressive, with per-object atomicity only**: conditional `If-None-Match: *` writes create missing objects, and existing objects must match byte-for-byte, including signatures and certificates. A failed upload @@ -271,7 +315,8 @@ After the corrected workflow is merged, an authorized maintainer can dispatch with `release_commit` set to the original approved release merge. The dispatch re-runs the existing verification jobs on current main, verifies that the release merge is an ancestor, requires the same package version and unchanged -versioned artifacts, rechecks original final-head human approval, and fences +versioned artifacts, rechecks original final-head maintainer permission and +merge provenance, and fences every publication phase. It never silently repairs from an unrelated push. A new main commit during the attempt requires another explicit dispatch and verification of the new head. If a later package version has already landed, @@ -289,6 +334,15 @@ and schema/compliance enumeration verification. This version is externally exposed; absence of a GitHub Release is not permission to edit or retire its bytes. +**rc.3 recovery is quarantined.** The reviewed head `e7d42cbe` has parent +`decd95f6303ba08d8d482c93078adad0770a4bf3`, but release merge `71f9cd54` has +parent `eb3cbd605fc11be6e398c5b83f4e53d332a8e365`; their trees also differ. +The intervening base changes include a protocol changeset, compliance source, +and storyboard tooling. The original final-head review does not establish +approval of the exact 71f tree. The provenance gate rejects ordinary dispatch +for 71f; the previously proposed dispatch-only recovery is withdrawn. There +is no grandfather exception or provenance override in this patch. + The stale `eb3cbd605fc11be6e398c5b83f4e53d332a8e365` release run `34842463221` subsequently created #7508 at `102916bb28c156b1710f031aa28e4655bcf71cdc`, tree @@ -299,34 +353,38 @@ display obscures that omission. The coordinator marked it Draft, disabled auto-merge, and posted a [HOLD](https://github.com/adcontextprotocol/adcp/pull/7508#issuecomment-5664735087). It is not a recovery candidate. Do not merge, close, replace, or otherwise -mutate it without coordinating with that owner; retain the original approved +mutate it without coordinating with that owner; retain the original committed 71f release artifacts. The fencing regression tests cover this stale-parent shape, missing later changeset, and conflicting rc.3 bytes. -The exact recovery sequence, requiring separate maintainer authorization, is: +The safe incident handoff, requiring separate maintainer authorization, is: -1. Review and merge the ordering fix and JSONL filter fix through normal human - review. Coordinate #7502/#7503's immutability guard and documentation claims; +1. Review and land the ordering fix first, then rebase/requalify and review the + JSONL fix. Coordinate #7502/#7503's immutability guard and documentation claims; this patch changes no release content. Account for every existing release and deploy run using the old workflow, including `34842463221` and `34847610745`, before enabling recovery. New workflow code does not fence runs that already loaded the old code. Maintainers must quiesce those old writers and reserve a quiet publication window. -2. Recheck current main, original approval, ancestry, exact package version, +2. Recheck current main, original review and current reviewer permission, the + reviewed-head and merge-parent/tree relationship, ancestry, package version, the full committed rc.3 tree, checksum, Sigstore identity, tag and release state, and every existing R2 object's bytes. Any conflicting tag, release asset, or R2 bytes is a stop; do not overwrite, delete, regenerate, roll back, or retag. Preserve already exposed bytes and record discrepancies. -3. Dispatch `release.yml` on that current main with `release_commit` equal to - `71f9cd5414454e94ccfef87ce25777ead6fad228`. Let all original verification and - human-approval checks run. It stages/reuses the identical four assets, - publishes `v3.2.0-rc.3` at **71f**, and conditionally fills missing rc.3 R2 - objects. It does not tag the later tested main or publish `latest`. -4. Independently verify the tag target, prerelease flag, all four GitHub assets, +3. **Stop at the 71f provenance mismatch.** Obtain a separately reviewed plan + binding fresh maintainer authority to the exact original 71f tree and its + already exposed bytes. No such exception or recovery execution is authorized + here. Do not rerun/dispatch the ordinary workflow expecting to bypass the + gate, regenerate artifacts, retarget the original merge, or overwrite R2. + Any future approved recovery must retain tag target 71f, stage its unchanged + signed tuple before release/R2 publication, and enforce conditional writes. +4. After a separately approved recovery, independently verify the tag target, + prerelease flag, all four GitHub assets, all 2,093 schema/compliance paths (including JSONL) plus the four protocol files, byte hashes, checksum and Sigstore identity (2,097 files total). Check deployment and docs/SDK readiness separately. If - anything fails or main advances, record the partial state and repeat only - through explicit recovery from the next verified current head. + anything fails or main advances, record the partial state and stop for + explicit reauthorization on the next verified current head. These are recovery instructions, not a record of operations performed. The ordering-fix work must not publish, delete, roll back, tag, release, deploy, diff --git a/package.json b/package.json index a32c3d7672b..9acaf02c313 100644 --- a/package.json +++ b/package.json @@ -121,7 +121,7 @@ "test:build-schemas-hoist-marked": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-hoist-marked.test.cjs", "test:build-schemas-async-response-refs": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-async-response-refs.test.cjs", "test:build-schemas-vendor-metric-uniqueness": "node --test --test-force-exit --test-timeout=30000 tests/build-schemas-vendor-metric-uniqueness.test.cjs", - "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs", + "test:release-workflow": "node tests/release-workflow-immutability.test.cjs && node --test tests/release-publication-ordering.test.cjs tests/changesets-fence.test.cjs && node tests/promote-release-candidate.test.cjs && node tests/published-compliance-versions.test.cjs", "promote:rc": "node scripts/promote-release-candidate.mjs", "test:immutable-release-artifacts": "node tests/immutable-release-artifacts.test.cjs", "test:patch-3-0-compat-bundle": "node --test --test-force-exit --test-timeout=30000 tests/patch-3-0-compat-bundle.test.cjs", diff --git a/scripts/check-release-state.cjs b/scripts/check-release-state.cjs index fd8c202c727..1f3ca1be977 100644 --- a/scripts/check-release-state.cjs +++ b/scripts/check-release-state.cjs @@ -79,11 +79,15 @@ function published( "tagName,isDraft,isPrerelease,assets", ]), ); - const names = release.assets.map((asset) => asset.name); + const names = Array.isArray(release?.assets) + ? release.assets.map((asset) => asset?.name) + : []; if ( release.tagName !== tag || - release.isDraft || + release.isDraft !== false || release.isPrerelease !== version.includes("-") || + names.length !== 4 || + new Set(names).size !== 4 || !["", ".sha256", ".sig", ".crt"].every((suffix) => names.includes(`${version}.tgz${suffix}`), ) @@ -144,6 +148,173 @@ function recovery(source) { ); } +function provenance(source, head) { + if (!shaPattern.test(source || "") || !shaPattern.test(head || "")) + throw new Error( + "Release provenance requires exact merge and reviewed head SHAs.", + ); + const generated = git("rev-list", "--parents", "-n", "1", head).split(" "); + const merged = git("rev-list", "--parents", "-n", "1", source).split(" "); + if ( + generated.length !== 2 || + ![2, 3].includes(merged.length) || + generated[1] !== merged[1] || + (merged.length === 3 && merged[2] !== head) || + git("rev-parse", `${head}^{tree}`) !== git("rev-parse", `${source}^{tree}`) + ) { + throw new Error( + "Quarantined release: the reviewed generated head must have the merge's exact base parent and released tree. A stale existing PR update cannot authorize publication; see RELEASING.md.", + ); + } +} + +function approval() { + const tested = current(); + const source = process.env.RELEASE_SHA; + const repository = process.env.GITHUB_REPOSITORY; + const branch = process.env.PUBLICATION_BRANCH || process.env.GITHUB_REF_NAME; + if ( + !shaPattern.test(source || "") || + !/^[\w.-]+\/[\w.-]+$/.test(repository || "") + ) + throw new Error("Approval requires an exact release SHA and repository."); + git("merge-base", "--is-ancestor", source, tested); + const api = (endpoint, paginate = false) => { + const value = JSON.parse( + run("gh", [ + "api", + ...(paginate ? ["--paginate", "--slurp"] : []), + `/repos/${repository}${endpoint}`, + ]), + ); + if (!paginate) return value; + if (!Array.isArray(value) || !value.every(Array.isArray)) + throw new Error("Ambiguous paginated approval response."); + return value.flat(); + }; + const associated = api(`/commits/${source}/pulls`, true); + if ( + !associated.every( + (pr) => + pr && + Number.isSafeInteger(pr.number) && + pr.number > 0 && + typeof pr.base?.ref === "string" && + (pr.merged_at === null || + (typeof pr.merged_at === "string" && + Number.isFinite(Date.parse(pr.merged_at)))), + ) + ) + throw new Error("Malformed associated release PR response."); + const pulls = associated.filter( + (pr) => pr.merged_at && pr.base?.ref === branch, + ); + if ( + pulls.length !== 1 || + !Number.isSafeInteger(pulls[0].number) || + pulls[0].number < 1 + ) + throw new Error( + "Release commit must identify exactly one merged release PR.", + ); + const pr = api(`/pulls/${pulls[0].number}`); + const head = pr.head?.sha; + const identity = (user) => + Number.isSafeInteger(user?.id) && + user.id > 0 && + typeof user.login === "string" && + /^[a-z0-9][a-z0-9-]{0,38}(?:\[bot\])?$/i.test(user.login); + if ( + pr.number !== pulls[0].number || + pr.merged !== true || + pr.merge_commit_sha !== source || + pr.base?.ref !== branch || + pr.base.repo?.full_name !== repository || + !shaPattern.test(head || "") || + !identity(pr.user) + ) + throw new Error("Ambiguous merged release PR identity."); + + const reviews = api(`/pulls/${pulls[0].number}/reviews`, true); + if ( + new Set(reviews.map((review) => review?.id)).size !== reviews.length || + !reviews.every( + (review) => + identity(review.user) && + Number.isSafeInteger(review.id) && + review.id > 0 && + typeof review.submitted_at === "string" && + Number.isFinite(Date.parse(review.submitted_at)), + ) + ) + throw new Error("Ambiguous release review identity or ordering."); + const latest = new Map(); + reviews.sort( + (a, b) => + Date.parse(a.submitted_at) - Date.parse(b.submitted_at) || a.id - b.id, + ); + for (const review of reviews) latest.set(review.user.id, review); + let trusted = 0; + for (const review of latest.values()) { + const user = review.user; + if ( + review.state !== "APPROVED" || + review.commit_id !== head || + user.type !== "User" || + user.id === pr.user.id || + user.login.toLowerCase() === pr.user.login.toLowerCase() + ) + continue; + // Current effective access, not public-review ability or author_association. + // GitHub maps maintain to permission=write and triage to permission=read. + // https://docs.github.com/en/rest/collaborators/collaborators#get-repository-permissions-for-a-user + const access = api( + `/collaborators/${encodeURIComponent(user.login)}/permission`, + ); + if ( + !identity(access?.user) || + access.user.id !== user.id || + access.user.type !== "User" || + access.user.login.toLowerCase() !== user.login.toLowerCase() + ) + throw new Error("Ambiguous collaborator permission identity."); + const role = `${access.permission}:${access.role_name}`; + if ( + [ + "write:write", + "write:maintain", + "maintain:maintain", + "admin:admin", + ].includes(role) + ) + trusted++; + else if (!["read:read", "read:triage", "none:none"].includes(role)) + throw new Error("Ambiguous or unsupported collaborator permission."); + } + if (!trusted) + throw new Error( + "Release PR has no current write/maintain/admin non-author approval on its final head.", + ); + + // An App push updates an already-open PR before the post-push fence runs. + // Validate immutable commit provenance again at publication, even if an + // administrator merged that stale update through non-strict branch rules. + try { + git("cat-file", "-e", `${head}^{commit}`); + } catch { + git("fetch", "--no-tags", "origin", `refs/pull/${pulls[0].number}/head`); + if (git("rev-parse", "FETCH_HEAD") !== head) + throw new Error( + "Fetched release PR head differs from its approval record.", + ); + } + provenance(source, head); + current(); + console.log( + `Release PR #${pulls[0].number} has ${trusted} trusted maintainer approval(s) and matching merge provenance.`, + ); +} + try { const [mode, argument] = process.argv.slice(2); if (mode === "current") current(); @@ -162,10 +333,17 @@ try { ]; if (surfaces.some((surface) => fs.existsSync(surface))) published(version); } else if (mode === "recovery") recovery(argument); - else throw new Error("Expected current, published, pending, or recovery."); + else if (mode === "approval") approval(); + else if (mode === "provenance") { + current(); + provenance(process.env.RELEASE_SHA, argument); + } else + throw new Error( + "Expected current, published, pending, recovery, approval, or provenance.", + ); } catch (error) { console.error( - `::error::${error.message}\nPublication stopped. A missing/incomplete release must be recovered explicitly from its original approved merge on current tested main; see RELEASING.md. Do not rerun an obsolete workflow.`, + `::error::${error.message}\nPublication stopped. Recovery requires current tested main and original release authority; provenance mismatches remain quarantined pending a separately reviewed plan. See RELEASING.md. Do not rerun an obsolete workflow.`, ); process.exitCode = 1; } diff --git a/scripts/fence-changesets.cjs b/scripts/fence-changesets.cjs new file mode 100644 index 00000000000..def048810f6 --- /dev/null +++ b/scripts/fence-changesets.cjs @@ -0,0 +1,485 @@ +#!/usr/bin/env node +// Transaction containment for the pinned Changesets git-CLI push. State lives +// outside the generated checkout and is retained as a workflow artifact. +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const directory = path.join(process.env.RUNNER_TEMP, "release-hooks"); +const stateFile = path.join(directory, "state.json"); +const realGit = fs + .readFileSync(path.join(directory, "git-path"), "utf8") + .trim(); +function invoke(command, args, mutation = false) { + // GNU timeout bounds the whole process group, including Git's transport + // children. The optional test/operator setting can only shorten the bound. + const maximum = mutation ? 600000 : 60000; + const requested = Number(process.env.CHANGESETS_FENCE_TIMEOUT_MS || maximum); + if (!Number.isSafeInteger(requested) || requested < 1) + throw Error("Invalid command timeout"); + const limit = Math.min(requested, maximum); + const env = { ...process.env }; + if ( + (command === realGit && + args.some((arg) => ["ls-remote", "fetch", "push"].includes(arg))) || + command === process.execPath + ) { + // The always-run reconciler has its own fresh token. Override the expired + // checkout/action header without storing tokens in state, argv, or config. + if (!env.GH_TOKEN || !state?.url) + throw Error("Fresh Git credentials are unavailable"); + const count = Number(env.GIT_CONFIG_COUNT || 0); + if (!Number.isSafeInteger(count) || count < 0 || count > 1000) + throw Error("Ambiguous Git authentication config"); + env.GIT_CONFIG_COUNT = String(count + 2); + for (let offset = 0; offset < 2; offset++) + env[`GIT_CONFIG_KEY_${count + offset}`] = `http.${state.url}.extraheader`; + env[`GIT_CONFIG_VALUE_${count}`] = ""; + env[`GIT_CONFIG_VALUE_${count + 1}`] = + `AUTHORIZATION: basic ${Buffer.from(`x-access-token:${env.GH_TOKEN}`).toString("base64")}`; + } + return spawnSync( + "timeout", + ["--kill-after=5s", `${limit / 1000}s`, command, ...args], + { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: limit + 10000, + killSignal: "SIGKILL", + env, + }, + ); +} +function run(command, args) { + const result = invoke(command, args); + if (result.status !== 0) + throw Error(`${path.basename(command)} command failed or timed out`); + return result.stdout.trim(); +} +const git = (...args) => run(realGit, args); +const sha = (value) => /^[a-f0-9]{40}$/.test(value || ""); +let state; +function save(event) { + state.events.push(event); + fs.writeFileSync(`${stateFile}.tmp`, JSON.stringify(state, null, 2)); + fs.renameSync(`${stateFile}.tmp`, stateFile); + // Actions logs and summary survive even when a token expires or artifact + // upload fails. Never include credentials or raw API responses. + console.log(`Changesets fence: ${JSON.stringify(event)}`); + if (process.env.GITHUB_STEP_SUMMARY) + fs.appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + `\nChangesets fence: ${JSON.stringify(event)}\n`, + ); +} +function api(endpoint, options = [], pages = false) { + const result = JSON.parse( + run("gh", [ + "api", + ...(pages ? ["--paginate", "--slurp"] : []), + endpoint, + ...options, + ]), + ); + if (!pages) return result; + if (!Array.isArray(result) || !result.every(Array.isArray)) + throw Error("Ambiguous PR pagination"); + return result.flat(); +} +function current() { + originIdentity(); + if (remote(`refs/heads/${state.branch}`) !== state.tested) + throw Error("Main/release branch advanced"); +} +function remote(ref = state.ref) { + noUrlRewrites(); + // An empty successful result proves absence; transport/API errors do not. + const value = git("ls-remote", state.url, ref); + if (!value) return null; + const parts = value.split("\t"); + if (parts.length !== 2 || !sha(parts[0]) || parts[1] !== ref) + throw Error("Ambiguous remote ref"); + return parts[0]; +} +function originIdentity() { + const expected = `https://github.com/${state.repository}`; + const fetch = git("remote", "get-url", "--all", "origin"); + const push = git("remote", "get-url", "--push", "--all", "origin"); + if ( + ![expected, `${expected}.git`].includes(fetch) || + ![expected, `${expected}.git`].includes(push) || + fetch !== push + ) + throw Error("Origin does not identify the authorized repository"); + noUrlRewrites(); + if (state.url && (state.url !== push || state.fetchUrl !== fetch)) + throw Error("Origin changed after capture"); + state.url = push; + state.fetchUrl = fetch; +} +function noUrlRewrites() { + const rewrites = invoke(realGit, ["config", "--get-regexp", "^url\\."]); + if (rewrites.status !== 1 || rewrites.stdout || rewrites.stderr) + throw Error("URL rewrites or ambiguous Git config are not allowed"); +} +function validPR(pr) { + if ( + !pr || + !Number.isSafeInteger(pr.number) || + pr.number < 1 || + typeof pr.node_id !== "string" || + !pr.node_id || + pr.state !== "open" || + typeof pr.draft !== "boolean" || + pr.title !== state.title || + pr.base?.ref !== state.branch || + pr.base?.repo?.full_name !== state.repository || + pr.head?.ref !== state.releaseBranch || + pr.head?.repo?.full_name !== state.repository || + !sha(pr.head.sha) + ) + throw Error("Ambiguous Version Packages PR identity"); + return pr; +} +function expectedTitle() { + const file = ".changeset/pre.json"; + const present = git("ls-tree", "--name-only", state.tested, "--", file); + if (!present) return "Version Packages"; + if (present !== file) throw Error("Ambiguous tested prerelease state"); + const pre = JSON.parse(git("show", `${state.tested}:${file}`)); + if (!pre || typeof pre !== "object" || Array.isArray(pre)) + throw Error("Malformed tested prerelease state"); + // Mirrors the pinned readChangesetState -> finalPrTitle contract. Exit mode + // supplies no preState; only active pre mode appends its exact tested tag. + if (pre.mode !== "pre") return "Version Packages"; + if ( + typeof pre.tag !== "string" || + !/^[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*$/.test(pre.tag) + ) + throw Error("Invalid tested prerelease tag"); + return `Version Packages (${pre.tag})`; +} +function openPR() { + const owner = state.repository.split("/")[0]; + const endpoint = `/repos/${state.repository}/pulls?state=open&base=${encodeURIComponent(state.branch)}&head=${encodeURIComponent(`${owner}:${state.releaseBranch}`)}&per_page=100`; + const prs = api(endpoint, [], true); + if (prs.length > 1) throw Error("Multiple open Version Packages PRs"); + return prs.length ? validPR(prs[0]) : null; +} +function readPR() { + const pr = validPR( + api(`/repos/${state.repository}/pulls/${state.pr.number}`), + ); + if (pr.node_id !== state.pr.node_id) throw Error("PR identity changed"); + return pr; +} +function draft() { + let pr = readPR(); + if (!pr.draft) { + const converted = api("graphql", [ + "-f", + "query=mutation($id:ID!){convertPullRequestToDraft(input:{pullRequestId:$id}){pullRequest{id isDraft}}}", + "-f", + `id=${pr.node_id}`, + ]); + const result = converted?.data?.convertPullRequestToDraft?.pullRequest; + if ( + converted.errors || + result?.id !== pr.node_id || + result?.isDraft !== true + ) + throw Error("Draft mutation response is ambiguous"); + pr = readPR(); + } + if (!pr.draft) throw Error("Draft quarantine could not be verified"); + save({ event: "draft-verified", pr: pr.number, head: pr.head.sha }); + return pr; +} +function hold(reason) { + const body = `HOLD: generated release branch requires human reconciliation/review. ${reason}\nTested base: ${state.tested}\nPrior ref: ${state.prior || "absent"}\nCandidate: ${state.candidate || "not pushed"}\nObserved ref: ${state.observed || "absent/unknown"}\nWorkflow: ${process.env.GITHUB_SERVER_URL || "https://github.com"}/${state.repository}/actions/runs/${process.env.GITHUB_RUN_ID || "unknown"}\nDo not mark Ready or merge without checking this run and fresh release provenance.`; + const posted = api( + `/repos/${state.repository}/issues/${state.pr.number}/comments`, + ["-f", `body=${body}`], + ); + const validate = (comment) => { + if ( + !Number.isSafeInteger(comment?.id) || + comment.id < 1 || + comment.body !== body || + comment.issue_url !== + `https://api.github.com/repos/${state.repository}/issues/${state.pr.number}` + ) + throw Error("HOLD comment response is ambiguous"); + }; + validate(posted); + const recorded = api( + `/repos/${state.repository}/issues/comments/${posted.id}`, + ); + validate(recorded); + if (recorded.id !== posted.id) throw Error("HOLD comment identity changed"); + save({ event: "hold-verified", pr: state.pr.number, comment: posted.id }); +} +function prepare() { + if (fs.existsSync(stateFile)) + throw Error("Fence already prepared; reconcile the recorded transaction"); + const branch = process.env.PUBLICATION_BRANCH; + const repository = process.env.GITHUB_REPOSITORY; + if ( + !["main", "3.1.x", "3.0.x"].includes(branch) || + !sha(process.env.TESTED_SHA) || + !/^[\w.-]+\/[\w.-]+$/.test(repository || "") + ) + throw Error("Invalid fence authority"); + state = { + branch, + repository, + tested: process.env.TESTED_SHA, + releaseBranch: `changeset-release/${branch}`, + ref: `refs/heads/changeset-release/${branch}`, + events: [], + status: "preparing", + }; + current(); + run(process.execPath, [path.join(directory, "check.cjs"), "pending"]); + state.title = expectedTitle(); + const pr = openPR(); + state.prior = remote(); + state.pr = pr + ? { number: pr.number, node_id: pr.node_id, wasDraft: pr.draft } + : null; + save({ event: "captured", prior: state.prior, pr: state.pr }); + // Never adopt/delete an orphan branch, or mutate a PR whose ref is missing. + if ( + Boolean(pr) !== Boolean(state.prior) || + (pr && pr.head.sha !== state.prior) + ) + throw Error("PR and release ref disagree; manual reconciliation required"); + if (pr) { + git("fetch", "--no-tags", state.url, state.ref); + if (git("rev-parse", "FETCH_HEAD") !== state.prior) + throw Error("Release ref changed during capture"); + state.priorTree = git("rev-parse", `${state.prior}^{tree}`); + draft(); + hold("Automation is preparing a leased update. The PR must remain Draft."); + } + current(); + if (remote() !== state.prior) + throw Error("Release ref changed during preparation"); + state.status = "prepared"; + save({ event: "prepared" }); +} +function verifyPR(expected) { + const pr = openPR(); + if (state.pr) { + if ( + !pr || + pr.number !== state.pr.number || + pr.node_id !== state.pr.node_id || + pr.head.sha !== expected || + !pr.draft + ) + throw Error("Release PR changed or left Draft"); + } else if (pr) throw Error("Another PR appeared during generation"); +} +function reconcile(reason) { + state.status = "quarantined"; + state.restored = false; + save({ event: "quarantine", reason }); + const errors = []; + try { + // A new action-created PR may now exist. Adopt only this run's exact head; + // never mutate an unrelated PR if lookup is ambiguous or the lease lost. + if (!state.pr) { + const pr = openPR(); + if (pr) { + if (pr.head.sha !== state.candidate) + throw Error("Unrelated PR appeared"); + state.pr = { + number: pr.number, + node_id: pr.node_id, + wasDraft: pr.draft, + }; + save({ event: "action-created-pr", pr: state.pr }); + } + } + if (state.pr) draft(); + } catch { + errors.push( + "Draft quarantine not currently verifiable; manual reconciliation required", + ); + } + try { + state.observed = remote(); + if ( + state.candidate && + state.observed === state.candidate && + state.candidate !== state.prior + ) { + // Rollback must work after main advances. Bypass only our current-main + // pre-push hook, and only with a lease on this transaction's new OID. + const result = invoke( + realGit, + [ + "-c", + "core.hooksPath=/dev/null", + "push", + state.url, + `${state.prior || ""}:${state.ref}`, + `--force-with-lease=${state.ref}:${state.candidate}`, + ], + true, + ); + state.observed = remote(); + save({ + event: "rollback", + acknowledged: result.status === 0, + observed: state.observed, + }); + } + if (state.observed !== state.prior) + throw Error("Rollback lost lease or failed; observed ref preserved"); + if ( + state.prior && + git("rev-parse", `${state.prior}^{tree}`) !== state.priorTree + ) + throw Error("Restored content differs"); + state.restored = true; + save({ event: "restored", observed: state.observed }); + } catch { + errors.push( + "Ref restoration unverified or lease lost; do not overwrite the observed ref", + ); + } + try { + if (state.pr) { + // Deleting a newly created ref may close its newly created PR. Existing + // PRs are never deleted: their prior ref is restored and they stay Draft. + if (state.prior || !state.restored) draft(); + hold( + `${reason}. ${errors.join(". ") || "Prior ref restored and verified."}`, + ); + } + } catch { + errors.push( + "GitHub HOLD/update unavailable; use retained transaction evidence", + ); + } + state.errors = errors; + save({ + event: "reconciliation", + observed: state.observed, + restored: state.restored === true, + errors, + }); +} +function push(args) { + if ( + state.status !== "prepared" || + JSON.stringify(args) !== + JSON.stringify([ + "push", + "origin", + `HEAD:${state.releaseBranch}`, + "--force", + ]) + ) + throw Error("Unexpected or replayed Changesets push"); + current(); + verifyPR(state.prior); + if (remote() !== state.prior) + throw Error("Release ref lease changed before push"); + const candidate = git("rev-parse", "HEAD"); + if ( + !sha(candidate) || + git("rev-list", "--parents", "-n", "1", candidate) !== + `${candidate} ${state.tested}` + ) + throw Error("Generated head must have exactly the tested base parent"); + state.candidate = candidate; + state.status = "pushing"; + save({ event: "push-intent", candidate }); + try { + current(); + const result = invoke( + realGit, + [ + "push", + state.url, + `${candidate}:${state.ref}`, + `--force-with-lease=${state.ref}:${state.prior || ""}`, + ], + true, + ); + state.observed = remote(); + save({ + event: "push-result", + acknowledged: result.status === 0, + observed: state.observed, + }); + if (result.status !== 0 || state.observed !== candidate) + throw Error("Push failed or acknowledgement/ref ambiguous"); + current(); + verifyPR(candidate); + state.status = "pushed"; + save({ event: "pushed", candidate }); + } catch (error) { + reconcile(error.message); + throw error; + } +} +function finalize() { + if (state.status === "complete") { + // A completed transaction cannot be reopened to roll back a later human + // Ready transition or branch change. Replays are read-only. + current(); + if (remote() !== (state.candidate || state.prior)) + throw Error("Completed transaction ref changed"); + verifyPR(state.candidate || state.prior); + return; + } + if (state.status === "quarantined" || state.status === "pushing") { + reconcile("Incomplete or previously quarantined action"); + throw Error("Changesets transaction remains quarantined"); + } + try { + if (process.env.CHANGESETS_OUTCOME !== "success") + throw Error("Changesets action did not complete successfully"); + current(); + const expected = state.candidate || state.prior; + if (remote() !== expected) throw Error("Release ref changed after action"); + const pr = openPR(); + if (state.candidate && !pr) + throw Error("Successful push has no verifiable Draft PR"); + if (pr) { + if ( + (state.pr && + (pr.number !== state.pr.number || pr.node_id !== state.pr.node_id)) || + pr.head.sha !== expected || + !pr.draft + ) + throw Error("Final PR identity/head/Draft mismatch"); + state.pr ||= { + number: pr.number, + node_id: pr.node_id, + wasDraft: pr.draft, + }; + } + state.status = "complete"; + save({ event: "complete", pr: state.pr, head: expected }); + } catch (error) { + reconcile(error.message); + throw error; + } +} +try { + const [mode, ...args] = process.argv.slice(2); + if (mode === "prepare") prepare(); + else { + state = JSON.parse(fs.readFileSync(stateFile, "utf8")); + if (mode === "push") push(args); + else if (mode === "finalize") finalize(); + else throw Error("Expected prepare, push, or finalize"); + } +} catch (error) { + console.error(`::error::Changesets fence stopped: ${error.message}`); + process.exitCode = 1; +} diff --git a/tests/changesets-fence.test.cjs b/tests/changesets-fence.test.cjs new file mode 100644 index 00000000000..eda0eafee83 --- /dev/null +++ b/tests/changesets-fence.test.cjs @@ -0,0 +1,813 @@ +const { test } = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const os = require("node:os"); +const { execFileSync, spawnSync } = require("node:child_process"); +const YAML = require("yaml"); +const root = path.resolve(__dirname, ".."); +const workflow = YAML.parse( + fs.readFileSync(path.join(root, ".github/workflows/release.yml"), "utf8"), +); +const step = (name) => + workflow.jobs.release.steps.find((step) => step.name === name); +const realGit = execFileSync("which", ["git"], { encoding: "utf8" }).trim(); +const ref = "refs/heads/changeset-release/main"; +const repositoryUrl = "https://github.com/adcontextprotocol/adcp.git"; + +test("the intercepted push shape matches the pinned Changesets producer", () => { + const source = fs.readFileSync( + path.join(root, "tests/fixtures/changesets-action-v2.1.2/github-push.ts"), + "utf8", + ); + const args = JSON.parse( + source + .match(/await exec\("git", (\[.*\]), options\)/)[1] + .replace("`HEAD:${branch}`", '"HEAD:changeset-release/main"'), + ); + assert.deepEqual(args, [ + "push", + "origin", + "HEAD:changeset-release/main", + "--force", + ]); + assert.equal( + step("Create Release Pull Request or Tag Release").env.GH_TOKEN, + "${{ steps.changesets-token.outputs.token }}", + ); + assert.equal( + step("Reconcile Changesets transaction").if, + "always() && steps.release-artifacts.outputs.has_release_artifacts == 'false'", + ); + assert.equal( + step("Retain Changesets reconciliation evidence").if, + step("Reconcile Changesets transaction").if, + ); +}); + +function gitStub() { + const fs = require("node:fs"); + const { spawnSync, execFileSync } = require("node:child_process"); + const args = process.argv.slice(2); + const env = process.env; + const command = (extra) => + execFileSync(env.REAL_GIT, extra, { encoding: "utf8" }).trim(); + const update = (ref, oid) => + command(["--git-dir", env.REMOTE_REPO, "update-ref", ref, oid]); + const rollback = args.includes("core.hooksPath=/dev/null"); + const push = args.includes("push"); + if ( + env.ENFORCE_GIT_AUTH && + args.some((arg) => ["ls-remote", "fetch", "push"].includes(arg)) + ) { + const count = Number(env.GIT_CONFIG_COUNT); + const expected = + "AUTHORIZATION: basic " + + Buffer.from( + "x-access-token:" + (env.EXPECT_GIT_TOKEN || env.GH_TOKEN), + ).toString("base64"); + if ( + env[`GIT_CONFIG_VALUE_${count - 2}`] !== "" || + env[`GIT_CONFIG_VALUE_${count - 1}`] !== expected || + env[`GIT_CONFIG_KEY_${count - 1}`] !== + `http.${env.REPOSITORY_URL}.extraheader` + ) + throw Error("Missing fresh command-scoped Git credentials"); + } + if (args[0] === "remote" && args[1] === "get-url") { + console.log( + env.FAULT === "wrong-origin" || + (env.FAULT === "wrong-push-url" && args.includes("--push")) + ? "https://github.com/other/unrelated.git" + : env.FAULT === "multiple-push-urls" && args.includes("--push") + ? `${env.REPOSITORY_URL}\n${env.REPOSITORY_URL}` + : env.REPOSITORY_URL, + ); + return; + } + if ( + args[0] === "config" && + args.includes("--get-regexp") && + env.FAULT === "url-rewrite" + ) { + console.log("url.https://other.invalid/.insteadof https://github.com/"); + return; + } + if (args[0] === "ls-remote" && fs.existsSync("ref-read-error")) { + fs.unlinkSync("ref-read-error"); + process.exit(1); + } + if (args[0] === "ls-remote" && env.FAULT === "lookup") process.exit(1); + if (push) { + fs.appendFileSync("git-calls.jsonl", JSON.stringify(args) + "\n"); + if (!args.some((arg) => arg.startsWith("--force-with-lease="))) + throw Error("Unleased branch mutation"); + if (rollback && env.FAULT === "rollback-lease") + update(env.RELEASE_REF, env.THIRD_SHA); + if (rollback && env.FAULT === "rollback-failure") process.exit(1); + if (rollback && env.FAULT === "rollback-timeout-before") + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10000); + if (!rollback && env.FAULT === "push-lease") + update(env.RELEASE_REF, env.THIRD_SHA); + if (!rollback && env.FAULT === "push-reject") process.exit(1); + } + const result = spawnSync( + env.REAL_GIT, + args.map((arg) => (arg === env.REPOSITORY_URL ? env.REMOTE_REPO : arg)), + { + stdio: ["ignore", "pipe", "pipe"], + encoding: "utf8", + }, + ); + if (push) { + const value = command(["ls-remote", "origin", env.RELEASE_REF]).split( + "\t", + )[0]; + const prs = JSON.parse(fs.readFileSync("prs.json", "utf8")); + for (const pr of prs) { + if (value) pr.head.sha = value; + else pr.state = "closed"; + } + fs.writeFileSync("prs.json", JSON.stringify(prs)); + if ( + !rollback && + [ + "drift", + "rollback-lease", + "rollback-failure", + "rollback-lost-ack", + "api-after-push", + "rollback-timeout-before", + "rollback-timeout-after", + ].includes(env.FAULT) + ) + update("refs/heads/main", env.ADVANCED_SHA); + if (!rollback && env.FAULT === "api-after-push") + fs.writeFileSync("api-down", "1"); + if (!rollback && env.FAULT === "ref-read-error") + fs.writeFileSync("ref-read-error", "1"); + if ( + (!rollback && env.FAULT === "push-timeout") || + (rollback && env.FAULT === "rollback-timeout-after") + ) + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10000); + if ( + (!rollback && env.FAULT === "lost-ack") || + (rollback && env.FAULT === "rollback-lost-ack") + ) + process.exit(1); + } + process.stdout.write(result.stdout || ""); + process.stderr.write(result.stderr || ""); + process.exit(result.status ?? 1); +} +function ghStub() { + const fs = require("node:fs"); + const args = process.argv.slice(2); + const env = process.env; + fs.appendFileSync("gh-calls.jsonl", JSON.stringify(args) + "\n"); + if (env.FAULT === "api" || fs.existsSync("api-down")) process.exit(1); + if (!env.GH_TOKEN) process.exit(1); + if (env.FAULT === "api-timeout") + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10000); + const prs = JSON.parse(fs.readFileSync("prs.json", "utf8")); + const url = args.find((arg) => arg.startsWith("/repos/")); + let result; + if (args.includes("graphql")) { + if (env.FAULT === "draft-failure") process.exit(1); + const pr = prs.find((pr) => args.includes(`id=${pr.node_id}`)); + if (!pr) throw Error("Wrong Draft target"); + if (env.FAULT !== "draft-noop") pr.draft = true; + fs.writeFileSync("prs.json", JSON.stringify(prs)); + result = { + data: { + convertPullRequestToDraft: { + pullRequest: { id: pr.node_id, isDraft: pr.draft }, + }, + }, + }; + if (env.FAULT === "draft-response") + result = { errors: [{ message: "mutation rejected" }] }; + if (env.FAULT === "draft-lost-ack") process.exit(1); + } else if (url.endsWith("/comments")) { + if (env.FAULT === "hold-failure") process.exit(1); + const comments = fs.existsSync("comments.json") + ? JSON.parse(fs.readFileSync("comments.json", "utf8")) + : []; + result = { + id: comments.length + 1, + body: args.find((arg) => arg.startsWith("body=")).slice(5), + issue_url: "https://api.github.com" + url.replace(/\/comments$/, ""), + }; + comments.push(result); + fs.writeFileSync("comments.json", JSON.stringify(comments)); + if (env.FAULT === "hold-lost-ack") process.exit(1); + if (env.FAULT === "hold-response") result.body = "not the hold"; + if (env.FAULT === "hold-wrong-issue") + result.issue_url = "https://api.github.com/repos/other/repo/issues/1"; + } else if (url.includes("/issues/comments/")) { + if (env.FAULT === "hold-read") process.exit(1); + result = JSON.parse(fs.readFileSync("comments.json", "utf8")).find( + (comment) => String(comment.id) === url.split("/").at(-1), + ); + if (env.FAULT === "hold-read-body") result.body = "changed hold"; + } else if (url.includes("?")) + result = [prs.filter((pr) => pr.state === "open")]; + else result = prs.find((pr) => url.endsWith(`/pulls/${pr.number}`)); + console.log(JSON.stringify(result)); +} +function fixture(t, existing = true, pre = { mode: "pre", tag: "rc" }) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "changesets-fence-")); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + const write = (file, bytes) => { + fs.mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + fs.writeFileSync(path.join(dir, file), bytes); + }; + const git = (...args) => + execFileSync(realGit, args, { + cwd: dir, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + git("init", "-q"); + git("config", "user.name", "Test"); + git("config", "user.email", "test@example.invalid"); + git("config", "commit.gpgSign", "false"); + write("package.json", '{"version":"3.2.0-rc.3"}'); + git("add", "package.json"); + if (pre !== null) { + write(".changeset/pre.json", JSON.stringify(pre)); + git("add", ".changeset/pre.json"); + } + git("commit", "-qm", "Initial"); + const initial = git("rev-parse", "HEAD"); + const commit = (file, text) => { + write(file, text); + git("add", file); + git("commit", "-qm", text); + return git("rev-parse", "HEAD"); + }; + const prior = commit("release", "previous generated release"); + git("checkout", "--detach", initial); + const tested = commit("main", "tested main"); + const advanced = commit("later", "advanced main"); + const third = commit("third", "third-party ref"); + git("checkout", "--detach", tested); + const candidate = commit("release", "new generated release"); + const bare = path.join(dir, "remote.git"); + git("init", "--bare", "-q", bare); + git("remote", "add", "origin", bare); + git( + "push", + "-q", + "origin", + `${tested}:refs/heads/main`, + `${advanced}:refs/heads/advanced`, + `${third}:refs/heads/third`, + ...(existing ? [`${prior}:${ref}`] : []), + ); + git("checkout", "--detach", tested); + const pr = { + number: 8, + node_id: "PR_8", + title: + pre?.mode === "pre" + ? `Version Packages (${pre.tag})` + : "Version Packages", + state: "open", + draft: false, + head: { + ref: "changeset-release/main", + sha: prior, + repo: { full_name: "adcontextprotocol/adcp" }, + }, + base: { ref: "main", repo: { full_name: "adcontextprotocol/adcp" } }, + }; + write("prs.json", JSON.stringify(existing ? [pr] : [])); + write("git-calls.jsonl", ""); + write("gh-calls.jsonl", ""); + for (const name of ["check-release-state.cjs", "fence-changesets.cjs"]) + write(`scripts/${name}`, fs.readFileSync(path.join(root, "scripts", name))); + for (const [name, stub] of [ + ["git", gitStub], + ["gh", ghStub], + ]) { + write(`bin/${name}`, `#!${process.execPath}\n(${stub.toString()})();`); + fs.chmodSync(path.join(dir, "bin", name), 0o755); + } + const temporary = path.join(dir, "temp"); + const wrapper = path.join(temporary, "release-hooks"); + const env = { + ...process.env, + PATH: `${dir}/bin:${process.env.PATH}`, + RUNNER_TEMP: temporary, + GITHUB_PATH: path.join(dir, "github-path"), + GITHUB_STEP_SUMMARY: path.join(dir, "summary"), + TESTED_SHA: tested, + PUBLICATION_BRANCH: "main", + GITHUB_REPOSITORY: "adcontextprotocol/adcp", + GH_TOKEN: "fixture-token", + ENFORCE_GIT_AUTH: "1", + REAL_GIT: realGit, + REPOSITORY_URL: repositoryUrl, + REMOTE_REPO: bare, + RELEASE_REF: ref, + ADVANCED_SHA: advanced, + THIRD_SHA: third, + }; + const run = (cmd, args, extra = {}) => + spawnSync(cmd, args, { + cwd: dir, + env: { ...env, ...extra }, + encoding: "utf8", + }); + const install = (extra) => + run("bash", ["-c", step("Fence Changesets mutations").run], extra); + const push = ( + extra, + args = ["push", "origin", "HEAD:changeset-release/main", "--force"], + ) => { + git("-c", "core.hooksPath=/dev/null", "checkout", "--detach", candidate); + return run(path.join(wrapper, "git"), args, extra); + }; + const finalize = (extra) => + run("bash", ["-c", step("Reconcile Changesets transaction").run], { + CHANGESETS_OUTCOME: "success", + ...extra, + }); + const prs = () => + JSON.parse(fs.readFileSync(path.join(dir, "prs.json"), "utf8")); + const state = () => + JSON.parse(fs.readFileSync(path.join(wrapper, "state.json"), "utf8")); + const observed = () => git("ls-remote", "origin", ref).split("\t")[0] || null; + const calls = () => + fs + .readFileSync(path.join(dir, "git-calls.jsonl"), "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map(JSON.parse); + return { + run, + wrapper, + dir, + write, + git, + install, + push, + finalize, + prs, + state, + observed, + calls, + prior, + tested, + candidate, + advanced, + third, + pr, + bare, + }; +} +function ok(result) { + assert.equal(result.status, 0, result.stderr); +} +function stopped(result) { + assert.notEqual(result.status, 0, result.stdout); +} + +test("ready existing PR is verified Draft before a leased push and stays Draft on success", (t) => { + const f = fixture(t); + ok(f.install()); + assert.equal(f.prs()[0].draft, true); + assert.equal(f.observed(), f.prior); + ok(f.push()); + assert.equal(f.observed(), f.candidate); + ok(f.finalize()); + assert.equal(f.state().status, "complete"); + assert.equal(f.prs()[0].draft, true); + assert.deepEqual(f.calls()[0], [ + "push", + repositoryUrl, + `${f.candidate}:${ref}`, + `--force-with-lease=${ref}:${f.prior}`, + ]); + assert.equal( + step("Create Release Pull Request or Tag Release").with["pr-draft"], + "always", + ); +}); +test("no PR/ref creates with an empty lease; action-created PR must be Draft", (t) => { + const f = fixture(t, false); + ok(f.install()); + ok(f.push()); + assert.equal(f.calls()[0].at(-1), `--force-with-lease=${ref}:`); + f.write( + "prs.json", + JSON.stringify([ + { ...f.pr, draft: true, head: { ...f.pr.head, sha: f.candidate } }, + ]), + ); + ok(f.finalize()); + assert.equal(f.state().status, "complete"); +}); +for (const fault of [ + "api", + "lookup", + "draft-failure", + "draft-noop", + "draft-response", + "draft-lost-ack", + "hold-failure", + "hold-lost-ack", + "hold-response", + "hold-wrong-issue", + "hold-read", + "hold-read-body", + "wrong-origin", + "wrong-push-url", + "multiple-push-urls", + "url-rewrite", +]) + test(`preparation fails closed before a branch write: ${fault}`, (t) => { + const f = fixture(t); + stopped(f.install({ FAULT: fault })); + assert.deepEqual(f.calls(), []); + assert.equal(f.observed(), f.prior); + }); +for (const kind of [ + "duplicate", + "orphan", + "missing-ref", + "wrong-head", + "wrong-repository", + "wrong-title", + "wrong-tag", + "missing-suffix", + "malformed", +]) + test(`ambiguous capture stops: ${kind}`, (t) => { + const f = fixture(t); + if (kind === "duplicate") + f.write( + "prs.json", + JSON.stringify([f.pr, { ...f.pr, number: 9, node_id: "PR_9" }]), + ); + if (kind === "orphan") f.write("prs.json", "[]"); + if (kind === "missing-ref") + f.git("--git-dir", f.bare, "update-ref", "-d", ref); + if (kind === "wrong-head") + f.write( + "prs.json", + JSON.stringify([{ ...f.pr, head: { ...f.pr.head, sha: f.third } }]), + ); + if (kind === "wrong-repository") + f.write( + "prs.json", + JSON.stringify([ + { + ...f.pr, + head: { ...f.pr.head, repo: { full_name: "other/repository" } }, + }, + ]), + ); + if (kind === "wrong-title") + f.write( + "prs.json", + JSON.stringify([{ ...f.pr, title: "Unrelated pull request" }]), + ); + if (kind === "wrong-tag" || kind === "missing-suffix") + f.write( + "prs.json", + JSON.stringify([ + { + ...f.pr, + title: + kind === "wrong-tag" + ? "Version Packages (beta)" + : "Version Packages", + }, + ]), + ); + if (kind === "malformed") f.write("prs.json", '[{"state":"open"}]'); + stopped(f.install()); + assert.deepEqual(f.calls(), []); + }); +test("main advances during long generation: no push, existing PR remains Draft", (t) => { + const f = fixture(t); + ok(f.install()); + f.git("--git-dir", f.bare, "update-ref", "refs/heads/main", f.advanced); + stopped(f.push()); + assert.deepEqual(f.calls(), []); + assert.equal(f.observed(), f.prior); + assert.equal(f.prs()[0].draft, true); +}); +for (const existing of [true, false]) + for (const fault of [ + "drift", + "lost-ack", + "rollback-lost-ack", + "ref-read-error", + ]) + test(`${existing ? "existing" : "new"} ref is restored with an exact lease after ${fault}`, (t) => { + const f = fixture(t, existing); + ok(f.install()); + stopped(f.push({ FAULT: fault })); + assert.equal(f.observed(), existing ? f.prior : null); + assert.equal(f.state().restored, true); + assert.equal(f.calls().length, 2); + assert.equal( + f.calls()[1].at(-1), + `--force-with-lease=${ref}:${f.candidate}`, + ); + if (existing) { + assert.equal(f.prs()[0].draft, true); + assert.equal(f.prs()[0].head.sha, f.prior); + } + const before = f.calls().length; + stopped(f.finalize({ CHANGESETS_OUTCOME: "failure" })); + stopped(f.push()); + assert.equal( + f.calls().length, + before, + "replay must not mutate any branch", + ); + }); +for (const fault of ["rollback-lease", "rollback-failure", "api-after-push"]) + test(`failed restoration retains containment and durable evidence: ${fault}`, (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({ FAULT: fault })); + assert.equal(f.prs()[0].draft, true); + assert.equal(f.state().status, "quarantined"); + assert.ok(f.state().errors.length); + assert.ok( + fs + .readFileSync(path.join(f.dir, "summary"), "utf8") + .includes("reconciliation"), + ); + assert.equal( + f.observed(), + fault === "rollback-lease" + ? f.third + : fault === "api-after-push" + ? f.prior + : f.candidate, + ); + if (fault === "rollback-lease") { + const count = f.calls().length; + stopped(f.finalize()); + assert.equal(f.observed(), f.third); + assert.equal(f.calls().length, count); + } + }); +test("concurrent branch update loses initial lease without overwriting it", (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({ FAULT: "push-lease" })); + assert.equal(f.observed(), f.third); + assert.equal(f.calls().length, 1); + assert.equal(f.prs()[0].draft, true); +}); +test("rejected push is a verified no-op and stays quarantined", (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({ FAULT: "push-reject" })); + assert.equal(f.observed(), f.prior); + assert.equal(f.state().restored, true); + assert.equal(f.calls().length, 1); +}); +for (const args of [ + [ + "-c", + "core.hooksPath=/dev/null", + "push", + "origin", + "HEAD:changeset-release/main", + "--force", + ], + ["-C", ".", "push", "origin", "HEAD:changeset-release/main", "--force"], + ["push", "other", "HEAD:changeset-release/main", "--force"], + ["push", "origin", "HEAD:unrelated", "--force"], + ["push", "origin", "HEAD:changeset-release/main", "--force", "--delete"], +]) + test(`unexpected push is refused: ${args.join(" ")}`, (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({}, args)); + assert.deepEqual(f.calls(), []); + }); +test("drift after the push and action API call is reconciled by the installed finalizer", (t) => { + const f = fixture(t); + ok(f.install()); + ok(f.push()); + f.git("--git-dir", f.bare, "update-ref", "refs/heads/main", f.advanced); + stopped(f.finalize()); + assert.equal(f.observed(), f.prior); + assert.equal(f.prs()[0].draft, true); +}); +test("action failure after successful push is restored, including PR API outage", (t) => { + const f = fixture(t); + ok(f.install()); + ok(f.push()); + stopped(f.finalize({ CHANGESETS_OUTCOME: "failure", FAULT: "api" })); + assert.equal(f.observed(), f.prior); + assert.equal(f.prs()[0].draft, true); + assert.ok(f.state().errors.length); +}); +test("replay after success never pushes twice", (t) => { + const f = fixture(t); + ok(f.install()); + ok(f.push()); + ok(f.finalize()); + ok(f.finalize()); + stopped(f.push()); + assert.equal(f.calls().length, 1); +}); +test("pinned read-only Git identity probes still work through the installed wrapper", (t) => { + const f = fixture(t); + ok(f.install()); + const source = fs.readFileSync( + path.join(root, "tests/fixtures/changesets-action-v2.1.2/github-push.ts"), + "utf8", + ); + const probes = [ + ...source.matchAll(/getExecOutput\(\s*"git",\s*(\[[^\]]+\])/g), + ].map((match) => JSON.parse(match[1])); + assert.equal(probes.length, 2); + for (const probe of probes) + ok(f.run(path.join(f.wrapper, "git"), [...probe])); + assert.equal(f.calls().length, 0); +}); +test("completed replay cannot roll back a later human Ready transition", (t) => { + const f = fixture(t); + ok(f.install()); + ok(f.push()); + ok(f.finalize()); + f.write( + "prs.json", + JSON.stringify(f.prs().map((pr) => ({ ...pr, draft: false }))), + ); + stopped(f.finalize()); + assert.equal(f.calls().length, 1); + assert.equal(f.prs()[0].draft, false); + assert.equal(f.observed(), f.candidate); +}); +for (const changed of ["ready", "missing-ref", "new-pr", "api"]) + test(`state changes during generation stop before push: ${changed}`, (t) => { + const f = fixture(t, changed !== "new-pr"); + ok(f.install()); + if (changed === "ready") + f.write( + "prs.json", + JSON.stringify(f.prs().map((pr) => ({ ...pr, draft: false }))), + ); + if (changed === "missing-ref") + f.git("--git-dir", f.bare, "update-ref", "-d", ref); + if (changed === "new-pr") f.write("prs.json", JSON.stringify([f.pr])); + stopped(f.push(changed === "api" ? { FAULT: "api" } : {})); + assert.equal(f.calls().length, 0); + }); +test("new ref is conditionally removed if action fails before creating its PR", (t) => { + const f = fixture(t, false); + ok(f.install()); + ok(f.push()); + stopped(f.finalize({ CHANGESETS_OUTCOME: "failure", FAULT: "api" })); + assert.equal(f.observed(), null); + assert.equal(f.state().restored, true); + assert.ok(f.state().errors.length); +}); +test("a new Ready PR from a broken action is quarantined before deleting only its owned new ref", (t) => { + const f = fixture(t, false); + ok(f.install()); + ok(f.push()); + f.write( + "prs.json", + JSON.stringify([{ ...f.pr, head: { ...f.pr.head, sha: f.candidate } }]), + ); + stopped(f.finalize()); + assert.equal(f.observed(), null); + assert.equal(f.prs()[0].draft, true); + assert.equal(f.prs()[0].state, "closed"); +}); +test("an empty initial lease never overwrites a concurrently created ref", (t) => { + const f = fixture(t, false); + ok(f.install()); + stopped(f.push({ FAULT: "push-lease" })); + assert.equal(f.observed(), f.third); + assert.equal(f.calls().length, 1); +}); +for (const fault of [ + "push-timeout", + "rollback-timeout-before", + "rollback-timeout-after", +]) + test(`bounded subprocess timeout reconciles a possibly applied write: ${fault}`, (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({ FAULT: fault, CHANGESETS_FENCE_TIMEOUT_MS: "1000" })); + assert.equal( + f.observed(), + fault === "rollback-timeout-before" ? f.candidate : f.prior, + ); + assert.equal(f.prs()[0].draft, true); + assert.equal(f.state().status, "quarantined"); + stopped(f.finalize({ CHANGESETS_OUTCOME: "failure" })); + assert.equal(f.observed(), f.prior); + }); +test("API timeout before mutation leaves the release ref unchanged", (t) => { + const f = fixture(t); + stopped( + f.install({ FAULT: "api-timeout", CHANGESETS_FENCE_TIMEOUT_MS: "1000" }), + ); + assert.equal(f.observed(), f.prior); + assert.equal(f.calls().length, 0); +}); +test("a changed push target during generation cannot redirect the leased write", (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({ FAULT: "wrong-push-url" })); + assert.equal(f.observed(), f.prior); + assert.equal(f.calls().length, 0); +}); +test("reconciliation replay reports a later third-party ref as unrestored without overwriting it", (t) => { + const f = fixture(t); + ok(f.install()); + stopped(f.push({ FAULT: "drift" })); + assert.equal(f.state().restored, true); + f.git("--git-dir", f.bare, "update-ref", ref, f.third); + const before = f.calls().length; + stopped(f.finalize()); + assert.equal(f.state().restored, false); + assert.equal(f.observed(), f.third); + assert.equal(f.calls().length, before); +}); +test("reconciliation uses its fresh token instead of inherited expired Git headers", (t) => { + const f = fixture(t); + ok(f.install()); + ok(f.push()); + stopped( + f.finalize({ + CHANGESETS_OUTCOME: "failure", + GH_TOKEN: "refreshed-token", + EXPECT_GIT_TOKEN: "refreshed-token", + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `http.${repositoryUrl}.extraheader`, + GIT_CONFIG_VALUE_0: "expired-header", + }), + ); + assert.equal(f.observed(), f.prior); + assert.equal(f.state().restored, true); + assert.doesNotMatch( + JSON.stringify(f.state()), + /refreshed-token|expired-header|AUTHORIZATION/, + ); +}); +test("failed token refresh never falls back to expired credentials; later reconciliation is resumable", (t) => { + const f = fixture(t); + ok(f.install()); + ok(f.push()); + stopped(f.finalize({ CHANGESETS_OUTCOME: "failure", GH_TOKEN: "" })); + assert.equal(f.observed(), f.candidate); + assert.equal(f.state().restored, false); + stopped( + f.finalize({ CHANGESETS_OUTCOME: "failure", GH_TOKEN: "refreshed-token" }), + ); + assert.equal(f.observed(), f.prior); +}); +test("mutation and reconciliation mint separate tokens after setup, including failed actions", () => { + const steps = workflow.jobs.release.steps; + const index = (name) => steps.findIndex((step) => step.name === name); + assert.ok( + index("Mint Changesets mutation token") > index("Install Dependencies"), + ); + assert.ok( + index("Mint Changesets mutation token") < + index("Fence Changesets mutations"), + ); + assert.match(step("Mint Changesets reconciliation token").if, /^always\(\)/); + assert.equal( + step("Reconcile Changesets transaction").env.GH_TOKEN, + "${{ steps.reconciliation-token.outputs.token }}", + ); + assert.equal( + step("Create Release Pull Request or Tag Release")["timeout-minutes"], + 40, + ); +}); +for (const pre of [ + null, + { mode: "exit", tag: "rc" }, + { mode: "pre", tag: "beta" }, +]) + test(`expected PR title comes from the tested prerelease state: ${JSON.stringify(pre)}`, (t) => { + const f = fixture(t, true, pre); + ok(f.install()); + assert.equal(f.state().title, f.pr.title); + }); +test("working-tree prerelease edits cannot change the captured tested title", (t) => { + const f = fixture(t); + f.write(".changeset/pre.json", JSON.stringify({ mode: "pre", tag: "beta" })); + ok(f.install()); + assert.equal(f.state().title, "Version Packages (rc)"); +}); diff --git a/tests/fixtures/changesets-action-v2.1.2/github-push.ts b/tests/fixtures/changesets-action-v2.1.2/github-push.ts new file mode 100644 index 00000000000..31b75d580a5 --- /dev/null +++ b/tests/fixtures/changesets-action-v2.1.2/github-push.ts @@ -0,0 +1,18 @@ +// Exact push helper excerpt from changesets/action src/github.ts at +// ae32849d5ba541f9ae29e40e22a623bc13562f51 (v2.1.2), MIT licensed. +// https://github.com/changesets/action/blob/ae32849d5ba541f9ae29e40e22a623bc13562f51/src/github.ts +const push = async (branch: string, options: GitOptions) => { + await exec("git", ["push", "origin", `HEAD:${branch}`, "--force"], options); +}; + +// Exact identity-probe calls from ensureGitUser in the same pinned source. +const authorIdentity = await getExecOutput( + "git", + ["-c", "user.useConfigOnly=true", "var", "GIT_AUTHOR_IDENT"], + { cwd: this.cwd, ignoreReturnCode: true, silent: true }, +); +const committerIdentity = await getExecOutput( + "git", + ["-c", "user.useConfigOnly=true", "var", "GIT_COMMITTER_IDENT"], + { cwd: this.cwd, ignoreReturnCode: true, silent: true }, +); diff --git a/tests/release-publication-ordering.test.cjs b/tests/release-publication-ordering.test.cjs index 2155a542859..11b2d3f7015 100644 --- a/tests/release-publication-ordering.test.cjs +++ b/tests/release-publication-ordering.test.cjs @@ -40,10 +40,16 @@ function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { for (const suffix of ["", ".sha256", ".sig", ".crt"]) write(`dist/protocol/${version}.tgz${suffix}`, `signed${suffix}`); if (includeReleaseHistory) { - for (let rc = 0; rc <= 3; rc++) { - const historical = `3.2.0-rc.${rc}`; + for (const historical of [ + "3.1.22", + "3.1.23", + ...[0, 1, 2, 3].map((rc) => `3.2.0-rc.${rc}`), + ]) { write(`dist/schemas/${historical}/index.json`, '{"release":true}'); - write(`dist/compliance/${historical}/${rowsPath}`, '{"immutable":true}\n'); + write( + `dist/compliance/${historical}/${rowsPath}`, + '{"immutable":true}\n', + ); for (const suffix of ["", ".sha256", ".sig", ".crt"]) write(`dist/protocol/${historical}.tgz${suffix}`, `signed${suffix}`); } @@ -58,6 +64,7 @@ function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { git("init", "-q"); git("config", "user.name", "Test"); git("config", "user.email", "test@example.invalid"); + git("commit", "--allow-empty", "-qm", "Prior main"); git("add", "."); git("commit", "-qm", "Approved release"); const source = git("rev-parse", "HEAD"); @@ -68,6 +75,7 @@ function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { write("remote", head); write("tag", source); write("calls", ""); + write("permission-calls", ""); write( "release.json", JSON.stringify({ @@ -84,12 +92,11 @@ function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { write(`bin/${name}`, `#!/usr/bin/env node\n${code}`); fs.chmodSync(path.join(dir, "bin", name), 0o755); }; - const common = `const fs=require('node:fs'),path=require('node:path');const args=process.argv.slice(2);const read=f=>fs.readFileSync(f,'utf8');const log=s=>fs.appendFileSync('calls',s+'\\n');const change=()=>fs.writeFileSync('remote','f'.repeat(40));`; + const common = `const fs=require('node:fs'),path=require('node:path');const args=process.argv.slice(2);const read=f=>fs.readFileSync(f,'utf8');const log=s=>fs.appendFileSync('calls',s+'\\n');const change=()=>fs.writeFileSync('remote',process.env.ADVANCE_TO||'f'.repeat(40));`; executable( "git", common + ` - if(args[0]==='push' && process.env.ADVANCE_PUSH) { log('branch push');change();process.exit(0); } if(args[0]==='ls-remote') { if(process.env.REMOTE_ERROR) process.exit(1); const ref=args.find(a=>a.startsWith('refs/heads/')); @@ -108,9 +115,15 @@ function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { const save=r=>fs.writeFileSync('release.json',JSON.stringify(r)); if(args[0]==='api') { const url=args.find(a=>a.startsWith('/repos/')); - if(url.includes('/commits/')) console.log(JSON.stringify([{number:1,merged_at:'2026-09-14',base:{ref:'main'}}])); - else if(url.endsWith('/reviews')) console.log(process.env.REVIEWS||'[]'); - else console.log(JSON.stringify({head:{sha:'a'.repeat(40)},merge_commit_sha:process.env.RELEASE_SHA,user:{login:'author'}})); + const output=value=>console.log(JSON.stringify(args.includes('--slurp')?[value]:value)); + if(url.includes('/commits/')) { if(process.env.PULLS_ERROR) process.exit(1); output(JSON.parse(process.env.ASSOCIATED_PRS||'[{"number":1,"merged_at":"2026-09-14","base":{"ref":"main"}}]')); } + else if(url.endsWith('/reviews')) output(JSON.parse(process.env.REVIEWS||'[]')); + else if(url.includes('/collaborators/')) { + fs.appendFileSync('permission-calls',url+'\\n'); + if(process.env.PERMISSION_ERROR && (!process.env.PERMISSION_ERROR_LOGIN || url.includes('/'+process.env.PERMISSION_ERROR_LOGIN+'/'))) {console.error(process.env.PERMISSION_ERROR);process.exit(1);} + console.log(process.env.PERMISSION_RESPONSE||JSON.stringify({permission:'write',role_name:'write',user:{id:123,login:'reviewer',type:'User'}})); + if(process.env.ADVANCE_PERMISSION) change(); + } else output(JSON.parse(process.env.MERGED_PR||JSON.stringify({number:1,head:{sha:process.env.RELEASE_SHA},merged:true,merge_commit_sha:process.env.RELEASE_SHA,base:{ref:'main',repo:{full_name:'adcontextprotocol/adcp'}},user:{id:456,login:'author',type:'User'}}))); } else if(args[1]==='view') { if(!fs.existsSync('release.json')) process.exit(1); const r=release(); @@ -195,18 +208,19 @@ function fixture(t, fixtureVersion = version, includeReleaseHistory = false) { }; } const immutable = ["--version", version, "--skip-latest"]; -const approval = (extra) => +const approval = (head, extra = {}) => JSON.stringify([ { - user: { type: "User", login: "reviewer" }, + id: 1, + user: { id: 123, type: "User", login: "reviewer" }, state: "APPROVED", - commit_id: "a".repeat(40), + commit_id: head, submitted_at: "2026-09-14", ...extra, }, ]); -test("app-only deploy cannot backfill immutable JSONL or historical rc.0–rc.3 artifacts", (t) => { +test("app-only deploy cannot backfill #7507 stable artifacts or #7509 immutable rc.0–rc.3 JSONL", (t) => { const f = fixture(t, version, true); for (let rc = 0; rc <= 3; rc++) { const file = `dist/compliance/3.2.0-rc.${rc}/${rowsPath}`; @@ -217,11 +231,12 @@ test("app-only deploy cannot backfill immutable JSONL or historical rc.0–rc.3 assert.notEqual(bulk.status, 0); assert.match(bulk.stderr, /Live bulk backfill is disabled/); assert.equal(f.calls(), ""); - // The destination boundary holds independently of #7509's JSONL filter. + // The destination boundary holds independently of #7509's JSONL filter + // and #7507's curated stable artifact additions. const result = f.upload(["--latest-only", "--build-latest"]); assert.equal(result.status, 0, result.stderr); assert.ok(f.calls().includes("schemas/latest")); - assert.doesNotMatch(f.calls(), /3\.2\.0-rc\.[0-3]|r2 put /); + assert.doesNotMatch(f.calls(), /3\.2\.0-rc\.[0-3]|3\.1\.2[23]|r2 put /); assert.ok( f .calls() @@ -264,11 +279,7 @@ test("main advances during a long app deploy after its successful pre-deploy gat }); test("old run cannot update Changesets after verification/versioning or enter publication", (t) => { const f = fixture(t); - const install = f.run("bash", ["-c", step("Fence Changesets mutations").run]); - assert.equal(install.status, 0, install.stderr); f.write("remote", "f".repeat(40)); - const hook = f.run("sh", [path.join(f.dir, "temp/release-hooks/pre-push")]); - assert.notEqual(hook.status, 0); assert.notEqual(f.script("current").status, 0); assert.equal(f.calls(), ""); }); @@ -300,11 +311,12 @@ test("queued committed release survives later pushes; lost/manual cancellations 0, ); }); -for (const [name, reviews] of [ - ["none", "[]"], - ["bot", approval({ user: { type: "Bot", login: "bot" } })], - ["stale", approval({ commit_id: "b".repeat(40) })], - ["dismissed", approval({ state: "DISMISSED" })], +for (const [name, overrides] of [ + ["none", null], + ["bot", { user: { id: 123, type: "Bot", login: "bot" } }], + ["stale", { commit_id: "b".repeat(40) }], + ["dismissed", { state: "DISMISSED" }], + ["author", { user: { id: 456, type: "User", login: "author" } }], ]) test(`no ${name} approval can publish`, (t) => { const f = fixture(t); @@ -314,7 +326,7 @@ for (const [name, reviews] of [ "-c", step("Require human approval for committed release artifacts").run, ], - { REVIEWS: reviews }, + { REVIEWS: overrides ? approval(f.source, overrides) : "[]" }, ); assert.notEqual(r.status, 0); assert.equal(f.calls(), ""); @@ -327,14 +339,15 @@ test("approved publication stages complete GitHub tuple before release and scope "bash", [ "-c", - step("Require human approval for committed release artifacts").run + + "set -e\n" + + step("Require human approval for committed release artifacts").run + "\n" + step("Upload protocol tarball to GitHub Release").run.replaceAll( "${{ steps.release-artifacts.outputs.version }}", version, ), ], - { REVIEWS: approval() }, + { REVIEWS: approval(f.source) }, ); assert.equal(r.status, 0, r.stderr); const upload = f.upload(immutable); @@ -347,6 +360,223 @@ test("approved publication stages complete GitHub tuple before release and scope assert.equal(f.upload(immutable).status, 0, "identical retries skip objects"); assert.equal(f.calls(), calls.join("\n") + "\n"); }); + +const permissionResponse = (permission, role = permission, user = {}) => + JSON.stringify({ + permission, + role_name: role, + user: { id: 123, login: "reviewer", type: "User", ...user }, + }); + +for (const [name, values] of [ + ["zero associated PRs", { ASSOCIATED_PRS: "[]" }], + [ + "multiple associated merged PRs", + { + ASSOCIATED_PRS: JSON.stringify( + [1, 2].map((number) => ({ + number, + merged_at: "2026-09-14", + base: { ref: "main" }, + })), + ), + }, + ], + [ + "duplicate associated PR", + { + ASSOCIATED_PRS: JSON.stringify( + [1, 1].map((number) => ({ + number, + merged_at: "2026-09-14", + base: { ref: "main" }, + })), + ), + }, + ], + ["malformed associated PR", { ASSOCIATED_PRS: "[null]" }], + ["malformed associated page", { ASSOCIATED_PRS: "{}" }], + ["associated PR API outage", { PULLS_ERROR: "1" }], +]) + test(`approval fails closed on ${name}`, (t) => { + const f = fixture(t); + const result = f.script("approval", { + REVIEWS: approval(f.source), + ...values, + }); + assert.notEqual(result.status, 0); + assert.equal(f.calls(), ""); + assert.equal( + fs.readFileSync(path.join(f.dir, "permission-calls"), "utf8"), + "", + ); + }); + +for (const kind of ["extra", "duplicate", "missing", "malformed"]) + test(`published authority requires exactly four signed assets: ${kind}`, (t) => { + const f = fixture(t); + const file = path.join(f.dir, "release.json"); + const release = JSON.parse(fs.readFileSync(file)); + if (kind === "extra") release.assets.push({ name: "unreviewed.txt" }); + if (kind === "duplicate") release.assets.push(release.assets[0]); + if (kind === "missing") release.assets.pop(); + if (kind === "malformed") release.assets = {}; + fs.writeFileSync(file, JSON.stringify(release)); + assert.notEqual(f.upload(immutable).status, 0); + assert.equal(f.calls(), ""); + }); + +for (const kind of ["extra", "duplicate"]) + test(`a staged release with ${kind} assets cannot become public`, (t) => { + const f = fixture(t); + const release = JSON.parse( + fs.readFileSync(path.join(f.dir, "release.json")), + ); + release.isDraft = true; + release.assets.push( + kind === "extra" ? { name: "unexpected.txt" } : release.assets[0], + ); + f.write("release.json", JSON.stringify(release)); + const result = f.run("bash", [ + "-c", + step("Upload protocol tarball to GitHub Release").run.replaceAll( + "${{ steps.release-artifacts.outputs.version }}", + version, + ), + ]); + assert.notEqual(result.status, 0); + assert.equal(f.calls(), ""); + assert.equal( + JSON.parse(fs.readFileSync(path.join(f.dir, "release.json"))).isDraft, + true, + ); + }); + +for (const [name, env] of [ + [ + "outside User with public read access", + { PERMISSION_RESPONSE: permissionResponse("read") }, + ], + ["triage", { PERMISSION_RESPONSE: permissionResponse("read", "triage") }], + ["no repository access", { PERMISSION_RESPONSE: permissionResponse("none") }], + ["missing collaborator", { PERMISSION_ERROR: "HTTP 404 Not Found" }], + ["permission denied", { PERMISSION_ERROR: "HTTP 403 Forbidden" }], + ["lookup timeout", { PERMISSION_ERROR: "Request timed out" }], + ["malformed response", { PERMISSION_RESPONSE: "not JSON" }], + [ + "missing identity", + { PERMISSION_RESPONSE: '{"permission":"admin","role_name":"admin"}' }, + ], + ["ambiguous array", { PERMISSION_RESPONSE: "[]" }], + [ + "different identity", + { PERMISSION_RESPONSE: permissionResponse("admin", "admin", { id: 999 }) }, + ], + [ + "different login", + { + PERMISSION_RESPONSE: permissionResponse("admin", "admin", { + login: "another", + }), + }, + ], + [ + "bot identity", + { + PERMISSION_RESPONSE: permissionResponse("admin", "admin", { + type: "Bot", + }), + }, + ], + [ + "contradictory role", + { PERMISSION_RESPONSE: permissionResponse("write", "read") }, + ], + [ + "missing permission", + { PERMISSION_RESPONSE: permissionResponse(undefined, "write") }, + ], + ["main drift during permission lookup", { ADVANCE_PERMISSION: "1" }], +]) { + test(`approval fails closed for ${name}`, (t) => { + const f = fixture(t); + const result = f.run( + "bash", + [ + "-c", + "set -e\n" + + step("Require human approval for committed release artifacts").run + + '\nprintf "publication attempted\\n" >> calls', + ], + { REVIEWS: approval(f.source), ...env }, + ); + assert.notEqual(result.status, 0); + assert.equal(f.calls(), ""); + assert.match( + fs.readFileSync(path.join(f.dir, "permission-calls"), "utf8"), + /\/collaborators\/reviewer\/permission/, + ); + }); +} + +for (const [permission, role] of [ + ["write", "write"], + ["write", "maintain"], + ["admin", "admin"], +]) { + test(`current ${role} maintainer approval authorizes matching release provenance`, (t) => { + const f = fixture(t); + const result = f.script("approval", { + REVIEWS: approval(f.source), + PERMISSION_RESPONSE: permissionResponse(permission, role), + }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /1 trusted maintainer approval/); + }); +} + +test("a later dismissed review invalidates the same maintainer's prior approval", (t) => { + const f = fixture(t); + const approved = JSON.parse(approval(f.source))[0]; + const dismissed = { + ...approved, + id: 2, + state: "DISMISSED", + submitted_at: "2026-09-15", + }; + const result = f.script("approval", { + REVIEWS: JSON.stringify([dismissed, approved]), + }); + assert.notEqual(result.status, 0); + assert.equal( + fs.readFileSync(path.join(f.dir, "permission-calls"), "utf8"), + "", + ); +}); + +test("a trusted approval does not hide a second reviewer's failed permission lookup", (t) => { + const f = fixture(t); + const trusted = JSON.parse(approval(f.source))[0]; + const outside = { + ...trusted, + id: 2, + user: { id: 124, type: "User", login: "outside" }, + }; + const result = f.script("approval", { + REVIEWS: JSON.stringify([trusted, outside]), + PERMISSION_ERROR_LOGIN: "outside", + PERMISSION_ERROR: "HTTP 404 Not Found", + }); + assert.notEqual(result.status, 0); + assert.equal( + fs + .readFileSync(path.join(f.dir, "permission-calls"), "utf8") + .trim() + .split("\n").length, + 2, + ); + assert.equal(f.calls(), ""); +}); for (const [name, env] of [ ["drift during put", { ADVANCE_AWS: "1" }], ["conditional create race", { PUT_RACE: "1" }], @@ -532,30 +762,178 @@ for (const suffix of ["", ".sha256", ".sig", ".crt"]) { }); } -test("a successful long Changesets push followed by drift stops before the PR API call", (t) => { +test("an existing PR updated by a stale push cannot publish even after trusted approval and merge", (t) => { const f = fixture(t); - const installed = f.run("bash", [ - "-c", - step("Fence Changesets mutations").run, - ]); - assert.equal(installed.status, 0, installed.stderr); - // GITHUB_PATH is applied to the next action exactly as Actions applies it. - const wrapperDirectory = fs - .readFileSync(path.join(f.dir, "github-path"), "utf8") - .trim(); + const pr = { + head: { sha: f.source }, + merged: false, + merge_commit_sha: null, + base: { ref: "main", repo: { full_name: "adcontextprotocol/adcp" } }, + user: { id: 456, login: "release-bot[bot]", type: "Bot" }, + }; + f.write("existing-pr.json", JSON.stringify(pr)); + + f.git("checkout", "-qb", "generated", f.head); + const next = "3.2.0-rc.4"; + f.write("package.json", JSON.stringify({ version: next })); + for (const area of ["schemas", "compliance"]) + f.write(`dist/${area}/${next}/index.json`, '{"newRelease":true}'); + for (const suffix of ["", ".sha256", ".sig", ".crt"]) + f.write(`dist/protocol/${next}.tgz${suffix}`, `signed${suffix}`); + f.git("add", "package.json", "dist"); + f.git("commit", "-qm", "Version Packages"); + const generated = f.git("rev-parse", "HEAD"); + f.git("checkout", "--detach", f.head); + f.write(".changeset/another.md", "Changeset arriving during the push"); + f.git("add", ".changeset/another.md"); + f.git("commit", "-qm", "Main advances"); + const advanced = f.git("rev-parse", "HEAD"); + f.git("checkout", "generated"); + + // Simulate the already-observed historical unfenced update. The new leased + // wrapper is separately fault-tested against a real bare Git remote. + f.git("checkout", "--detach", advanced); + f.git("merge", "--squash", generated); + f.git("commit", "-qm", "Merge the stale existing release PR"); + const merged = f.git("rev-parse", "HEAD"); + f.write("remote", merged); const result = f.run( "bash", [ "-c", - 'set -e\ngit push origin HEAD:changeset-release/main --force\nprintf "PR API write\\n" >> calls', + "set -e\n" + + step("Require human approval for committed release artifacts").run + + '\nprintf "publication attempted\\n" >> calls', ], { - PATH: `${wrapperDirectory}:${path.join(f.dir, "bin")}:${process.env.PATH}`, - ADVANCE_PUSH: "1", + TESTED_SHA: merged, + GITHUB_SHA: merged, + RELEASE_SHA: merged, + MERGED_PR: JSON.stringify({ + ...pr, + head: { sha: generated }, + number: 1, + merged: true, + merge_commit_sha: merged, + }), + REVIEWS: approval(generated), }, ); assert.notEqual(result.status, 0); - assert.equal(f.calls(), "branch push\n"); + assert.match(result.stderr, /Quarantined release/); + assert.match( + fs.readFileSync(path.join(f.dir, "permission-calls"), "utf8"), + /reviewer\/permission/, + ); + assert.equal(f.calls(), ""); +}); + +for (const kind of ["squash", "merge"]) { + test(`trusted approval accepts an unchanged generated head merged on its exact base (${kind})`, (t) => { + const f = fixture(t); + f.git("checkout", "--detach", `${f.source}^`); + if (kind === "squash") { + f.git("merge", "--squash", f.source); + f.git("commit", "-qm", "Squash approved release"); + } else f.git("merge", "--no-ff", "-m", "Merge approved release", f.source); + const merged = f.git("rev-parse", "HEAD"); + f.write("remote", merged); + const result = f.script("approval", { + TESTED_SHA: merged, + RELEASE_SHA: merged, + MERGED_PR: JSON.stringify({ + head: { sha: f.source }, + number: 1, + merged: true, + merge_commit_sha: merged, + base: { ref: "main", repo: { full_name: "adcontextprotocol/adcp" } }, + user: { id: 456, login: "release-bot[bot]", type: "Bot" }, + }), + REVIEWS: approval(f.source), + }); + assert.equal(result.status, 0, result.stderr); + }); +} + +test("matching base alone cannot authorize a merge tree changed after review", (t) => { + const f = fixture(t); + f.write(`dist/schemas/${version}/index.json`, "unreviewed change"); + f.git("add", `dist/schemas/${version}/index.json`); + const tree = f.git("write-tree"); + const merged = f.git( + "commit-tree", + tree, + "-p", + `${f.source}^`, + "-m", + "Altered merge", + ); + f.write("remote", merged); + const result = f.run( + "node", + ["scripts/check-release-state.cjs", "provenance", f.source], + { TESTED_SHA: merged, RELEASE_SHA: merged }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Quarantined release/); +}); + +test("same-version recovery cannot bypass the quarantined rc.3 merge provenance", (t) => { + // Real incident: approved e7d42cbe parent=decd95f6; 71f9cd54 parent=eb3cbd60. + const f = fixture(t); + const merged = f.git( + "commit-tree", + `${f.source}^{tree}`, + "-p", + f.head, + "-m", + "Release merged after its generation base advanced", + ); + f.write("remote", merged); + const env = { TESTED_SHA: merged, RELEASE_SHA: merged }; + const recovered = f.run( + "node", + ["scripts/check-release-state.cjs", "recovery", merged], + env, + ); + assert.equal(recovered.status, 0, recovered.stderr); + const authorized = f.script("approval", { + ...env, + MERGED_PR: JSON.stringify({ + head: { sha: f.source }, + number: 1, + merged: true, + merge_commit_sha: merged, + base: { ref: "main", repo: { full_name: "adcontextprotocol/adcp" } }, + user: { id: 456, login: "release-bot[bot]", type: "Bot" }, + }), + REVIEWS: approval(f.source), + }); + assert.notEqual(authorized.status, 0); + assert.match(authorized.stderr, /Quarantined release/); + assert.equal(f.calls(), ""); +}); + +test("a multi-parent generated head cannot substitute for fresh release generation", (t) => { + const f = fixture(t); + const head = f.git( + "commit-tree", + `${f.source}^{tree}`, + "-p", + `${f.source}^`, + "-p", + f.source, + "-m", + "Merge base into an old generated head", + ); + f.write("remote", head); + const result = f.run( + "node", + ["scripts/check-release-state.cjs", "provenance", head], + { TESTED_SHA: head, RELEASE_SHA: head }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Quarantined release/); }); for (const mismatch of [false, true]) { diff --git a/tests/release-workflow-immutability.test.cjs b/tests/release-workflow-immutability.test.cjs index f181b0debb6..6c3e0474895 100644 --- a/tests/release-workflow-immutability.test.cjs +++ b/tests/release-workflow-immutability.test.cjs @@ -240,18 +240,12 @@ assert( 'Human approval must be required whenever a commit contains release artifacts.' ); -assert( - approvalGate.includes('/commits/${RELEASE_SHA}/pulls') && - approvalGate.includes('.base.ref == $base') && - approvalGate.includes('.merged_at != null'), - 'The approval gate must resolve the merged PR associated with the release commit and branch.' -); - -assert( - approvalGate.includes('select(.user.type == "User")') && - approvalGate.includes('map(last)') && - approvalGate.includes('select(.state == "APPROVED" and .commit_id == $head)'), - 'Only human approvals submitted against the final release PR head may authorize publication.' +assert.strictEqual( + workflowConfig.jobs.release.steps.find( + step => step.name === 'Require human approval for committed release artifacts' + ).run, + 'node scripts/check-release-state.cjs approval', + 'Committed release publication must use the permission and merge-provenance gate.' ); assert( @@ -277,17 +271,20 @@ assert.deepStrictEqual( name: 'Create Release Pull Request or Tag Release', if: "steps.release-artifacts.outputs.has_release_artifacts != 'true'", id: 'changesets', + 'timeout-minutes': 40, uses: `changesets/action@${changesetsActionSha}`, with: { - 'github-token': '${{ steps.app-token.outputs.token }}', + 'github-token': '${{ steps.changesets-token.outputs.token }}', 'version-script': 'npm run version', 'commit-message': 'Version Packages', 'pr-title': 'Version Packages', + 'pr-draft': 'always', 'create-github-releases': false, 'push-with-git-cli': true, }, env: { HUSKY: '0', + GH_TOKEN: '${{ steps.changesets-token.outputs.token }}', }, }, 'Release automation must preserve the pinned Changesets v2.1.2 input contract and git-CLI push mode.' From d97795da0704fdde2f9738617e2e03269d75690c Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 18:42:16 +0000 Subject: [PATCH 7/9] fix(release): bound reconciliation token requests --- .github/workflows/release.yml | 2 ++ RELEASING.md | 9 +++++++-- tests/changesets-fence.test.cjs | 5 +++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 19b9d28f9f0..0848b9a4d56 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -191,6 +191,7 @@ jobs: - name: Mint Changesets mutation token id: changesets-token + timeout-minutes: 2 if: steps.release-artifacts.outputs.has_release_artifacts == 'false' uses: actions/create-github-app-token@v3.2.0 with: @@ -262,6 +263,7 @@ jobs: - name: Mint Changesets reconciliation token id: reconciliation-token + timeout-minutes: 2 if: always() && steps.release-artifacts.outputs.has_release_artifacts == 'false' && steps.changesets-token.outcome == 'success' uses: actions/create-github-app-token@v3.2.0 with: diff --git a/RELEASING.md b/RELEASING.md index 94e91aa4abc..a731bb48f1c 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -236,7 +236,11 @@ only prepares release PRs; it has no independent tag/publish command. Missing signatures must be fixed through review, never regenerated during publication. Release runs use `queue: max` without in-progress cancellation. GitHub permits -100 pending runs; this is not an unlimited or guaranteed delivery queue. Old +100 pending runs; this is not an unlimited or guaranteed delivery queue. +See [GitHub's concurrency contract](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency). +The exact queue/cancellation combination is regression-tested; older actionlint +parsers require only the specific unsupported `queue`-key diagnostic exception. +Old runs fail their current-branch fence. A later push with a stranded committed version fails explicitly, including an app-only push, and must use the recovery procedure below before Changesets advances the package again. @@ -266,7 +270,8 @@ Ready transition and exact-head review. Changesets receives a fresh App token after setup, with preparation and action bounds of 10 and 40 minutes. An `always()` step mints a separate reconciliation -token. Git recovery uses that token through temporary command-scoped headers, +token. Each token mint has a two-minute bound so a hung request cannot prevent +finalization. Git recovery uses that token through temporary command-scoped headers, never an expired checkout credential. Failed refresh leaves explicit unverified reconciliation evidence; it does not fall back to stale credentials. diff --git a/tests/changesets-fence.test.cjs b/tests/changesets-fence.test.cjs index eda0eafee83..9f90d93b1c2 100644 --- a/tests/changesets-fence.test.cjs +++ b/tests/changesets-fence.test.cjs @@ -786,6 +786,11 @@ test("mutation and reconciliation mint separate tokens after setup, including fa index("Fence Changesets mutations"), ); assert.match(step("Mint Changesets reconciliation token").if, /^always\(\)/); + assert.equal( + step("Mint Changesets reconciliation token")["timeout-minutes"], + 2, + ); + assert.equal(step("Mint Changesets mutation token")["timeout-minutes"], 2); assert.equal( step("Reconcile Changesets transaction").env.GH_TOKEN, "${{ steps.reconciliation-token.outputs.token }}", From aa0c4a23d07e228c38532438195c87c32278e603 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 18:49:33 +0000 Subject: [PATCH 8/9] fix(release): verify published retries without mutations --- .github/workflows/release.yml | 7 ++++++ scripts/check-release-state.cjs | 1 - tests/release-publication-ordering.test.cjs | 27 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0848b9a4d56..12da52d1916 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -353,6 +353,13 @@ jobs: fi release_state="$(gh release view "$TAG" --json isDraft,isPrerelease,targetCommitish)" + jq -e '(.isDraft | type) == "boolean"' <<< "$release_state" >/dev/null + if [ "$(jq -r '.isDraft' <<< "$release_state")" = false ]; then + # Published releases are immutable: a retry only verifies authority + # and bytes. Missing assets require a separate recovery plan. + node scripts/check-release-state.cjs published "$VERSION" + exit 0 + fi if [ "$(jq -r '.isDraft' <<< "$release_state")" = true ] && [ "$(jq -r '.targetCommitish' <<< "$release_state")" != "$RELEASE_SHA" ]; then echo '::error::Existing draft targets a different release commit.' exit 1 diff --git a/scripts/check-release-state.cjs b/scripts/check-release-state.cjs index 1f3ca1be977..f15cfb23bb3 100644 --- a/scripts/check-release-state.cjs +++ b/scripts/check-release-state.cjs @@ -283,7 +283,6 @@ function approval() { [ "write:write", "write:maintain", - "maintain:maintain", "admin:admin", ].includes(role) ) diff --git a/tests/release-publication-ordering.test.cjs b/tests/release-publication-ordering.test.cjs index 11b2d3f7015..e7cac555a9b 100644 --- a/tests/release-publication-ordering.test.cjs +++ b/tests/release-publication-ordering.test.cjs @@ -452,6 +452,29 @@ for (const kind of ["extra", "duplicate"]) ); }); +for (const kind of ["complete", "missing", "different bytes", "malformed draft flag"]) + test(`published release reuse performs no GitHub writes: ${kind}`, (t) => { + const f = fixture(t); + const release = JSON.parse(fs.readFileSync(path.join(f.dir, "release.json"))); + if (kind === "missing") release.assets.pop(); + if (kind === "malformed draft flag") release.isDraft = "false"; + f.write("release.json", JSON.stringify(release)); + const before = fs.readFileSync(path.join(f.dir, "release.json"), "utf8"); + const result = f.run("bash", [ + "-c", + step("Upload protocol tarball to GitHub Release").run.replaceAll( + "${{ steps.release-artifacts.outputs.version }}", + version, + ), + ], kind === "different bytes" ? { DIFFERING_GITHUB_ASSET: `${version}.tgz` } : {}); + if (kind === "complete") assert.equal(result.status, 0, result.stderr); + else assert.notEqual(result.status, 0); + if (kind === "missing") assert.match(result.stderr, /complete signed tuple/); + if (kind === "different bytes") assert.match(result.stderr, /differs from the tagged local tuple/); + assert.equal(f.calls(), ""); + assert.equal(fs.readFileSync(path.join(f.dir, "release.json"), "utf8"), before); + }); + for (const [name, env] of [ [ "outside User with public read access", @@ -492,6 +515,10 @@ for (const [name, env] of [ "contradictory role", { PERMISSION_RESPONSE: permissionResponse("write", "read") }, ], + [ + "unsupported maintain permission value", + { PERMISSION_RESPONSE: permissionResponse("maintain", "maintain") }, + ], [ "missing permission", { PERMISSION_RESPONSE: permissionResponse(undefined, "write") }, From 22047cc647fdcb23aebb6582476af8f1fe341e54 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Mon, 14 Sep 2026 19:07:32 +0000 Subject: [PATCH 9/9] fix(test): replace mock comment store atomically --- tests/changesets-fence.test.cjs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/changesets-fence.test.cjs b/tests/changesets-fence.test.cjs index 9f90d93b1c2..2bb50a6510d 100644 --- a/tests/changesets-fence.test.cjs +++ b/tests/changesets-fence.test.cjs @@ -191,16 +191,17 @@ function ghStub() { if (env.FAULT === "draft-lost-ack") process.exit(1); } else if (url.endsWith("/comments")) { if (env.FAULT === "hold-failure") process.exit(1); - const comments = fs.existsSync("comments.json") - ? JSON.parse(fs.readFileSync("comments.json", "utf8")) - : []; + const comments = JSON.parse(fs.readFileSync("comments.json", "utf8")); result = { id: comments.length + 1, body: args.find((arg) => arg.startsWith("body=")).slice(5), issue_url: "https://api.github.com" + url.replace(/\/comments$/, ""), }; comments.push(result); - fs.writeFileSync("comments.json", JSON.stringify(comments)); + // Each isolated fixture invokes this mock synchronously. Replace the + // initialized store atomically without an existence-check/write gap. + fs.writeFileSync("comments.next.json", JSON.stringify(comments), { flag: "wx" }); + fs.renameSync("comments.next.json", "comments.json"); if (env.FAULT === "hold-lost-ack") process.exit(1); if (env.FAULT === "hold-response") result.body = "not the hold"; if (env.FAULT === "hold-wrong-issue") @@ -284,6 +285,7 @@ function fixture(t, existing = true, pre = { mode: "pre", tag: "rc" }) { base: { ref: "main", repo: { full_name: "adcontextprotocol/adcp" } }, }; write("prs.json", JSON.stringify(existing ? [pr] : [])); + write("comments.json", "[]"); write("git-calls.jsonl", ""); write("gh-calls.jsonl", ""); for (const name of ["check-release-state.cjs", "fence-changesets.cjs"])