From f5a27ea65d33f3f7811a5b9ecef287b859f477d5 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 10:55:29 +0000 Subject: [PATCH 1/5] ci(deploy): record green machines' logs and events during the rollout Every production deploy since #9072 has failed the same way: one of the two bluegreen green machines never passes /ready, Fly rolls back after the 10-minute wait and deletes it, and its boot output and events go with it. The forensics step only runs after the rollback, so its 500-line log tail has never contained a line from the stuck machine. The Deploy step now, for the whole rollout: - streams `flyctl logs -a adcp-docs` to a file (stopped on every exit path by a trap); and - every 15s records the status (`flyctl machine status -d`), checks and events of each machine that did not exist before the deploy, replacing a snapshot only when a new one succeeds, so the last one taken before a rollback deletes the machine survives. On failure, the forensics step applies the existing secret redaction to all of it and adds fly-logs-deploy.txt (last 20,000 lines), one fly-logs-green-.txt per green machine, green-.json (state, checks, events, including any OOM exit) and green--status.txt, and lists the green machines in summary.txt. Checked with a fake flyctl: a green machine deleted mid-rollout keeps its last snapshot and boot lines, secret-shaped lines are redacted, and the log stream stops when the step exits. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 57 ++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8cef84ec15..2d35bb4298 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -262,6 +262,41 @@ jobs: before_release_version=$(flyctl releases --json | jq -er 'map(.Version) | max // 0') echo "Fly release before deployment: v${before_release_version}" + # Record what the new (green) machines do while the rollout runs. + # A green machine that never passes /ready is deleted when Fly rolls + # back, taking its boot output and events with it, and the forensics + # step below only runs afterwards. Stream app logs for the whole + # rollout, and keep the latest status and events of every machine + # that did not exist before the deploy. The forensics step filters + # and uploads them if the job fails. + live=/tmp/fly-live + mkdir -p "$live" + flyctl machines list -a adcp-docs --json | jq -r '.[].id' > "$live/before-ids.txt" || true + timeout 2400 flyctl logs -a adcp-docs > "$live/logs-raw.txt" 2>&1 & + logs_pid=$! + ( + set +e + while [ ! -f "$live/stop" ]; do + if machines=$(flyctl machines list -a adcp-docs --json 2>/dev/null); then + for id in $(echo "$machines" | jq -r '.[].id'); do + grep -qx "$id" "$live/before-ids.txt" && continue + echo "$machines" | jq --arg id "$id" '.[] | select(.id == $id) | {id, name, state, region, image_ref, created_at, updated_at, checks, events}' \ + > "$live/green-$id.json.tmp" && mv "$live/green-$id.json.tmp" "$live/green-$id.json" + flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \ + && mv "$live/green-$id-status.tmp" "$live/green-$id-status.txt" + done + fi + sleep 15 + done + ) & + watch_pid=$! + stop_capture() { + touch "$live/stop" + kill "$logs_pid" 2>/dev/null || true + wait "$watch_pid" 2>/dev/null || true + } + trap stop_capture EXIT + set +e # Keep the serving web fleet intact until every replacement passes # /ready. Update the singleton worker separately to avoid overlapping @@ -553,16 +588,34 @@ jobs: # shapes before persisting. This is best-effort — anything truly # secret-shaped that doesn't match these patterns will still slip # through, but most known leakage shapes are covered. + redact='authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ' flyctl logs -a adcp-docs --no-tail 2>&1 \ | tail -500 \ - | grep -v -iE 'authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ' \ + | grep -v -iE "$redact" \ > /tmp/forensics/fly-logs-tail.txt || true + # The rollout as the Deploy step recorded it: app logs streamed for + # the whole deploy, and the last status, checks and events of each + # green machine before a rollback deleted it. Same redaction. + live=/tmp/fly-live + if [ -s "$live/logs-raw.txt" ]; then + grep -v -iE "$redact" "$live/logs-raw.txt" | tail -20000 > /tmp/forensics/fly-logs-deploy.txt || true + fi + for f in "$live"/green-*.json "$live"/green-*-status.txt; do + [ -f "$f" ] || continue + grep -v -iE "$redact" "$f" > "/tmp/forensics/$(basename "$f")" || true + done + for f in "$live"/green-*.json; do + [ -f "$f" ] || continue + id=$(basename "$f" .json); id=${id#green-} + grep -F "[$id]" /tmp/forensics/fly-logs-deploy.txt > "/tmp/forensics/fly-logs-green-$id.txt" 2>/dev/null || true + done # Capture the deploy-outcome we computed so the artifact is # self-describing in 30 days. echo "deploy_outcome=${{ steps.deploy.outputs.deploy_outcome }}" > /tmp/forensics/summary.txt echo "github_sha=${{ github.event.workflow_run.head_sha }}" >> /tmp/forensics/summary.txt echo "github_run_id=${{ github.run_id }}" >> /tmp/forensics/summary.txt - echo "note=fly-logs-tail.txt filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only." >> /tmp/forensics/summary.txt + echo "note=fly-logs-*.txt and green-* files filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only." >> /tmp/forensics/summary.txt + echo "green_machines=$(ls "$live"/green-*.json 2>/dev/null | sed -E 's#.*/green-##; s#\.json##' | tr '\n' ' ')" >> /tmp/forensics/summary.txt ls -la /tmp/forensics/ - name: Upload Fly forensics From 955d389a9128226fa88dfda73de01bcf6a28e455 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 13:02:29 +0000 Subject: [PATCH 2/5] ci(deploy): upload only the green machines' redacted log lines Review on #9088: uploading the whole rollout's log stream (up to 20,000 lines, including the serving fleet's request logs) through a best-effort redaction regex into a 30-day artifact widened the exposure surface far beyond what the diagnosis needs. The stream now stays on the runner. On failure the forensics step uploads only each green machine's own lines (redacted, last 3,000 per machine), which are boot output from machines that serve no traffic until they pass /ready, plus their redacted status, checks and events as before. fly-logs-deploy.txt is gone. Re-checked with a fake flyctl: no serving-fleet line reaches the artifact, green machines' lines and a deleted machine's last snapshot survive, and Bearer lines are redacted. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2d35bb4298..ac64c378e3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -266,9 +266,10 @@ jobs: # A green machine that never passes /ready is deleted when Fly rolls # back, taking its boot output and events with it, and the forensics # step below only runs afterwards. Stream app logs for the whole - # rollout, and keep the latest status and events of every machine - # that did not exist before the deploy. The forensics step filters - # and uploads them if the job fails. + # rollout (kept on the runner only), and keep the latest status and + # events of every machine that did not exist before the deploy. If + # the job fails, the forensics step uploads only the green machines' + # redacted lines, status and events. live=/tmp/fly-live mkdir -p "$live" flyctl machines list -a adcp-docs --json | jq -r '.[].id' > "$live/before-ids.txt" || true @@ -593,13 +594,14 @@ jobs: | tail -500 \ | grep -v -iE "$redact" \ > /tmp/forensics/fly-logs-tail.txt || true - # The rollout as the Deploy step recorded it: app logs streamed for - # the whole deploy, and the last status, checks and events of each - # green machine before a rollback deleted it. Same redaction. + # The rollout as the Deploy step recorded it: the last status, + # checks and events of each green machine before a rollback deleted + # it, and that machine's own log lines. Only green machines' lines + # are kept: they are boot output from machines that serve no traffic + # until they pass /ready, while the stream also carries the serving + # fleet's request logs, which the diagnosis does not need. Same + # redaction, capped per machine. live=/tmp/fly-live - if [ -s "$live/logs-raw.txt" ]; then - grep -v -iE "$redact" "$live/logs-raw.txt" | tail -20000 > /tmp/forensics/fly-logs-deploy.txt || true - fi for f in "$live"/green-*.json "$live"/green-*-status.txt; do [ -f "$f" ] || continue grep -v -iE "$redact" "$f" > "/tmp/forensics/$(basename "$f")" || true @@ -607,14 +609,16 @@ jobs: for f in "$live"/green-*.json; do [ -f "$f" ] || continue id=$(basename "$f" .json); id=${id#green-} - grep -F "[$id]" /tmp/forensics/fly-logs-deploy.txt > "/tmp/forensics/fly-logs-green-$id.txt" 2>/dev/null || true + grep -F "[$id]" "$live/logs-raw.txt" 2>/dev/null \ + | grep -v -iE "$redact" \ + | tail -3000 > "/tmp/forensics/fly-logs-green-$id.txt" || true done # Capture the deploy-outcome we computed so the artifact is # self-describing in 30 days. echo "deploy_outcome=${{ steps.deploy.outputs.deploy_outcome }}" > /tmp/forensics/summary.txt echo "github_sha=${{ github.event.workflow_run.head_sha }}" >> /tmp/forensics/summary.txt echo "github_run_id=${{ github.run_id }}" >> /tmp/forensics/summary.txt - echo "note=fly-logs-*.txt and green-* files filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only." >> /tmp/forensics/summary.txt + echo "note=fly-logs-*.txt and green-* files filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only. fly-logs-green-.txt holds only that green machine's lines (last 3000)." >> /tmp/forensics/summary.txt echo "green_machines=$(ls "$live"/green-*.json 2>/dev/null | sed -E 's#.*/green-##; s#\.json##' | tr '\n' ' ')" >> /tmp/forensics/summary.txt ls -la /tmp/forensics/ From 44858386486a2306e81d77e0ef02e80f0bfa722d Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 15:37:39 +0000 Subject: [PATCH 3/5] ci(deploy): keep the deploy's own errexit toggle first in the Deploy step deploy-workflow.test.ts runs the Deploy step's flyctl calls by slicing the script from the first `set +e` to `deploy_exit=$?`. The green-machine watcher added a `set +e` of its own earlier in the script, so the slice started inside the watcher loop and ran it in the foreground forever: Server unit tests (2/4) hung until the 20-minute job timeout on both CI runs of #9088 (the test runs in 64ms on main). The watcher now uses `set +o errexit`, so the slice starts at the deploy's own toggle again. deploy-workflow.test.ts passes (24 tests). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ac64c378e3..382f3fb1a5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -276,7 +276,9 @@ jobs: timeout 2400 flyctl logs -a adcp-docs > "$live/logs-raw.txt" 2>&1 & logs_pid=$! ( - set +e + # Spelled `+o errexit` on purpose: deploy-workflow.test.ts slices + # this script from the deploy's own errexit toggle further down. + set +o errexit while [ ! -f "$live/stop" ]; do if machines=$(flyctl machines list -a adcp-docs --json 2>/dev/null); then for id in $(echo "$machines" | jq -r '.[].id'); do From df2b7b39afd98c7ff6f2b4ed7a3e596121a7798b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 15:54:44 +0000 Subject: [PATCH 4/5] ci(deploy): fail closed when the pre-deploy machine list is unavailable Review on #9088: green machines are "every machine not listed before the deploy", and `flyctl machines list ... || true` left that list empty on a transient Machines API failure, which would have classified the serving fleet as green and uploaded its request logs on a failed deploy. The list is now retried three times and must be non-empty. Without it, the step warns and captures nothing: no log stream, no watcher, so no machine's lines can reach the artifact. stop_capture tolerates either background job being absent. Simulated with a fake flyctl: with the list available only the green machines are captured and no serving-fleet line reaches the artifact; with the list failing, nothing is captured. deploy-workflow.test.ts passes (24 tests). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 64 +++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 382f3fb1a5..a6e6e0071c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -272,31 +272,49 @@ jobs: # redacted lines, status and events. live=/tmp/fly-live mkdir -p "$live" - flyctl machines list -a adcp-docs --json | jq -r '.[].id' > "$live/before-ids.txt" || true - timeout 2400 flyctl logs -a adcp-docs > "$live/logs-raw.txt" 2>&1 & - logs_pid=$! - ( - # Spelled `+o errexit` on purpose: deploy-workflow.test.ts slices - # this script from the deploy's own errexit toggle further down. - set +o errexit - while [ ! -f "$live/stop" ]; do - if machines=$(flyctl machines list -a adcp-docs --json 2>/dev/null); then - for id in $(echo "$machines" | jq -r '.[].id'); do - grep -qx "$id" "$live/before-ids.txt" && continue - echo "$machines" | jq --arg id "$id" '.[] | select(.id == $id) | {id, name, state, region, image_ref, created_at, updated_at, checks, events}' \ - > "$live/green-$id.json.tmp" && mv "$live/green-$id.json.tmp" "$live/green-$id.json" - flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \ - && mv "$live/green-$id-status.tmp" "$live/green-$id-status.txt" - done - fi - sleep 15 - done - ) & - watch_pid=$! + logs_pid="" + watch_pid="" + # Fail closed: every machine not in this list is treated as green and + # its log lines are uploaded on failure, so an empty list would sweep + # in the serving fleet's request logs. Without a non-empty list, + # capture nothing. + before_listed=false + for attempt in 1 2 3; do + if ids=$(flyctl machines list -a adcp-docs --json | jq -er '.[].id') && [ -n "$ids" ]; then + printf '%s\n' "$ids" > "$live/before-ids.txt" + before_listed=true + break + fi + sleep 5 + done + if [ "$before_listed" = true ]; then + timeout 2400 flyctl logs -a adcp-docs > "$live/logs-raw.txt" 2>&1 & + logs_pid=$! + ( + # Spelled `+o errexit` on purpose: deploy-workflow.test.ts slices + # this script from the deploy's own errexit toggle further down. + set +o errexit + while [ ! -f "$live/stop" ]; do + if machines=$(flyctl machines list -a adcp-docs --json 2>/dev/null); then + for id in $(echo "$machines" | jq -r '.[].id'); do + grep -qx "$id" "$live/before-ids.txt" && continue + echo "$machines" | jq --arg id "$id" '.[] | select(.id == $id) | {id, name, state, region, image_ref, created_at, updated_at, checks, events}' \ + > "$live/green-$id.json.tmp" && mv "$live/green-$id.json.tmp" "$live/green-$id.json" + flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \ + && mv "$live/green-$id-status.tmp" "$live/green-$id-status.txt" + done + fi + sleep 15 + done + ) & + watch_pid=$! + else + echo "::warning::Could not list the machines running before the deploy; skipping green-machine capture so serving machines are never mistaken for green ones." + fi stop_capture() { touch "$live/stop" - kill "$logs_pid" 2>/dev/null || true - wait "$watch_pid" 2>/dev/null || true + if [ -n "$logs_pid" ]; then kill "$logs_pid" 2>/dev/null || true; fi + if [ -n "$watch_pid" ]; then wait "$watch_pid" 2>/dev/null || true; fi } trap stop_capture EXIT From cd4def845843a9090df04562368822f3c4582308 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 18 Nov 2026 16:11:38 +0000 Subject: [PATCH 5/5] ci(deploy): bound the green-machine watcher so it cannot hang the deploy Review on #9088: the EXIT trap waited for the watcher, whose flyctl calls had no timeout, so a hung Machines API call during a failing deploy (the case this capture exists for) would have held the Deploy step until the job timeout, with no forensics uploaded. Each watcher flyctl call now runs under `timeout 30`, and stop_capture kills the watcher before reaping it. Snapshots are moved into place whole, so the last good one survives the kill. Simulated with a fake flyctl whose `machine status` hangs: the step exits as soon as the deploy work does, and the snapshot taken before the hang is kept. deploy-workflow.test.ts passes (24 tests). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/deploy.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a6e6e0071c..576e1810e9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -295,12 +295,14 @@ jobs: # this script from the deploy's own errexit toggle further down. set +o errexit while [ ! -f "$live/stop" ]; do - if machines=$(flyctl machines list -a adcp-docs --json 2>/dev/null); then + # Every API call is bounded, so a hung call cannot outlive the + # rollout; stop_capture also kills the loop outright. + if machines=$(timeout 30 flyctl machines list -a adcp-docs --json 2>/dev/null); then for id in $(echo "$machines" | jq -r '.[].id'); do grep -qx "$id" "$live/before-ids.txt" && continue echo "$machines" | jq --arg id "$id" '.[] | select(.id == $id) | {id, name, state, region, image_ref, created_at, updated_at, checks, events}' \ > "$live/green-$id.json.tmp" && mv "$live/green-$id.json.tmp" "$live/green-$id.json" - flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \ + timeout 30 flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \ && mv "$live/green-$id-status.tmp" "$live/green-$id-status.txt" done fi @@ -314,7 +316,10 @@ jobs: stop_capture() { touch "$live/stop" if [ -n "$logs_pid" ]; then kill "$logs_pid" 2>/dev/null || true; fi - if [ -n "$watch_pid" ]; then wait "$watch_pid" 2>/dev/null || true; fi + # Kill rather than wait for the watcher: its last good snapshots are + # already on disk (each is moved into place whole), and waiting on + # an in-flight API call must never hold up the step. + if [ -n "$watch_pid" ]; then kill "$watch_pid" 2>/dev/null || true; wait "$watch_pid" 2>/dev/null || true; fi } trap stop_capture EXIT