diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8cef84ec15..576e1810e9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -262,6 +262,67 @@ jobs: before_release_version=$(flyctl releases --json | jq -er 'map(.Version) | max // 0') echo "Fly release before deployment: v${before_release_version}" + # Record what the new (green) machines do while the rollout runs. + # A green machine that never passes /ready is deleted when Fly rolls + # back, taking its boot output and events with it, and the forensics + # step below only runs afterwards. Stream app logs for the whole + # rollout (kept on the runner only), and keep the latest status and + # events of every machine that did not exist before the deploy. If + # the job fails, the forensics step uploads only the green machines' + # redacted lines, status and events. + live=/tmp/fly-live + mkdir -p "$live" + logs_pid="" + watch_pid="" + # Fail closed: every machine not in this list is treated as green and + # its log lines are uploaded on failure, so an empty list would sweep + # in the serving fleet's request logs. Without a non-empty list, + # capture nothing. + before_listed=false + for attempt in 1 2 3; do + if ids=$(flyctl machines list -a adcp-docs --json | jq -er '.[].id') && [ -n "$ids" ]; then + printf '%s\n' "$ids" > "$live/before-ids.txt" + before_listed=true + break + fi + sleep 5 + done + if [ "$before_listed" = true ]; then + timeout 2400 flyctl logs -a adcp-docs > "$live/logs-raw.txt" 2>&1 & + logs_pid=$! + ( + # Spelled `+o errexit` on purpose: deploy-workflow.test.ts slices + # this script from the deploy's own errexit toggle further down. + set +o errexit + while [ ! -f "$live/stop" ]; do + # Every API call is bounded, so a hung call cannot outlive the + # rollout; stop_capture also kills the loop outright. + if machines=$(timeout 30 flyctl machines list -a adcp-docs --json 2>/dev/null); then + for id in $(echo "$machines" | jq -r '.[].id'); do + grep -qx "$id" "$live/before-ids.txt" && continue + echo "$machines" | jq --arg id "$id" '.[] | select(.id == $id) | {id, name, state, region, image_ref, created_at, updated_at, checks, events}' \ + > "$live/green-$id.json.tmp" && mv "$live/green-$id.json.tmp" "$live/green-$id.json" + timeout 30 flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \ + && mv "$live/green-$id-status.tmp" "$live/green-$id-status.txt" + done + fi + sleep 15 + done + ) & + watch_pid=$! + else + echo "::warning::Could not list the machines running before the deploy; skipping green-machine capture so serving machines are never mistaken for green ones." + fi + stop_capture() { + touch "$live/stop" + if [ -n "$logs_pid" ]; then kill "$logs_pid" 2>/dev/null || true; fi + # Kill rather than wait for the watcher: its last good snapshots are + # already on disk (each is moved into place whole), and waiting on + # an in-flight API call must never hold up the step. + if [ -n "$watch_pid" ]; then kill "$watch_pid" 2>/dev/null || true; wait "$watch_pid" 2>/dev/null || true; fi + } + trap stop_capture EXIT + set +e # Keep the serving web fleet intact until every replacement passes # /ready. Update the singleton worker separately to avoid overlapping @@ -553,16 +614,37 @@ jobs: # shapes before persisting. This is best-effort — anything truly # secret-shaped that doesn't match these patterns will still slip # through, but most known leakage shapes are covered. + redact='authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ' flyctl logs -a adcp-docs --no-tail 2>&1 \ | tail -500 \ - | grep -v -iE 'authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ' \ + | grep -v -iE "$redact" \ > /tmp/forensics/fly-logs-tail.txt || true + # The rollout as the Deploy step recorded it: the last status, + # checks and events of each green machine before a rollback deleted + # it, and that machine's own log lines. Only green machines' lines + # are kept: they are boot output from machines that serve no traffic + # until they pass /ready, while the stream also carries the serving + # fleet's request logs, which the diagnosis does not need. Same + # redaction, capped per machine. + live=/tmp/fly-live + for f in "$live"/green-*.json "$live"/green-*-status.txt; do + [ -f "$f" ] || continue + grep -v -iE "$redact" "$f" > "/tmp/forensics/$(basename "$f")" || true + done + for f in "$live"/green-*.json; do + [ -f "$f" ] || continue + id=$(basename "$f" .json); id=${id#green-} + grep -F "[$id]" "$live/logs-raw.txt" 2>/dev/null \ + | grep -v -iE "$redact" \ + | tail -3000 > "/tmp/forensics/fly-logs-green-$id.txt" || true + done # Capture the deploy-outcome we computed so the artifact is # self-describing in 30 days. echo "deploy_outcome=${{ steps.deploy.outputs.deploy_outcome }}" > /tmp/forensics/summary.txt echo "github_sha=${{ github.event.workflow_run.head_sha }}" >> /tmp/forensics/summary.txt echo "github_run_id=${{ github.run_id }}" >> /tmp/forensics/summary.txt - echo "note=fly-logs-tail.txt filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only." >> /tmp/forensics/summary.txt + echo "note=fly-logs-*.txt and green-* files filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only. fly-logs-green-.txt holds only that green machine's lines (last 3000)." >> /tmp/forensics/summary.txt + echo "green_machines=$(ls "$live"/green-*.json 2>/dev/null | sed -E 's#.*/green-##; s#\.json##' | tr '\n' ' ')" >> /tmp/forensics/summary.txt ls -la /tmp/forensics/ - name: Upload Fly forensics