Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 84 additions & 2 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,67 @@ jobs:
before_release_version=$(flyctl releases --json | jq -er 'map(.Version) | max // 0')
echo "Fly release before deployment: v${before_release_version}"

# Record what the new (green) machines do while the rollout runs.
# A green machine that never passes /ready is deleted when Fly rolls
# back, taking its boot output and events with it, and the forensics
# step below only runs afterwards. Stream app logs for the whole
# rollout (kept on the runner only), and keep the latest status and
# events of every machine that did not exist before the deploy. If
# the job fails, the forensics step uploads only the green machines'
# redacted lines, status and events.
live=/tmp/fly-live
mkdir -p "$live"
logs_pid=""
watch_pid=""
# Fail closed: every machine not in this list is treated as green and
# its log lines are uploaded on failure, so an empty list would sweep
# in the serving fleet's request logs. Without a non-empty list,
# capture nothing.
before_listed=false
for attempt in 1 2 3; do
if ids=$(flyctl machines list -a adcp-docs --json | jq -er '.[].id') && [ -n "$ids" ]; then
printf '%s\n' "$ids" > "$live/before-ids.txt"
before_listed=true
break
fi
sleep 5
done
if [ "$before_listed" = true ]; then
timeout 2400 flyctl logs -a adcp-docs > "$live/logs-raw.txt" 2>&1 &
logs_pid=$!
(
# Spelled `+o errexit` on purpose: deploy-workflow.test.ts slices
# this script from the deploy's own errexit toggle further down.
set +o errexit
while [ ! -f "$live/stop" ]; do
# Every API call is bounded, so a hung call cannot outlive the
# rollout; stop_capture also kills the loop outright.
if machines=$(timeout 30 flyctl machines list -a adcp-docs --json 2>/dev/null); then
for id in $(echo "$machines" | jq -r '.[].id'); do
grep -qx "$id" "$live/before-ids.txt" && continue
echo "$machines" | jq --arg id "$id" '.[] | select(.id == $id) | {id, name, state, region, image_ref, created_at, updated_at, checks, events}' \
> "$live/green-$id.json.tmp" && mv "$live/green-$id.json.tmp" "$live/green-$id.json"
timeout 30 flyctl machine status -d "$id" -a adcp-docs > "$live/green-$id-status.tmp" 2>&1 \
&& mv "$live/green-$id-status.tmp" "$live/green-$id-status.txt"
done
fi
sleep 15
done
) &
watch_pid=$!
else
echo "::warning::Could not list the machines running before the deploy; skipping green-machine capture so serving machines are never mistaken for green ones."
fi
stop_capture() {
touch "$live/stop"
if [ -n "$logs_pid" ]; then kill "$logs_pid" 2>/dev/null || true; fi
# Kill rather than wait for the watcher: its last good snapshots are
# already on disk (each is moved into place whole), and waiting on
# an in-flight API call must never hold up the step.
if [ -n "$watch_pid" ]; then kill "$watch_pid" 2>/dev/null || true; wait "$watch_pid" 2>/dev/null || true; fi
}
trap stop_capture EXIT

set +e
# Keep the serving web fleet intact until every replacement passes
# /ready. Update the singleton worker separately to avoid overlapping
Expand Down Expand Up @@ -553,16 +614,37 @@ jobs:
# shapes before persisting. This is best-effort — anything truly
# secret-shaped that doesn't match these patterns will still slip
# through, but most known leakage shapes are covered.
redact='authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ'
flyctl logs -a adcp-docs --no-tail 2>&1 \
| tail -500 \
| grep -v -iE 'authorization|bearer |authn=|secret|password|api[-_]?key|sk-[a-z0-9]|wos_[a-z0-9]|fly_[a-z0-9]{20,}|eyJ[A-Za-z0-9_-]{10,}\.eyJ' \
| grep -v -iE "$redact" \
> /tmp/forensics/fly-logs-tail.txt || true
# The rollout as the Deploy step recorded it: the last status,
# checks and events of each green machine before a rollback deleted
# it, and that machine's own log lines. Only green machines' lines
# are kept: they are boot output from machines that serve no traffic
# until they pass /ready, while the stream also carries the serving
# fleet's request logs, which the diagnosis does not need. Same
# redaction, capped per machine.
live=/tmp/fly-live
for f in "$live"/green-*.json "$live"/green-*-status.txt; do
[ -f "$f" ] || continue
grep -v -iE "$redact" "$f" > "/tmp/forensics/$(basename "$f")" || true
done
for f in "$live"/green-*.json; do
[ -f "$f" ] || continue
id=$(basename "$f" .json); id=${id#green-}
grep -F "[$id]" "$live/logs-raw.txt" 2>/dev/null \
| grep -v -iE "$redact" \
| tail -3000 > "/tmp/forensics/fly-logs-green-$id.txt" || true
done
# Capture the deploy-outcome we computed so the artifact is
# self-describing in 30 days.
echo "deploy_outcome=${{ steps.deploy.outputs.deploy_outcome }}" > /tmp/forensics/summary.txt
echo "github_sha=${{ github.event.workflow_run.head_sha }}" >> /tmp/forensics/summary.txt
echo "github_run_id=${{ github.run_id }}" >> /tmp/forensics/summary.txt
echo "note=fly-logs-tail.txt filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only." >> /tmp/forensics/summary.txt
echo "note=fly-logs-*.txt and green-* files filtered for common secret shapes (Bearer/sk-/wos_/JWT/etc). Not exhaustive — treat as read-only. fly-logs-green-<id>.txt holds only that green machine's lines (last 3000)." >> /tmp/forensics/summary.txt
echo "green_machines=$(ls "$live"/green-*.json 2>/dev/null | sed -E 's#.*/green-##; s#\.json##' | tr '\n' ' ')" >> /tmp/forensics/summary.txt
ls -la /tmp/forensics/

- name: Upload Fly forensics
Expand Down
Loading