ci: Track release pipeline failures and stuck releases in an issue

.github/scripts/check-release-staleness.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+# Reports a stuck @sentry/api release through pipeline-issue.sh. A release is
+# stuck when a "Generated" schema commit on main has waited more than a day to
+# reach npm, or when a publish request has had failed CI for more than a day.
+#
+# Run from a checkout of main with full history and tags.
+#
+# Environment:
+#   GH_TOKEN  Token that can read this repository's and the publish
+#             repository's issues, and write this repository's issues.
+# GITHUB_REPOSITORY, GITHUB_REPOSITORY_OWNER and GITHUB_SERVER_URL come from
+# GitHub Actions.
+set -eu
+
+max_age=86400
+here=$(dirname "$0")
+repo_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
+# Craft files publish requests in <owner>/publish by default.
+publish_repo="$GITHUB_REPOSITORY_OWNER/publish"
+now=$(date +%s)
+problems=
+
+package=$(jq -er .name package.json)
+latest=$(npm view "$package" version)
+
+if ! git rev-parse -q --verify "refs/tags/$latest^{commit}" >/dev/null; then
+  problems="$problems
+- \`$package@$latest\` is the newest version on npm, but this repository has no \`$latest\` tag."
+else
+  # Oldest first, so the first line is the commit that has waited longest.
+  waiting=$(git log --no-merges --reverse --grep='^Generated$' --format='%H %ct' "$latest..HEAD")
+  read -r sha committed <<EOF
+$waiting
+EOF
+  if [ -n "$sha" ] && [ $((now - committed)) -gt "$max_age" ]; then
+    count=$(git rev-list --no-merges --count --grep='^Generated$' "$latest..HEAD")
+    problems="$problems
+- $count \"Generated\" commit(s) on \`main\` are not on npm yet. The oldest, [\`$sha\`]($repo_url/commit/$sha), landed $(((now - committed) / 3600))h ago. The newest npm release is \`$package@$latest\`."
+  fi
+fi
+
+# --paginate prints one JSON array per page, so jq reads them with `inputs`.
+requests=$(gh api --paginate "repos/$publish_repo/issues?state=open&labels=ci-failed&per_page=100")
+numbers=$(jq -nr --arg p "publish: $GITHUB_REPOSITORY@" 'inputs[] | select(.title | startswith($p)) | .number' <<EOF
+$requests
+EOF
+)
+for number in $numbers; do
+  events=$(gh api --paginate "repos/$publish_repo/issues/$number/events")
+  failed_at=$(jq -n '[inputs[] | select(.event == "labeled" and .label.name == "ci-failed") | .created_at | fromdateiso8601] | max // empty' <<EOF
+$events
+EOF
+)
+  if [ -n "$failed_at" ] && [ $((now - failed_at)) -gt "$max_age" ]; then
+    problems="$problems
+- $publish_repo#$number has had failed CI for $(((now - failed_at) / 3600))h."
+  fi
+done
+
+if [ -z "$problems" ]; then
+  printf 'The %s release is up to date.\n' "$package"
+  exit 0
+fi
+
+printf 'Found a stuck release:%s\n' "$problems"
+MESSAGE="The daily staleness check found a stuck release:$problems" "$here/pipeline-issue.sh" report

.github/scripts/pipeline-issue.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+# Keeps one issue open while the @sentry/api release pipeline is broken, so a
+# stuck release gets noticed. `report` opens the issue, or comments on it when
+# it is already open. `resolve` closes it.
+#
+# Usage: pipeline-issue.sh report|resolve
+#
+# Environment:
+#   MESSAGE   Markdown for the new issue, comment, or closing comment.
+#   GH_TOKEN  Token with issues: write on this repository.
+# GITHUB_REPOSITORY comes from GitHub Actions.
+set -eu
+
+title='Release pipeline failing on main'
+# Mentioned when the issue opens, which subscribes them to every comment after.
+notify=@BYK
+
+action=${1:-}
+case "$action" in
+  report | resolve) ;;
+  *)
+    printf 'usage: %s report|resolve\n' "$0" >&2
+    exit 2
+    ;;
+esac
+
+open_issues=$(gh issue list -R "$GITHUB_REPOSITORY" --state open --limit 1000 --json number,title)
+number=$(jq -r --arg t "$title" '[.[] | select(.title == $t)] | first | .number // empty' <<EOF
+$open_issues
+EOF
+)
+
+if [ "$action" = resolve ]; then
+  if [ -z "$number" ]; then
+    printf 'No open "%s" issue to close.\n' "$title"
+    exit 0
+  fi
+  gh issue close "$number" -R "$GITHUB_REPOSITORY" --comment "$MESSAGE"
+  printf 'Closed #%s.\n' "$number"
+  exit 0
+fi
+
+if [ -n "$number" ]; then
+  gh issue comment "$number" -R "$GITHUB_REPOSITORY" --body "$MESSAGE"
+  printf 'Commented on #%s.\n' "$number"
+  exit 0
+fi
+
+body="The \`@sentry/api\` release pipeline needs attention. New failures are added here as comments, and the next successful Release run closes this issue.
+
+$MESSAGE
+
+cc $notify"
+gh issue create -R "$GITHUB_REPOSITORY" --title "$title" --body "$body"

.github/workflows/release-pipeline-health.yml

@@ -0,0 +1,74 @@
+name: Release pipeline health
+
+on:
+  workflow_run:
+    workflows:
+      - Build
+      - Release
+    types:
+      - completed
+    branches:
+      - main
+  schedule:
+    - cron: "23 8 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+# One writer at a time, so two failures cannot open two issues.
+concurrency:
+  group: release-pipeline-health
+  cancel-in-progress: false
+
+jobs:
+  # Build and Release on main: report failures, and close the issue once a
+  # Release succeeds.
+  report-run:
+    name: Report ${{ github.event.workflow_run.name }} result
+    if: >-
+      github.event_name == 'workflow_run' &&
+      github.event.workflow_run.head_repository.full_name == github.repository &&
+      (github.event.workflow_run.name == 'Release' || github.event.workflow_run.event == 'push') &&
+      (
+        github.event.workflow_run.conclusion == 'failure' ||
+        (github.event.workflow_run.name == 'Release' && github.event.workflow_run.conclusion == 'success')
+      )
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+      RUN_LINK: "[${{ github.event.workflow_run.name }} run #${{ github.event.workflow_run.run_number }}](${{ github.event.workflow_run.html_url }})"
+      COMMIT_LINK: "[`${{ github.event.workflow_run.head_sha }}`](${{ github.server_url }}/${{ github.repository }}/commit/${{ github.event.workflow_run.head_sha }})"
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+
+      - name: Report the failure
+        if: github.event.workflow_run.conclusion == 'failure'
+        env:
+          MESSAGE: ${{ env.RUN_LINK }} failed on `main` at ${{ env.COMMIT_LINK }}.
+        run: .github/scripts/pipeline-issue.sh report
+
+      - name: Close the issue
+        if: github.event.workflow_run.conclusion == 'success'
+        env:
+          MESSAGE: ${{ env.RUN_LINK }} succeeded on `main` at ${{ env.COMMIT_LINK }}.
+        run: .github/scripts/pipeline-issue.sh resolve
+
+  # Catch releases that are stuck without a failing run, such as a release
+  # branch that never merged.
+  check-staleness:
+    name: Check for a stuck release
+    if: github.event_name != 'workflow_run'
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          ref: main
+          # The check needs history and tags, but not the schema blobs.
+          fetch-depth: 0
+          filter: blob:none
+
+      - run: .github/scripts/check-release-staleness.sh