From d8972f39f2ad631a72afdfa0179cee2c0241e90b Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Tue, 17 Nov 2026 00:15:27 +0000 Subject: [PATCH 1/5] ci: track release verification failures per tag without repeated comments --- .github/workflows/release-verify.yaml | 41 +++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release-verify.yaml b/.github/workflows/release-verify.yaml index 1f5d15f4c09..6b3d143f824 100644 --- a/.github/workflows/release-verify.yaml +++ b/.github/workflows/release-verify.yaml @@ -35,19 +35,36 @@ jobs: RELEASE_TAG: ${{ inputs.tag }} run: | CUTOFF=$(date -u -d "-14 days" +"%Y-%m-%dT%H:%M:%SZ") - TAGS=${RELEASE_TAG:-$(gh api "repos/$GH_REPO/releases?per_page=20" --jq ".[] | select(.published_at >= \"$CUTOFF\" and ((.body // \"\") | contains(\"## Release Verification: ✅ Verified\") | not)) | .tag_name")} + TAGS=${RELEASE_TAG:-$(gh api "repos/$GH_REPO/releases?per_page=20" | jq -r --arg cutoff "$CUTOFF" '.[] | select(.published_at >= $cutoff and ((.body // "") | contains("## Release Verification: ✅ Verified") | not)) | .tag_name')} FAILED=0 for TAG in $TAGS; do - ./dev/google-cloud release:verify "$TAG" --update-release -t "$GH_TOKEN" || FAILED=1 + TITLE="Release verification failed: $TAG" + if ./dev/google-cloud release:verify "$TAG" --update-release -t "$GH_TOKEN"; then + if [ "$GITHUB_REF" = "refs/heads/main" ]; then + OPEN_ISSUE=$(gh issue list --repo "$GH_REPO" --search "\"$TITLE\" in:title" --state open --json number,title | jq -r --arg title "$TITLE" 'first(.[] | select(.title == $title) | .number) // empty') + if [ -n "$OPEN_ISSUE" ]; then + IS_VERIFIED=$(gh api "repos/$GH_REPO/releases/tags/$TAG" --jq '(.body // "") | contains("## Release Verification: ✅ Verified")') + if [ "$IS_VERIFIED" = "true" ]; then + gh issue close "$OPEN_ISSUE" --repo "$GH_REPO" --comment "Release $TAG is now verified. Workflow Run: $GITHUB_SERVER_URL/$GH_REPO/actions/runs/$GITHUB_RUN_ID" + fi + fi + fi + else + FAILED=1 + if [ "$GITHUB_REF" = "refs/heads/main" ]; then + EXISTING_ISSUE=$(gh issue list --repo "$GH_REPO" --search "\"$TITLE\" in:title" --state all --json number,title | jq -r --arg title "$TITLE" 'first(.[] | select(.title == $title) | .number) // empty') + if [ -z "$EXISTING_ISSUE" ]; then + BODY=$(printf '`release:verify` failed for release %s.\n\n- Release: %s/%s/releases/tag/%s\n- Workflow Run: %s/%s/actions/runs/%s\n\n@googleapis/cloud-sdk-php-team' \ + "$TAG" "$GITHUB_SERVER_URL" "$GH_REPO" "$TAG" "$GITHUB_SERVER_URL" "$GH_REPO" "$GITHUB_RUN_ID") + gh issue create \ + --repo "$GH_REPO" \ + --title "$TITLE" \ + --label ":rotating_light: critical" \ + --body "$BODY" + else + echo "Issue #$EXISTING_ISSUE already exists for $TAG; skipping." + fi + fi + fi done exit $FAILED - - name: Create issue if previous step fails - if: ${{ failure() && github.ref == 'refs/heads/main' }} - uses: googleapis/librarian/.github/actions/create-issue-on-failure@3c8cc1528686a089cace64bec5f29642c328e3b8 # v0.51.0 - with: - title: "Release verification failed" - body: | - `release:verify` failed for a recent release. - - Please check the logs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - mention: "@googleapis/cloud-sdk-php-team" From a3915cc0994d67041996253b53026829407ab1b4 Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Tue, 17 Nov 2026 00:19:42 +0000 Subject: [PATCH 2/5] fix(dev): skip repo settings compliance check for new components --- dev/src/Command/RepoComplianceCommand.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/src/Command/RepoComplianceCommand.php b/dev/src/Command/RepoComplianceCommand.php index 83a0fec5f66..7bda12ae227 100644 --- a/dev/src/Command/RepoComplianceCommand.php +++ b/dev/src/Command/RepoComplianceCommand.php @@ -106,7 +106,8 @@ protected function execute(InputInterface $input, OutputInterface $output) } $repoCheck = $packagistCheck = $webhookCheck = $teamsCheck = true; if (!$this->checkSettingsCompliance($component, $details)) { - $repoCheck = false; + // New components are configured by repo:split on first release. + $repoCheck = $isNewComponent ? 'skipped' : false; $refreshDetails |= $this->askFixSettingsCompliance($input, $output, $component, $details); } if (!$this->checkWebhookCompliance($details)) { From 68ae8cb1d81dfff10f8d47dd94ee6f3b8e312cfd Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Tue, 17 Nov 2026 01:34:02 +0000 Subject: [PATCH 3/5] ci: simplify release-verify issue creation and auto-close on pass --- .github/workflows/release-verify.yaml | 53 ++++++++++++--------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release-verify.yaml b/.github/workflows/release-verify.yaml index 6b3d143f824..096c0806ffb 100644 --- a/.github/workflows/release-verify.yaml +++ b/.github/workflows/release-verify.yaml @@ -35,36 +35,31 @@ jobs: RELEASE_TAG: ${{ inputs.tag }} run: | CUTOFF=$(date -u -d "-14 days" +"%Y-%m-%dT%H:%M:%SZ") - TAGS=${RELEASE_TAG:-$(gh api "repos/$GH_REPO/releases?per_page=20" | jq -r --arg cutoff "$CUTOFF" '.[] | select(.published_at >= $cutoff and ((.body // "") | contains("## Release Verification: ✅ Verified") | not)) | .tag_name')} + TAGS=${RELEASE_TAG:-$(gh api "repos/$GH_REPO/releases?per_page=20" --jq ".[] | select(.published_at >= \"$CUTOFF\" and ((.body // \"\") | contains(\"## Release Verification: ✅ Verified\") | not)) | .tag_name")} FAILED=0 for TAG in $TAGS; do - TITLE="Release verification failed: $TAG" - if ./dev/google-cloud release:verify "$TAG" --update-release -t "$GH_TOKEN"; then - if [ "$GITHUB_REF" = "refs/heads/main" ]; then - OPEN_ISSUE=$(gh issue list --repo "$GH_REPO" --search "\"$TITLE\" in:title" --state open --json number,title | jq -r --arg title "$TITLE" 'first(.[] | select(.title == $title) | .number) // empty') - if [ -n "$OPEN_ISSUE" ]; then - IS_VERIFIED=$(gh api "repos/$GH_REPO/releases/tags/$TAG" --jq '(.body // "") | contains("## Release Verification: ✅ Verified")') - if [ "$IS_VERIFIED" = "true" ]; then - gh issue close "$OPEN_ISSUE" --repo "$GH_REPO" --comment "Release $TAG is now verified. Workflow Run: $GITHUB_SERVER_URL/$GH_REPO/actions/runs/$GITHUB_RUN_ID" - fi - fi - fi - else - FAILED=1 - if [ "$GITHUB_REF" = "refs/heads/main" ]; then - EXISTING_ISSUE=$(gh issue list --repo "$GH_REPO" --search "\"$TITLE\" in:title" --state all --json number,title | jq -r --arg title "$TITLE" 'first(.[] | select(.title == $title) | .number) // empty') - if [ -z "$EXISTING_ISSUE" ]; then - BODY=$(printf '`release:verify` failed for release %s.\n\n- Release: %s/%s/releases/tag/%s\n- Workflow Run: %s/%s/actions/runs/%s\n\n@googleapis/cloud-sdk-php-team' \ - "$TAG" "$GITHUB_SERVER_URL" "$GH_REPO" "$TAG" "$GITHUB_SERVER_URL" "$GH_REPO" "$GITHUB_RUN_ID") - gh issue create \ - --repo "$GH_REPO" \ - --title "$TITLE" \ - --label ":rotating_light: critical" \ - --body "$BODY" - else - echo "Issue #$EXISTING_ISSUE already exists for $TAG; skipping." - fi - fi - fi + ./dev/google-cloud release:verify "$TAG" --update-release -t "$GH_TOKEN" || FAILED=1 done exit $FAILED + - name: Create issue if previous step fails + if: ${{ failure() && github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + run: | + OPEN=$(gh issue list --repo "$GH_REPO" --search '"Release verification failed" in:title' --state open --json number --jq '.[0].number // empty') + if [ -z "$OPEN" ]; then + gh issue create \ + --repo "$GH_REPO" \ + --title "Release verification failed" \ + --label ":rotating_light: critical" \ + --body "$(printf '`release:verify` failed for a recent release.\n\nPlease check the logs: %s/%s/actions/runs/%s\n\n@googleapis/cloud-sdk-php-team' "$GITHUB_SERVER_URL" "$GH_REPO" "$GITHUB_RUN_ID")" + fi + - name: Close issue if previous step succeeds + if: ${{ success() && github.ref == 'refs/heads/main' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + run: | + gh issue list --repo "$GH_REPO" --search '"Release verification failed" in:title' --state open --json number --jq '.[0].number // empty' \ + | xargs -r -I{} gh issue close {} --repo "$GH_REPO" --comment "Release verification passed. Logs: $GITHUB_SERVER_URL/$GH_REPO/actions/runs/$GITHUB_RUN_ID" From c7e6094545731affae01ec4f5694bb8c9a2b4a3c Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Wed, 18 Nov 2026 19:49:17 +0000 Subject: [PATCH 4/5] ci: remove release:verify step from split release workflow --- .github/workflows/release.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9ea117f4de9..7d6f519746f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,14 +72,3 @@ jobs: --packagist-username="${VARS_PACKAGIST_USERNAME}" \ --packagist-token="$PACKAGIST_TOKEN" \ --packagist-safe-token="$PACKAGIST_SAFE_TOKEN" - - name: Verify all releases made it to Packagist - env: - RELEASE_TAG: ${{ inputs.tag != '' && inputs.tag || steps.getTag.outputs.tag }} - SPLIT_TOKEN: ${{ secrets.SPLIT_TOKEN }} - uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 - with: - command: | - ./dev/google-cloud release:verify "$RELEASE_TAG" -t "$SPLIT_TOKEN" - timeout_minutes: 1 - max_attempts: 3 - retry_wait_seconds: 30 From 4568c8c81cb7a6ec95e8c158ca8c4abfd050bf3d Mon Sep 17 00:00:00 2001 From: Brent Shaffer Date: Wed, 18 Nov 2026 20:43:52 +0000 Subject: [PATCH 5/5] feat(ci): trigger release-verify after split, append failure table, and ignore workflow check runs --- .github/workflows/release-verify.yaml | 19 +++++++++++++++---- dev/src/Command/ReleaseVerifyCommand.php | 7 +++++++ .../Unit/Command/ReleaseVerifyCommandTest.php | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-verify.yaml b/.github/workflows/release-verify.yaml index 096c0806ffb..a0d8478c6be 100644 --- a/.github/workflows/release-verify.yaml +++ b/.github/workflows/release-verify.yaml @@ -2,6 +2,10 @@ name: Verify Releases on: schedule: - cron: '0 9 * * *' + # zizmor: ignore[dangerous-triggers] -- triggered only by maintainer-dispatched Release Components and checks out default branch + workflow_run: + workflows: ["Release Components"] + types: [completed] workflow_dispatch: inputs: tag: @@ -41,19 +45,26 @@ jobs: ./dev/google-cloud release:verify "$TAG" --update-release -t "$GH_TOKEN" || FAILED=1 done exit $FAILED - - name: Create issue if previous step fails + - name: Create or update issue if previous step fails if: ${{ failure() && github.ref == 'refs/heads/main' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} run: | - OPEN=$(gh issue list --repo "$GH_REPO" --search '"Release verification failed" in:title' --state open --json number --jq '.[0].number // empty') - if [ -z "$OPEN" ]; then + ROW="| $(date -u +'%Y-%m-%d %H:%M UTC') | [#$GITHUB_RUN_ID]($GITHUB_SERVER_URL/$GH_REPO/actions/runs/$GITHUB_RUN_ID) |" + ISSUE=$(gh issue list --repo "$GH_REPO" --search '"Release verification failed" in:title' --state open --json number,body --jq '.[0] // empty') + if [ -z "$ISSUE" ]; then + BODY=$(printf '`release:verify` failed for a recent release.\n\n@googleapis/cloud-sdk-php-team\n\n| Date | Workflow Run |\n| --- | --- |\n%s' "$ROW") gh issue create \ --repo "$GH_REPO" \ --title "Release verification failed" \ --label ":rotating_light: critical" \ - --body "$(printf '`release:verify` failed for a recent release.\n\nPlease check the logs: %s/%s/actions/runs/%s\n\n@googleapis/cloud-sdk-php-team' "$GITHUB_SERVER_URL" "$GH_REPO" "$GITHUB_RUN_ID")" + --body "$BODY" + else + NUM=$(jq -r '.number' <<<"$ISSUE") + BODY=$(jq -r '.body' <<<"$ISSUE") + [[ "$BODY" == *"| Date | Workflow Run |"* ]] || BODY="$BODY"$'\n\n| Date | Workflow Run |\n| --- | --- |' + gh api "repos/$GH_REPO/issues/$NUM" -X PATCH -f body="$BODY"$'\n'"$ROW" fi - name: Close issue if previous step succeeds if: ${{ success() && github.ref == 'refs/heads/main' }} diff --git a/dev/src/Command/ReleaseVerifyCommand.php b/dev/src/Command/ReleaseVerifyCommand.php index 927db3aafc8..3a58da57ae0 100644 --- a/dev/src/Command/ReleaseVerifyCommand.php +++ b/dev/src/Command/ReleaseVerifyCommand.php @@ -42,6 +42,10 @@ class ReleaseVerifyCommand extends Command private const TABLE_HEADERS = ['Component', 'Version', 'Packagist', 'Docs']; private const VERIFY_MARKER_START = ''; private const VERIFY_MARKER_END = ''; + private const IGNORED_CHECK_RUNS = [ + 'Split and Release Components', + 'Verify Releases', + ]; private Client $httpClient; @@ -307,6 +311,9 @@ private function verifyReleaseCheckRuns( $checkRuns = $github->getCheckRuns(self::REPO_ID, $tag) ?? []; foreach ($checkRuns as $run) { $runName = $run['name'] ?? 'Unknown'; + if (in_array($runName, self::IGNORED_CHECK_RUNS, true)) { + continue; + } $runUrl = $run['html_url'] ?? ''; $markdownLabel = $runUrl ? sprintf('[%s](%s)', $runName, $runUrl) : $runName; diff --git a/dev/tests/Unit/Command/ReleaseVerifyCommandTest.php b/dev/tests/Unit/Command/ReleaseVerifyCommandTest.php index 3aba369db6c..201b77ac760 100644 --- a/dev/tests/Unit/Command/ReleaseVerifyCommandTest.php +++ b/dev/tests/Unit/Command/ReleaseVerifyCommandTest.php @@ -67,6 +67,18 @@ public function testVerifyTagSuccess() 'conclusion' => 'success', 'html_url' => 'https://github.com/googleapis/google-cloud-php/actions/runs/1/job/1', ], + [ + 'name' => 'Verify Releases', + 'status' => 'in_progress', + 'conclusion' => null, + 'html_url' => 'https://github.com/googleapis/google-cloud-php/actions/runs/2/job/1', + ], + [ + 'name' => 'Split and Release Components', + 'status' => 'completed', + 'conclusion' => 'failure', + 'html_url' => 'https://github.com/googleapis/google-cloud-php/actions/runs/3/job/1', + ], ], ]))); @@ -125,6 +137,8 @@ public function testVerifyTagSuccess() $display = $commandTester->getDisplay(); $this->assertStringContainsString('v0.349.0', $display); $this->assertStringContainsString('Coding Standards', $display); + $this->assertStringNotContainsString('Verify Releases', $display); + $this->assertStringNotContainsString('Split and Release Components', $display); $this->assertStringContainsString('google/cloud-billing-budgets', $display); $this->assertStringContainsString('google/cloud-build', $display); $this->assertStringContainsString('google/common-protos', $display);