From 1dfe322cf33f9cd76c3a7655d039e5aeb4394217 Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 16:22:27 +0200 Subject: [PATCH 01/19] Generalize mobile-scan to platform-agnostic CI outer-loop scanner Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...scan.lock.yml => ci-failure-scan.lock.yml} | 90 +++---- .github/workflows/ci-failure-scan.md | 252 ++++++++++++++++++ .github/workflows/mobile-scan.md | 160 ----------- 3 files changed, 297 insertions(+), 205 deletions(-) rename .github/workflows/{mobile-scan.lock.yml => ci-failure-scan.lock.yml} (93%) create mode 100644 .github/workflows/ci-failure-scan.md delete mode 100644 .github/workflows/mobile-scan.md diff --git a/.github/workflows/mobile-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml similarity index 93% rename from .github/workflows/mobile-scan.lock.yml rename to .github/workflows/ci-failure-scan.lock.yml index 23d4822e27064b..e536b54e9a8a95 100644 --- a/.github/workflows/mobile-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"f77456f33b61a0a920cfcf29a2ff065f923c2d75ef2c2d46909816fd0df9776e","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.5"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"70808df580451921c69c549032f52027be0ddff9d008bca44c1ee90ab2025be5","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.7"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc","version":"v0.68.1"}]} # ___ _ _ # / _ \ | | (_) @@ -22,7 +22,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Daily scan of the runtime-extra-platforms pipeline for Apple mobile and Android failures. Fixes per-test failures via PR; files an actionable tracking issue otherwise. +# Periodic platform-agnostic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Fixes per-test failures via PR; files an actionable tracking issue otherwise. # # Secrets used: # - COPILOT_GITHUB_TOKEN @@ -48,15 +48,15 @@ # - actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 # - github/gh-aw-actions/setup@2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc # v0.68.1 -name: "Mobile Platform Failure Scanner" +name: "CI Outer-Loop Failure Scanner" "on": # roles: # Roles processed as role check in pre-activation job # - admin # Roles processed as role check in pre-activation job # - maintainer # Roles processed as role check in pre-activation job # - write # Roles processed as role check in pre-activation job schedule: - - cron: "40 10 * * *" - # Friendly format: daily (scattered) + - cron: "34 */6 * * *" + # Friendly format: every 6h (scattered) # steps: # Steps injected into pre-activation job # - name: Checkout the select-copilot-pat action folder # uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -91,9 +91,9 @@ permissions: {} concurrency: cancel-in-progress: true - group: mobile-scan + group: ci-failure-scan -run-name: "Mobile Platform Failure Scanner" +run-name: "CI Outer-Loop Failure Scanner" jobs: activation: @@ -124,11 +124,11 @@ jobs: env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: "claude-sonnet-4.5" + GH_AW_INFO_MODEL: "claude-opus-4.7" GH_AW_INFO_VERSION: "1.0.21" GH_AW_INFO_AGENT_VERSION: "1.0.21" GH_AW_INFO_CLI_VERSION: "v0.68.1" - GH_AW_INFO_WORKFLOW_NAME: "Mobile Platform Failure Scanner" + GH_AW_INFO_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" @@ -163,7 +163,7 @@ jobs: id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: - GH_AW_WORKFLOW_FILE: "mobile-scan.lock.yml" + GH_AW_WORKFLOW_FILE: "ci-failure-scan.lock.yml" GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | @@ -197,19 +197,19 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_5b32ebfb2ce4676c_EOF' + cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' - GH_AW_PROMPT_5b32ebfb2ce4676c_EOF + GH_AW_PROMPT_1ab483264300fa84_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_5b32ebfb2ce4676c_EOF' + cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' - Tools: create_issue(max:3), create_pull_request(max:5), missing_tool, missing_data, noop - GH_AW_PROMPT_5b32ebfb2ce4676c_EOF + Tools: create_issue(max:5), create_pull_request(max:10), missing_tool, missing_data, noop + GH_AW_PROMPT_1ab483264300fa84_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_5b32ebfb2ce4676c_EOF' + cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' The following GitHub context information is available for this workflow: @@ -242,12 +242,12 @@ jobs: - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). - GH_AW_PROMPT_5b32ebfb2ce4676c_EOF + GH_AW_PROMPT_1ab483264300fa84_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_5b32ebfb2ce4676c_EOF' + cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' - {{#runtime-import .github/workflows/mobile-scan.md}} - GH_AW_PROMPT_5b32ebfb2ce4676c_EOF + {{#runtime-import .github/workflows/ci-failure-scan.md}} + GH_AW_PROMPT_1ab483264300fa84_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -331,7 +331,7 @@ jobs: GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_WORKFLOW_ID_SANITIZED: mobilescan + GH_AW_WORKFLOW_ID_SANITIZED: cifailurescan outputs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} @@ -413,16 +413,16 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_9c9d9c81bf97b513_EOF' - {"create_issue":{"labels":["agentic-workflows"],"max":3},"create_pull_request":{"allowed_files":["src/libraries/**/tests/**","src/libraries/Common/tests/**"],"draft":true,"labels":["agentic-workflows"],"max":5,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[mobile] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_9c9d9c81bf97b513_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_1d0cbd0432f58d3e_EOF' + {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**/tests/**","src/libraries/**/src/**","src/libraries/Common/tests/**","src/libraries/Common/src/**","src/coreclr/**/tests/**","src/mono/**/tests/**","src/tests/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_1d0cbd0432f58d3e_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | { "description_suffixes": { - "create_issue": " CONSTRAINTS: Maximum 3 issue(s) can be created. Labels [\"agentic-workflows\"] will be automatically added.", - "create_pull_request": " CONSTRAINTS: Maximum 5 pull request(s) can be created. Title will be prefixed with \"[mobile] \". Labels [\"agentic-workflows\"] will be automatically added. PRs will be created as drafts." + "create_issue": " CONSTRAINTS: Maximum 5 issue(s) can be created. Labels [\"agentic-workflows\"] will be automatically added.", + "create_pull_request": " CONSTRAINTS: Maximum 10 pull request(s) can be created. Title will be prefixed with \"[ci-scan] \". Labels [\"agentic-workflows\"] will be automatically added. PRs will be created as drafts." }, "repo_params": {}, "dynamic_tools": [] @@ -645,7 +645,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_17f3452edca3354e_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_fabcc418e0385e62_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -689,7 +689,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_17f3452edca3354e_EOF + GH_AW_MCP_CONFIG_fabcc418e0385e62_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -743,7 +743,7 @@ jobs: # --allow-tool shell(xargs) # --allow-tool shell(yq) # --allow-tool write - timeout-minutes: 60 + timeout-minutes: 90 run: | set -o pipefail touch /tmp/gh-aw/agent-step-summary.md @@ -754,7 +754,7 @@ jobs: env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} - COPILOT_MODEL: claude-sonnet-4.5 + COPILOT_MODEL: claude-opus-4.7 GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt @@ -954,7 +954,7 @@ jobs: issues: write pull-requests: write concurrency: - group: "gh-aw-conclusion-mobile-scan" + group: "gh-aw-conclusion-ci-failure-scan" cancel-in-progress: false outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} @@ -989,7 +989,7 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_NOOP_MAX: "1" - GH_AW_WORKFLOW_NAME: "Mobile Platform Failure Scanner" + GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} GH_AW_NOOP_REPORT_AS_ISSUE: "true" @@ -1006,7 +1006,7 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_MISSING_TOOL_CREATE_ISSUE: "true" - GH_AW_WORKFLOW_NAME: "Mobile Platform Failure Scanner" + GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | @@ -1020,7 +1020,7 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_REPORT_INCOMPLETE_CREATE_ISSUE: "true" - GH_AW_WORKFLOW_NAME: "Mobile Platform Failure Scanner" + GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | @@ -1034,10 +1034,10 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} - GH_AW_WORKFLOW_NAME: "Mobile Platform Failure Scanner" + GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "mobile-scan" + GH_AW_WORKFLOW_ID: "ci-failure-scan" GH_AW_ENGINE_ID: "copilot" GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} @@ -1048,7 +1048,7 @@ jobs: GH_AW_STALE_LOCK_FILE_FAILED: ${{ needs.activation.outputs.stale_lock_file_failed }} GH_AW_GROUP_REPORTS: "false" GH_AW_FAILURE_REPORT_AS_ISSUE: "true" - GH_AW_TIMEOUT_MINUTES: "60" + GH_AW_TIMEOUT_MINUTES: "90" with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} script: | @@ -1137,8 +1137,8 @@ jobs: if: always() && steps.detection_guard.outputs.run_detection == 'true' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: - WORKFLOW_NAME: "Mobile Platform Failure Scanner" - WORKFLOW_DESCRIPTION: "Daily scan of the runtime-extra-platforms pipeline for Apple mobile and Android failures. Fixes per-test failures via PR; files an actionable tracking issue otherwise." + WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" + WORKFLOW_DESCRIPTION: "Periodic platform-agnostic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Fixes per-test failures via PR; files an actionable tracking issue otherwise." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | @@ -1172,7 +1172,7 @@ jobs: env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} - COPILOT_MODEL: claude-sonnet-4.5 + COPILOT_MODEL: claude-opus-4.7 GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: v0.68.1 @@ -1270,12 +1270,12 @@ jobs: pull-requests: write timeout-minutes: 15 env: - GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/mobile-scan" + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/ci-failure-scan" GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: "claude-sonnet-4.5" - GH_AW_WORKFLOW_ID: "mobile-scan" - GH_AW_WORKFLOW_NAME: "Mobile Platform Failure Scanner" + GH_AW_ENGINE_MODEL: "claude-opus-4.7" + GH_AW_WORKFLOW_ID: "ci-failure-scan" + GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" outputs: code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} @@ -1354,7 +1354,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dev.azure.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,helix.dot.net,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":3},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**/tests/**\",\"src/libraries/Common/tests/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":5,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[mobile] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**/tests/**\",\"src/libraries/**/src/**\",\"src/libraries/Common/tests/**\",\"src/libraries/Common/src/**\",\"src/coreclr/**/tests/**\",\"src/mono/**/tests/**\",\"src/tests/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md new file mode 100644 index 00000000000000..3acc95a8f7ba5a --- /dev/null +++ b/.github/workflows/ci-failure-scan.md @@ -0,0 +1,252 @@ +--- +name: "CI Outer-Loop Failure Scanner" +description: "Periodic platform-agnostic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Fixes per-test failures via PR; files an actionable tracking issue otherwise." + +permissions: + contents: read + issues: read + pull-requests: read + +on: + schedule: every 6h + workflow_dispatch: + roles: [admin, maintainer, write] + +# ############################################################### +# Override the COPILOT_GITHUB_TOKEN secret usage for the workflow +# with a randomly-selected token from a pool of secrets. +# +# As soon as organization-level billing is offered for Agentic +# Workflows, this stop-gap approach will be removed. +# +# See: /.github/actions/select-copilot-pat/README.md +# ############################################################### + + # Add the pre-activation step of selecting a random PAT from the supplied secrets + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + name: Checkout the select-copilot-pat action folder + with: + persist-credentials: false + sparse-checkout: .github/actions/select-copilot-pat + sparse-checkout-cone-mode: true + fetch-depth: 1 + + - id: select-copilot-pat + name: Select Copilot token from pool + uses: ./.github/actions/select-copilot-pat + env: + SECRET_0: ${{ secrets.COPILOT_PAT_0 }} + SECRET_1: ${{ secrets.COPILOT_PAT_1 }} + SECRET_2: ${{ secrets.COPILOT_PAT_2 }} + SECRET_3: ${{ secrets.COPILOT_PAT_3 }} + SECRET_4: ${{ secrets.COPILOT_PAT_4 }} + SECRET_5: ${{ secrets.COPILOT_PAT_5 }} + SECRET_6: ${{ secrets.COPILOT_PAT_6 }} + SECRET_7: ${{ secrets.COPILOT_PAT_7 }} + SECRET_8: ${{ secrets.COPILOT_PAT_8 }} + SECRET_9: ${{ secrets.COPILOT_PAT_9 }} + +# Add the pre-activation output of the randomly selected PAT +jobs: + pre-activation: + outputs: + copilot_pat_number: ${{ steps.select-copilot-pat.outputs.copilot_pat_number }} + +# Override the COPILOT_GITHUB_TOKEN expression used in the activation job +# Consume the PAT number from the pre-activation step and select the corresponding secret +engine: + id: copilot + model: claude-opus-4.7 + env: + # We cannot use line breaks in this expression as it leads to a syntax error in the compiled workflow + # If none of the `COPILOT_PAT_#` secrets were selected, then the default COPILOT_GITHUB_TOKEN is used + COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} + +concurrency: + group: "ci-failure-scan" + cancel-in-progress: true + +tools: + github: + toolsets: [pull_requests, repos, issues, search] + min-integrity: approved + edit: + bash: ["dotnet", "git", "find", "ls", "cat", "grep", "head", "tail", "wc", "curl", "jq", "tee", "sed", "awk", "tr", "cut", "sort", "uniq", "xargs", "echo", "date", "mkdir", "test", "env", "basename", "dirname", "bash", "sh", "chmod"] + +checkout: + fetch-depth: 50 + +safe-outputs: + create-pull-request: + title-prefix: "[ci-scan] " + draft: true + max: 10 + protected-files: blocked + allowed-files: + - "src/libraries/**/tests/**" + - "src/libraries/**/src/**" + - "src/libraries/Common/tests/**" + - "src/libraries/Common/src/**" + - "src/coreclr/**/tests/**" + - "src/mono/**/tests/**" + - "src/tests/**" + - "eng/testing/**" + labels: [agentic-workflows] + create-issue: + max: 5 + labels: [agentic-workflows] + +timeout-minutes: 90 + +network: + allowed: + - defaults + - github + - dev.azure.com + - helix.dot.net + - "*.blob.core.windows.net" +--- + +# CI Outer-Loop Failure Scanner + +Platform-agnostic scan of `dnceng-public/public` outer-loop CI pipelines on `main`. Every actionable failure becomes either a draft PR (per-test fix) or a tracking issue (everything else). The intent is to keep outer-loop pipelines green without waiting on humans to file issues. + +## Pipelines to scan + +Iterate over every pipeline in this list. For each, fetch the latest completed build on branch `main`, then look back through ~10 prior completed builds to compute first-seen-in-window and occurrence counts. + +| Pipeline | Definition ID | Notes | +|----------|---------------|-------| +| runtime-extra-platforms | 154 | Apple mobile, Android, browser, wasi, NativeAOT outer loop | +| runtime-coreclr jitstress | 109 | JIT stress modes | +| runtime-coreclr jitstressregs | 110 | | +| runtime-coreclr jitstress2-jitstressregs | 111 | | +| runtime-coreclr gcstress0x3-gcstress0xc | 112 | | +| runtime-coreclr gcstress-extra | 113 | | +| runtime-coreclr jitstress-isas-x86 | 115 | | +| runtime-coreclr jitstress-isas-arm | 116 | | +| runtime-coreclr jitstressregs-x86 | 117 | | +| runtime-coreclr libraries-jitstressregs | 118 | | +| runtime-coreclr libraries-jitstress2-jitstressregs | 119 | | +| runtime-jit-experimental | 137 | OSR / partial compilation | +| runtime-coreclr libraries-jitstress | 138 | | +| runtime-coreclr ilasm | 140 | | +| runtime-coreclr pgo | 144 | | +| runtime-coreclr libraries-pgo | 145 | | +| runtime-coreclr superpmi-replay | 150 | | +| runtime-coreclr jit-cfg | 155 | Control flow guard | +| runtime-coreclr jitstress-random | 159 | Stress mode value comes from logs | +| runtime-coreclr libraries-jitstress-random | 160 | Stress mode value comes from logs | +| runtime-coreclr pgostress | 230 | | +| runtime-coreclr jitstress-isas-avx512 | 235 | | + +If a pipeline has no completed build in the last 7 days, skip it silently. + +## Skills to consult per failure + +Read the relevant skill before classifying / fixing. Skills live under `.github/skills/`. + +- **Mobile (`ios`, `tvos`, `maccatalyst`, `android`, `iossimulator`, `tvossimulator`)** → `mobile-platforms/SKILL.md`. Pipeline layout, platform helpers, code-path map. +- **JIT / GC / PGO stress** (definitions 109–160, 230, 235; `runtime-jit-experimental`) → `jit-regression-test/SKILL.md` for repro extraction; `ci-pipeline-monitor/SKILL.md` for triage and failure-shape recognition. JIT product fixes are out of scope for autofix — file an issue and `@`-mention the JIT area owners. +- **Browser/WASM, WASI** (extra-platforms) → consult `mobile-platforms/SKILL.md` (the WASM/WASI sections) for build-time conditional patterns; `extensions-review/SKILL.md` if the failure is in `Microsoft.Extensions.*` tests; `system-net-review/SKILL.md` if the failure is in `System.Net.*` tests. +- **NativeAOT outer loop** → check `eng/testing/tests.*aot*.targets` and the test `.csproj` for AOT-specific conditions before suggesting a fix. +- **Generic CI triage** → `ci-pipeline-monitor/SKILL.md` for known-failure-shape patterns and Build Analysis matching. + +## Outcome (per actionable failure) + +- **Per-test platform / configuration incompatibility** (e.g., test fails only under `jitstress=2`, `gcstress=0xC`, on a single mobile arch, on browser, on NativeAOT) → open a draft PR with a per-test attribute change: + - `[SkipOnPlatform(TestPlatforms., "")]` for platform-specific failures. + - `[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.))]` narrowed via existing helpers. + - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing an **existing** issue. + - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. +- **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. +- **Anything else** — product regression, native crash, multi-assembly cluster, JIT/GC product bug, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. + +Do not emit `noop`. Either a PR or an issue must come out of every actionable failure. + +Cap: **10 PRs and 5 issues per run.** Group failures that share one fix into a single PR. Group failures with the same root cause into a single issue. + +## Data sources + +- AzDO REST: `https://dev.azure.com/dnceng-public/public/_apis/build/...` — list completed builds per definition on branch `main`, get a build's timeline, download per-job AzDO logs. +- Helix REST: `https://helix.dot.net/api/jobs/{jobId}/workitems?api-version=2019-06-17`. Helix job IDs appear in AzDO logs as `Job on `. Each work item has `Name`, `State`, `ExitCode`, `ConsoleOutputUri`. Failed: `ExitCode != 0` or `State == "Failed"`. Console URIs containing `helix-workitem-deadletter` are dead-lettered (queue had no agent) — group as infra. +- Build Analysis (when present): `https://dev.azure.com/dnceng-public/public/_apis/build/builds/{id}/attachments/Build_Analysis_*` — use to dedupe against already-known issues. + +For each failure compute a `(work_item, queue, stress_mode, [FAIL] signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. + +Drill into one representative console log per signature to confirm the failure shape (`[FAIL]` markers, assertion text, JIT stress mode echoed in the log) before classifying. + +## PR body + +Five H2 sections, in this exact order: + +1. **Reasoning** — why the test fails on the affected platform/configuration; why the chosen attribute is the right fix; why this is a test-side fix and not a product bug. +2. **Impact on platforms** — bullet list of `(pipeline + platform/arch + Helix queue + stress mode + exit code)` per affected occurrence. +3. **Errors log** — sanitized excerpt from the Helix console log (the `[FAIL]` line, the assertion or exception, and the `Failed tests:` summary). Strip JWTs, bearer tokens, `ApplicationGatewayAffinity*=`, and per-user paths. +4. **First build it occurred** — first build in the scanned window where this signature appeared: build link, finish time, commit SHA, occurrences-in-window count. State explicitly that this is computed within the scanned window and may not be the true origin. +5. **Linked issue** (optional) — if an `ActiveIssue` reference is used, link the issue and quote the matching label set. + +Branch from `origin/main`. Stage only the files you intend to change with `git add `; never `git add -A`. Verify with `git diff --name-only --cached` before committing. Labels: at least one `os-*` (`os-android`, `os-ios`, `os-tvos`, `os-maccatalyst`, `os-browser`, `os-wasi`, `os-windows`, `os-linux`, `os-osx`) where applicable, plus the test's `area-*` label, plus `arch-*` for arch-specific failures, plus the relevant configuration label (`disabled-test`, `jit-stress`, `gc-stress`, `pgo`, `nativeaot`, …) when present. + +## Issue body + +Use this when a PR is not the right tool — product regression, native crash, multi-assembly cluster, infra requiring an owner, JIT/GC product bug. Same four sections as a PR (Reasoning, Impact on platforms, Errors log, First build it occurred), plus a fifth: + +5. **Recommended action** — concrete next step: which area owner, which file likely needs the fix, or what investigation would localize the root cause. For JIT/GC issues include the exact stress mode env vars and the JIT method-name from the log. Reference any related PR or issue you found via `search_issues`. The issue must be actionable — a checkbox-ready task list, not just "FYI". + +Same `os-*`, `area-*`, `arch-*` labels. + +## Known Build Error issue + +A Known Build Error is a tracking issue that Arcade Build Analysis (https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssueJsonStepByStep.md) automatically matches against future failures so PRs aren't blocked by an already-tracked flake. + +File one when **all** of the following hold: +- The failure has occurred ≥ 2 times in the scanned window on `main`. +- The error has a stable substring or regex signature that uniquely identifies it. +- No fix PR is currently open (verify via `search_pull_requests`). +- The failure is **not** a build break — only test failures, hangs, or infra issues. Build breaks must use a regular issue. + +Required structure (Build Analysis is strict — match the headings exactly): + +````markdown +## Build Information +Build: +Build error leg or test failing: - +Pull request: + +## Error Message + + + +```json +{ + "ErrorMessage": "", + "ErrorPattern": "", + "BuildRetry": false, + "ExcludeConsoleLog": false +} +``` +```` + +Choose `ErrorMessage` (substring) by default. Use `ErrorPattern` only when a regex is genuinely needed and confirm it has no catastrophic backtracking. Set `BuildRetry: true` **only** for confirmed infra/queue-side flakes (dead-letter, device-lost, agent disconnect) where retrying is safe. + +Title: `Test failure: ` for test failures, or `Known Build Error: ` for non-test build errors. + +Labels: `Known Build Error`, `blocking-clean-ci`, plus the test's `area-*` label and any `os-*` / `arch-*` labels that apply. + +Before filing, search for an existing Known Build Error issue with a matching `ErrorMessage` (`label:"Known Build Error" in:body ""`). If one exists and is open, do not duplicate — instead append the new build to the existing issue's body via an issue comment with the build link, leg, and timestamp. + +## Hard environment constraints + +These look like permission errors but are physical: + +- `curl` URLs containing `?` or `&` MUST be **single-quoted**. Double-quoted URLs trigger `Permission denied and could not request permission from user`. +- `>` and `-o` redirection at the agent's command line is blocked. Use `| tee /path/to/file`. +- `$(...)` and `${var@P}` are blocked at the command line. Compose values via `xargs -I{}` or by reading files inline. +- OData `$top` must be encoded as `%24top` in URLs. +- Bash allowlist: `dotnet`, `git`, `find`, `ls`, `cat`, `grep`, `head`, `tail`, `wc`, `curl`, `jq`, `tee`, `sed`, `awk`, `tr`, `cut`, `sort`, `uniq`, `xargs`, `echo`, `date`, `mkdir`, `test`, `env`, `basename`, `dirname`, `bash`, `sh`, `chmod`. No `gh`, no `pwsh`, no `python`. Each call runs in a fresh subshell — persist intermediate state to files under `/tmp/gh-aw/agent/`. + +## Submit + +Search existing issues and PRs (`search_issues`, `search_pull_requests`) before creating anything new — never duplicate. Cross-check against issues filed by the existing JIT failure-tracking bot (e.g. open issues authored by `JulieLeeMSFT` for JIT pipelines) and reference rather than re-file them. When using `search_pull_requests`, filter to `is:merged OR review:approved` so the integrity filter does not silently drop low-trust results. If an issue already tracks the failure, **prefer opening a PR that references it via `[ActiveIssue("https://github.com/dotnet/runtime/issues/")]`** rather than filing another issue. If `search_issues` returns no matches, proceed to file the issue. diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md deleted file mode 100644 index 3598c13b5a636e..00000000000000 --- a/.github/workflows/mobile-scan.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -name: "Mobile Platform Failure Scanner" -description: "Daily scan of the runtime-extra-platforms pipeline for Apple mobile and Android failures. Fixes per-test failures via PR; files an actionable tracking issue otherwise." - -permissions: - contents: read - issues: read - pull-requests: read - -on: - schedule: daily - workflow_dispatch: - roles: [admin, maintainer, write] - -# ############################################################### -# Override the COPILOT_GITHUB_TOKEN secret usage for the workflow -# with a randomly-selected token from a pool of secrets. -# -# As soon as organization-level billing is offered for Agentic -# Workflows, this stop-gap approach will be removed. -# -# See: /.github/actions/select-copilot-pat/README.md -# ############################################################### - - # Add the pre-activation step of selecting a random PAT from the supplied secrets - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Checkout the select-copilot-pat action folder - with: - persist-credentials: false - sparse-checkout: .github/actions/select-copilot-pat - sparse-checkout-cone-mode: true - fetch-depth: 1 - - - id: select-copilot-pat - name: Select Copilot token from pool - uses: ./.github/actions/select-copilot-pat - env: - SECRET_0: ${{ secrets.COPILOT_PAT_0 }} - SECRET_1: ${{ secrets.COPILOT_PAT_1 }} - SECRET_2: ${{ secrets.COPILOT_PAT_2 }} - SECRET_3: ${{ secrets.COPILOT_PAT_3 }} - SECRET_4: ${{ secrets.COPILOT_PAT_4 }} - SECRET_5: ${{ secrets.COPILOT_PAT_5 }} - SECRET_6: ${{ secrets.COPILOT_PAT_6 }} - SECRET_7: ${{ secrets.COPILOT_PAT_7 }} - SECRET_8: ${{ secrets.COPILOT_PAT_8 }} - SECRET_9: ${{ secrets.COPILOT_PAT_9 }} - -# Add the pre-activation output of the randomly selected PAT -jobs: - pre-activation: - outputs: - copilot_pat_number: ${{ steps.select-copilot-pat.outputs.copilot_pat_number }} - -# Override the COPILOT_GITHUB_TOKEN expression used in the activation job -# Consume the PAT number from the pre-activation step and select the corresponding secret -engine: - id: copilot - model: claude-sonnet-4.5 - env: - # We cannot use line breaks in this expression as it leads to a syntax error in the compiled workflow - # If none of the `COPILOT_PAT_#` secrets were selected, then the default COPILOT_GITHUB_TOKEN is used - COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} - -concurrency: - group: "mobile-scan" - cancel-in-progress: true - -tools: - github: - toolsets: [pull_requests, repos, issues, search] - min-integrity: approved - edit: - bash: ["dotnet", "git", "find", "ls", "cat", "grep", "head", "tail", "wc", "curl", "jq", "tee", "sed", "awk", "tr", "cut", "sort", "uniq", "xargs", "echo", "date", "mkdir", "test", "env", "basename", "dirname", "bash", "sh", "chmod"] - -checkout: - fetch-depth: 50 - -safe-outputs: - create-pull-request: - title-prefix: "[mobile] " - draft: true - max: 5 - protected-files: blocked - allowed-files: - - "src/libraries/**/tests/**" - - "src/libraries/Common/tests/**" - labels: [agentic-workflows] - create-issue: - max: 3 - labels: [agentic-workflows] - -timeout-minutes: 60 - -network: - allowed: - - defaults - - github - - dev.azure.com - - helix.dot.net - - "*.blob.core.windows.net" ---- - -# Mobile Platform Failure Scanner - -Scan the latest completed build of the `runtime-extra-platforms` pipeline (AzDO definition `154`, org `dnceng-public`, project `public`, branch `main`) for Apple mobile and Android failures. Every actionable failure becomes either a draft PR (per-test fix) or a tracking issue (everything else). Read `.github/skills/mobile-platforms/SKILL.md` first for the pipeline layout, platform helpers, and code-path map. - -## Outcome - -For each failed mobile work item in the latest completed build: - -- **Per-test platform incompatibility** → open a draft PR. Use a per-test attribute change: `[SkipOnPlatform(...)]`, a narrowed `[ConditionalFact]` predicate built from existing `PlatformDetection.*` helpers, or `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing an **existing** issue. Touch only files matching the `allowed-files` policy (`src/libraries/**/tests/**`, including test `.csproj`). -- **Anything else** — product regression, native crash, multi-assembly cluster, infrastructure (including queue exhaustion / dead-letter / device-lost) — file a tracking issue. The issue is the deliverable; do not paper over a product bug with `SkipOnPlatform`. Group all dead-letter / queue exhaustion / device-lost failures from one run into a single infrastructure issue. Before filing, `search_issues` for an open issue with the matching `area-Infrastructure` + `os-*` label and update its description in place rather than creating a duplicate. - -Do not emit `noop`. Either a PR or an issue must come out of every actionable failure. - -Cap: **5 PRs and 3 issues per run.** Group failures that share one fix into a single PR. Group failures with the same root cause into a single issue. - -## Data sources - -- AzDO REST: `https://dev.azure.com/dnceng-public/public/_apis/build/...` — list completed builds (definition 154, branch main), get a build's timeline, download per-job AzDO logs. Mobile job names match the regex `(ios|tvos|maccatalyst|android)` (case-insensitive). -- Helix REST: `https://helix.dot.net/api/jobs/{jobId}/workitems?api-version=2019-06-17` — Helix job IDs appear in AzDO logs as `Job on `. Each work item has `Name`, `State`, `ExitCode`, `ConsoleOutputUri`. Failed: `ExitCode != 0` or `State == "Failed"`. Console URIs containing `helix-workitem-deadletter` are dead-lettered (queue had no agent) and are pure infra — drop them. - -Look back through roughly the last 20 completed builds to compute a "first seen in scanned window" timestamp and occurrence count per `(work_item, queue)` signature. - -Drill into one representative console log per signature to confirm the failure shape (`[FAIL]` markers, assertion text) before classifying. - -## PR body - -Four H2 sections, in this exact order: - -1. **Reasoning** — why the test fails on the affected mobile platforms; why the chosen attribute is the right fix. -2. **Impact on platforms** — bullet list of `(platform/arch + Helix queue + exit code)` per affected occurrence. -3. **Errors log** — sanitized excerpt from the Helix console log (the `[FAIL]` line, the assertion or exception, and the `Failed tests:` summary). Strip JWTs, bearer tokens, `ApplicationGatewayAffinity*=`, and per-user paths. -4. **First build it occurred** — first build (in the scanned window) where this signature appeared: build link, finish time, commit SHA, occurrences-in-window count. State explicitly that this is computed within the scanned window and may not be the true origin. - -Branch from `origin/main`. Stage only the files you intend to change with `git add `; never `git add -A`. Verify with `git diff --name-only --cached` before committing. Labels: one or more `os-*` (`os-android`, `os-ios`, `os-tvos`, `os-maccatalyst`) plus the test's `area-*` label. - -## Issue body - -Use this when a PR is not the right tool — product regression, native crash, multi-assembly cluster, infra requiring an owner. Same four sections as a PR (Reasoning, Impact on platforms, Errors log, First build it occurred), plus a fifth: - -5. **Recommended action** — concrete next step: which area owner, which file likely needs the fix, or what investigation would localize the root cause. Reference any related PR or issue you found via `search_issues`. The issue must be actionable — a checkbox-ready task list, not just "FYI". - -Same `os-*` and `area-*` labels. - -## Hard environment constraints - -These look like permission errors but are physical: - -- `curl` URLs containing `?` or `&` MUST be **single-quoted**. Double-quoted URLs trigger `Permission denied and could not request permission from user`. -- `>` and `-o` redirection at the agent's command line is blocked. Use `| tee /path/to/file`. -- `$(...)` and `${var@P}` are blocked at the command line. Compose values via `xargs -I{}` or by reading files inline. -- OData `$top` must be encoded as `%24top` in URLs. -- Bash allowlist: `dotnet`, `git`, `find`, `ls`, `cat`, `grep`, `head`, `tail`, `wc`, `curl`, `jq`, `tee`, `sed`, `awk`, `tr`, `cut`, `sort`, `uniq`, `xargs`, `echo`, `date`, `mkdir`, `test`, `env`, `basename`, `dirname`, `bash`, `sh`, `chmod`. No `gh`, no `pwsh`, no `python`. Each call runs in a fresh subshell — persist intermediate state to files under `/tmp/gh-aw/agent/` (just files; you do not need to author a helper script). - -## Submit - -Search existing issues and PRs (`search_issues`, `search_pull_requests`) before creating anything new — never duplicate. When using `search_pull_requests`, filter to `is:merged OR review:approved` so the integrity filter does not silently drop low-trust results. If an issue already tracks the failure, **prefer opening a PR that references it via `[ActiveIssue("https://github.com/dotnet/runtime/issues/")]`** rather than filing another issue. If `search_issues` returns no matches, proceed to file the issue. From 4db1870c51aa579fc3351613b2ea8c0a09eb35dd Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 16:35:08 +0200 Subject: [PATCH 02/19] ci-failure-scan: handle build breaks, phase-only failures, canceled builds Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.md | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 3acc95a8f7ba5a..205565b655a113 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -114,7 +114,7 @@ Platform-agnostic scan of `dnceng-public/public` outer-loop CI pipelines on `mai ## Pipelines to scan -Iterate over every pipeline in this list. For each, fetch the latest completed build on branch `main`, then look back through ~10 prior completed builds to compute first-seen-in-window and occurrence counts. +Iterate over every pipeline in this list. For each, fetch builds on branch `main` filtered to `resultFilter=succeeded,failed,partiallySucceeded` (skip `canceled`). Pick the most recent such build as the "latest", then look back through ~10 prior completed builds to compute first-seen-in-window and occurrence counts. | Pipeline | Definition ID | Notes | |----------|---------------|-------| @@ -161,21 +161,36 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing an **existing** issue. - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. - **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. +- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. - **Anything else** — product regression, native crash, multi-assembly cluster, JIT/GC product bug, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. +For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. + Do not emit `noop`. Either a PR or an issue must come out of every actionable failure. Cap: **10 PRs and 5 issues per run.** Group failures that share one fix into a single PR. Group failures with the same root cause into a single issue. ## Data sources -- AzDO REST: `https://dev.azure.com/dnceng-public/public/_apis/build/...` — list completed builds per definition on branch `main`, get a build's timeline, download per-job AzDO logs. -- Helix REST: `https://helix.dot.net/api/jobs/{jobId}/workitems?api-version=2019-06-17`. Helix job IDs appear in AzDO logs as `Job on `. Each work item has `Name`, `State`, `ExitCode`, `ConsoleOutputUri`. Failed: `ExitCode != 0` or `State == "Failed"`. Console URIs containing `helix-workitem-deadletter` are dead-lettered (queue had no agent) — group as infra. -- Build Analysis (when present): `https://dev.azure.com/dnceng-public/public/_apis/build/builds/{id}/attachments/Build_Analysis_*` — use to dedupe against already-known issues. +- AzDO REST: `https://dev.azure.com/dnceng-public/public/_apis/build/...`. Anonymous access only — do **not** call `_apis/test/...` or `vstmr.dev.azure.com`; both redirect to sign-in. Stay on `builds`, `builds/{id}/timeline`, `builds/{id}/logs/{logId}`. + - List builds: `?definitions={id}&branchName=refs/heads/main&statusFilter=completed&resultFilter=succeeded,failed,partiallySucceeded&%24top=20&api-version=7.1`. + - Timeline: `/builds/{id}/timeline?api-version=7.1` returns a flat `records[]` array; reconstruct the tree via `parentId`. + - Failed-leaf rule: a record with `result == "failed"` whose log id is non-null is a leaf to inspect; failed Stage/Phase records without a failed child Job indicate a build break — open the parent Phase log and the most recent non-succeeded Task log. +- Helix REST: `https://helix.dot.net/api/jobs/{jobId}/workitems?api-version=2019-06-17`. Helix job IDs come from the `Send to Helix` Task log, which is a child of the failed Job. Each work item has `Name`, `State`, `ExitCode`, `ConsoleOutputUri`. Failed: `ExitCode != 0` or `State == "Failed"`. Console URIs containing `helix-workitem-deadletter` are dead-lettered (queue had no agent) — group as infra. +- Build Analysis attachment (best-effort, may 404): `https://dev.azure.com/dnceng-public/public/_apis/build/builds/{id}/attachments/Build_Analysis_KnownIssues_v1?api-version=7.1`. Use to dedupe against already-known issues. A 404 means none were attached; do not fail. + +## Failure classification + +Classify every failed timeline record before deciding whether to PR or file an issue. The timeline graph is `Stage → Phase → Job → Task`. Walk it as follows: -For each failure compute a `(work_item, queue, stress_mode, [FAIL] signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. +1. List every record with `result == "failed"`. For each failed Job, list its child Tasks (records whose `parentId == job.id`). +2. **Build break (no test ever ran)**: among the Job's Tasks, the failed Task is `Build product`, `Build native components`, `Configure CMake`, or any pre-test compile step, **and** the `Send to Helix` Task is `skipped`. → tracking issue. Do **not** attempt a test-side fix. +3. **Phase/Stage-only failure with no failed Job underneath**: typical of compile-time breaks aggregated at the phase level (e.g. `windows-arm64 checked` on the JIT stress pipelines). Open the Phase log and the latest log of any non-succeeded child Task; classify as build break and file a tracking issue. +4. **Send to Helix succeeded but the Job still failed**: open the `Send to Helix` log, extract Helix job IDs (look for `Job on ` or `JobId: `; the Helix info-mart log entry that always appears is `Sent Helix Job: `), then query Helix for failed work items. This is the test-failure path. +5. **Helix work item failure**: confirm via `ConsoleOutputUri`. `helix-workitem-deadletter` URIs → infra (group into one issue). Otherwise fetch the console log, find the `[FAIL]` line, and proceed to PR vs issue selection. +6. **Infra-shaped Job failure** without Helix workitems (e.g., `Initialize job` failed, agent disconnect, "Pool is offline") → file a single grouped infra issue, do not retry per-leg. -Drill into one representative console log per signature to confirm the failure shape (`[FAIL]` markers, assertion text, JIT stress mode echoed in the log) before classifying. +Drill into one representative console log per signature to confirm the shape before classifying. ## PR body From 98391acf9321b02a8d725a56fa7f2906f854db0d Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 16:54:57 +0200 Subject: [PATCH 03/19] Keep mobile-scan filename for dispatch compatibility Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ailure-scan.lock.yml => mobile-scan.lock.yml} | 16 ++++++++-------- .../{ci-failure-scan.md => mobile-scan.md} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{ci-failure-scan.lock.yml => mobile-scan.lock.yml} (99%) rename .github/workflows/{ci-failure-scan.md => mobile-scan.md} (100%) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/mobile-scan.lock.yml similarity index 99% rename from .github/workflows/ci-failure-scan.lock.yml rename to .github/workflows/mobile-scan.lock.yml index e536b54e9a8a95..0ba22c63fac3e5 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/mobile-scan.lock.yml @@ -55,7 +55,7 @@ name: "CI Outer-Loop Failure Scanner" # - maintainer # Roles processed as role check in pre-activation job # - write # Roles processed as role check in pre-activation job schedule: - - cron: "34 */6 * * *" + - cron: "53 */6 * * *" # Friendly format: every 6h (scattered) # steps: # Steps injected into pre-activation job # - name: Checkout the select-copilot-pat action folder @@ -163,7 +163,7 @@ jobs: id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: - GH_AW_WORKFLOW_FILE: "ci-failure-scan.lock.yml" + GH_AW_WORKFLOW_FILE: "mobile-scan.lock.yml" GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | @@ -246,7 +246,7 @@ jobs: cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' - {{#runtime-import .github/workflows/ci-failure-scan.md}} + {{#runtime-import .github/workflows/mobile-scan.md}} GH_AW_PROMPT_1ab483264300fa84_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates @@ -331,7 +331,7 @@ jobs: GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_WORKFLOW_ID_SANITIZED: cifailurescan + GH_AW_WORKFLOW_ID_SANITIZED: mobilescan outputs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} @@ -954,7 +954,7 @@ jobs: issues: write pull-requests: write concurrency: - group: "gh-aw-conclusion-ci-failure-scan" + group: "gh-aw-conclusion-mobile-scan" cancel-in-progress: false outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} @@ -1037,7 +1037,7 @@ jobs: GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "ci-failure-scan" + GH_AW_WORKFLOW_ID: "mobile-scan" GH_AW_ENGINE_ID: "copilot" GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} @@ -1270,11 +1270,11 @@ jobs: pull-requests: write timeout-minutes: 15 env: - GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/ci-failure-scan" + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/mobile-scan" GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" GH_AW_ENGINE_MODEL: "claude-opus-4.7" - GH_AW_WORKFLOW_ID: "ci-failure-scan" + GH_AW_WORKFLOW_ID: "mobile-scan" GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" outputs: code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/mobile-scan.md similarity index 100% rename from .github/workflows/ci-failure-scan.md rename to .github/workflows/mobile-scan.md From f39fd49187253e33c737bf51cef4a09e66f277ab Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 17:04:22 +0200 Subject: [PATCH 04/19] Iterate: variable-binding pattern + sonnet-4.6 model Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/mobile-scan.lock.yml | 34 +++++++++++++------------- .github/workflows/mobile-scan.md | 9 +++++-- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/mobile-scan.lock.yml b/.github/workflows/mobile-scan.lock.yml index 0ba22c63fac3e5..96e23512a2c337 100644 --- a/.github/workflows/mobile-scan.lock.yml +++ b/.github/workflows/mobile-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"70808df580451921c69c549032f52027be0ddff9d008bca44c1ee90ab2025be5","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.7"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"1183df8a729f5d21fe0a5e0f691050677c0d45a07527f1ed5e4f045eee0a7c77","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc","version":"v0.68.1"}]} # ___ _ _ # / _ \ | | (_) @@ -124,7 +124,7 @@ jobs: env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: "claude-opus-4.7" + GH_AW_INFO_MODEL: "claude-sonnet-4.6" GH_AW_INFO_VERSION: "1.0.21" GH_AW_INFO_AGENT_VERSION: "1.0.21" GH_AW_INFO_CLI_VERSION: "v0.68.1" @@ -197,19 +197,19 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' + cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' - GH_AW_PROMPT_1ab483264300fa84_EOF + GH_AW_PROMPT_bd8e55f06b00486c_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' + cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' Tools: create_issue(max:5), create_pull_request(max:10), missing_tool, missing_data, noop - GH_AW_PROMPT_1ab483264300fa84_EOF + GH_AW_PROMPT_bd8e55f06b00486c_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' + cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' The following GitHub context information is available for this workflow: @@ -242,12 +242,12 @@ jobs: - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). - GH_AW_PROMPT_1ab483264300fa84_EOF + GH_AW_PROMPT_bd8e55f06b00486c_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_1ab483264300fa84_EOF' + cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' {{#runtime-import .github/workflows/mobile-scan.md}} - GH_AW_PROMPT_1ab483264300fa84_EOF + GH_AW_PROMPT_bd8e55f06b00486c_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -413,9 +413,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_1d0cbd0432f58d3e_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_8c92f388958dbf74_EOF' {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**/tests/**","src/libraries/**/src/**","src/libraries/Common/tests/**","src/libraries/Common/src/**","src/coreclr/**/tests/**","src/mono/**/tests/**","src/tests/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_1d0cbd0432f58d3e_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_8c92f388958dbf74_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -645,7 +645,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_fabcc418e0385e62_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_6263a0955267e9f0_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -689,7 +689,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_fabcc418e0385e62_EOF + GH_AW_MCP_CONFIG_6263a0955267e9f0_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -754,7 +754,7 @@ jobs: env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} - COPILOT_MODEL: claude-opus-4.7 + COPILOT_MODEL: claude-sonnet-4.6 GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json GH_AW_PHASE: agent GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt @@ -1172,7 +1172,7 @@ jobs: env: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_GITHUB_TOKEN: ${{ case(needs.pre_activation.outputs.copilot_pat_number == '0', secrets.COPILOT_PAT_0, needs.pre_activation.outputs.copilot_pat_number == '1', secrets.COPILOT_PAT_1, needs.pre_activation.outputs.copilot_pat_number == '2', secrets.COPILOT_PAT_2, needs.pre_activation.outputs.copilot_pat_number == '3', secrets.COPILOT_PAT_3, needs.pre_activation.outputs.copilot_pat_number == '4', secrets.COPILOT_PAT_4, needs.pre_activation.outputs.copilot_pat_number == '5', secrets.COPILOT_PAT_5, needs.pre_activation.outputs.copilot_pat_number == '6', secrets.COPILOT_PAT_6, needs.pre_activation.outputs.copilot_pat_number == '7', secrets.COPILOT_PAT_7, needs.pre_activation.outputs.copilot_pat_number == '8', secrets.COPILOT_PAT_8, needs.pre_activation.outputs.copilot_pat_number == '9', secrets.COPILOT_PAT_9, secrets.COPILOT_GITHUB_TOKEN) }} - COPILOT_MODEL: claude-opus-4.7 + COPILOT_MODEL: claude-sonnet-4.6 GH_AW_PHASE: detection GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_VERSION: v0.68.1 @@ -1273,7 +1273,7 @@ jobs: GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/mobile-scan" GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: "claude-opus-4.7" + GH_AW_ENGINE_MODEL: "claude-sonnet-4.6" GH_AW_WORKFLOW_ID: "mobile-scan" GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" outputs: diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md index 205565b655a113..45e93515309af4 100644 --- a/.github/workflows/mobile-scan.md +++ b/.github/workflows/mobile-scan.md @@ -57,7 +57,7 @@ jobs: # Consume the PAT number from the pre-activation step and select the corresponding secret engine: id: copilot - model: claude-opus-4.7 + model: claude-sonnet-4.6 env: # We cannot use line breaks in this expression as it leads to a syntax error in the compiled workflow # If none of the `COPILOT_PAT_#` secrets were selected, then the default COPILOT_GITHUB_TOKEN is used @@ -256,7 +256,12 @@ Before filing, search for an existing Known Build Error issue with a matching `E These look like permission errors but are physical: -- `curl` URLs containing `?` or `&` MUST be **single-quoted**. Double-quoted URLs trigger `Permission denied and could not request permission from user`. +- **Pre-bind every URL to a shell variable on a line of its own, then `curl -s "$url"`.** Inline URLs with `?` or `&` are rejected as "Permission denied and could not request permission from user" even when single-quoted, because the Copilot CLI tool-approver treats query strings as interactive prompts. The only working pattern is: + ```bash + url='https://dev.azure.com/dnceng-public/public/_apis/build/builds?definitions=154&branchName=refs/heads/main&statusFilter=completed&resultFilter=succeeded,failed,partiallySucceeded&%24top=25&api-version=7.1' + curl -s "$url" | jq '.' | tee /tmp/gh-aw/agent/builds.json | jq -r '.value[0] | "\(.id) \(.result)"' + ``` + Do **not** retry an inline URL hoping the rejection will clear — it won't. Switch to the variable pattern immediately. - `>` and `-o` redirection at the agent's command line is blocked. Use `| tee /path/to/file`. - `$(...)` and `${var@P}` are blocked at the command line. Compose values via `xargs -I{}` or by reading files inline. - OData `$top` must be encoded as `%24top` in URLs. From d2c7a42ad51792fcb700097ae880522cfd8be2ed Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 17:37:16 +0200 Subject: [PATCH 05/19] Iterate: require companion ActiveIssue PRs for stress-mode product bugs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/mobile-scan.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md index 45e93515309af4..6b36e27d6043a4 100644 --- a/.github/workflows/mobile-scan.md +++ b/.github/workflows/mobile-scan.md @@ -162,10 +162,13 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. - **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. - **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. -- **Anything else** — product regression, native crash, multi-assembly cluster, JIT/GC product bug, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. +- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **always do BOTH**: (a) file a tracking issue with the diagnosis, AND (b) open a companion draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s), so the outer-loop pipeline goes green while the product bug is fixed. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the tracking issue. Reference the new issue number from the safe-output handler — it is exposed as the temporary id mapping printed when the issue is created. If you cannot get the issue number in time (because PRs and issues land in the same safe-output batch), use the placeholder `${{ aw_ }}` syntax that the safe-output handler resolves to the created issue URL. +- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. +**Convergence target**: leave each pipeline green-on-next-run by either (a) muting the failing test via PR or (b) filing a Known Build Error issue that Build Analysis can auto-match. A tracking-issue-only outcome is acceptable only for build breaks and infra failures where there is nothing to mute on the test side. + Do not emit `noop`. Either a PR or an issue must come out of every actionable failure. Cap: **10 PRs and 5 issues per run.** Group failures that share one fix into a single PR. Group failures with the same root cause into a single issue. From d7eb430ceeea22c0e9e0a8f50cbf23a8b10cb28c Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 17:38:20 +0200 Subject: [PATCH 06/19] Fix expression validator: drop ${{ }} wrapping around aw_id token Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/mobile-scan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md index 6b36e27d6043a4..5b1fb2041839e0 100644 --- a/.github/workflows/mobile-scan.md +++ b/.github/workflows/mobile-scan.md @@ -162,7 +162,7 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. - **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. - **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. -- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **always do BOTH**: (a) file a tracking issue with the diagnosis, AND (b) open a companion draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s), so the outer-loop pipeline goes green while the product bug is fixed. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the tracking issue. Reference the new issue number from the safe-output handler — it is exposed as the temporary id mapping printed when the issue is created. If you cannot get the issue number in time (because PRs and issues land in the same safe-output batch), use the placeholder `${{ aw_ }}` syntax that the safe-output handler resolves to the created issue URL. +- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **always do BOTH**: (a) file a tracking issue with the diagnosis, AND (b) open a companion draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s), so the outer-loop pipeline goes green while the product bug is fixed. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the tracking issue. Because issues and PRs land in the same safe-output batch, reference the issue by its temporary id token `aw_` (the safe-output handler resolves the token to the final issue URL after creation — emit it as plain text, not as a `${{ }}` expression). Emit the `create_issue` output **before** the corresponding `create_pull_request` output so the temporary id is in scope. - **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. From 24eb0598b6bd4574bed31cb7b5342acfacb6aef4 Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 17:39:20 +0200 Subject: [PATCH 07/19] Drop literal expression syntax from prose Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/mobile-scan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md index 5b1fb2041839e0..666a721970ad5f 100644 --- a/.github/workflows/mobile-scan.md +++ b/.github/workflows/mobile-scan.md @@ -162,7 +162,7 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. - **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. - **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. -- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **always do BOTH**: (a) file a tracking issue with the diagnosis, AND (b) open a companion draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s), so the outer-loop pipeline goes green while the product bug is fixed. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the tracking issue. Because issues and PRs land in the same safe-output batch, reference the issue by its temporary id token `aw_` (the safe-output handler resolves the token to the final issue URL after creation — emit it as plain text, not as a `${{ }}` expression). Emit the `create_issue` output **before** the corresponding `create_pull_request` output so the temporary id is in scope. +- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **always do BOTH**: (a) file a tracking issue with the diagnosis, AND (b) open a companion draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s), so the outer-loop pipeline goes green while the product bug is fixed. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the tracking issue. Because issues and PRs land in the same safe-output batch, reference the issue by its temporary id token `aw_` (emit the token as plain text in the PR body — the safe-output handler post-resolves it to the final issue URL). Emit the `create_issue` output **before** the corresponding `create_pull_request` output so the temporary id is in scope. - **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. From f6dc05df92a5afbca86407d6f40395d8e5b9089f Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 19:22:44 +0200 Subject: [PATCH 08/19] Two-pass approach: file issue first, file ActiveIssue PR on next run Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/mobile-scan.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md index 666a721970ad5f..85c1783d98a03b 100644 --- a/.github/workflows/mobile-scan.md +++ b/.github/workflows/mobile-scan.md @@ -162,7 +162,11 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. - **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. - **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. -- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **always do BOTH**: (a) file a tracking issue with the diagnosis, AND (b) open a companion draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s), so the outer-loop pipeline goes green while the product bug is fixed. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the tracking issue. Because issues and PRs land in the same safe-output batch, reference the issue by its temporary id token `aw_` (emit the token as plain text in the PR body — the safe-output handler post-resolves it to the final issue URL). Emit the `create_issue` output **before** the corresponding `create_pull_request` output so the temporary id is in scope. +- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **two-pass resolution across runs**: + 1. First run (no existing tracking issue found): file the tracking issue only. Do **not** open a PR — the issue number isn't available inside a patch (safe-outputs only resolve `aw_` tokens in body text, never inside diffs). + 2. Subsequent run (a matching tracking issue with `area-*` + `arch-*` + `os-*` + stress-mode signature already exists): open a draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s). The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the existing tracking issue. Hardcoding an issue number is allowed here because the issue already exists. + + To find the matching existing issue, `search_issues` for: `is:issue is:open label:area- label:os- in:title ""` and `is:issue is:open label:area- ""`. Match on test name + stress mode + arch. If multiple candidates, pick the one with the most recent activity. - **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. From 6e55e00a92b7786d841da0d28eee2fb11f9e7d6d Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 19:47:30 +0200 Subject: [PATCH 09/19] Make muting-PR step mandatory when tracking issue already exists Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/mobile-scan.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/mobile-scan.md index 85c1783d98a03b..17c869591ad679 100644 --- a/.github/workflows/mobile-scan.md +++ b/.github/workflows/mobile-scan.md @@ -164,9 +164,11 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. - **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **two-pass resolution across runs**: 1. First run (no existing tracking issue found): file the tracking issue only. Do **not** open a PR — the issue number isn't available inside a patch (safe-outputs only resolve `aw_` tokens in body text, never inside diffs). - 2. Subsequent run (a matching tracking issue with `area-*` + `arch-*` + `os-*` + stress-mode signature already exists): open a draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s). The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the existing tracking issue. Hardcoding an issue number is allowed here because the issue already exists. + 2. Subsequent run (a matching tracking issue with `area-*` + `arch-*` + `os-*` + stress-mode signature already exists, AND the failure is still occurring in the latest scanned build): you **MUST** open a draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s). This is the convergence step — without it, the pipeline never goes green. Skipping the PR because "the issue is already tracked" is incorrect; the existence of the issue is precisely what enables the PR. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the existing tracking issue. Hardcode the issue number in the diff because the issue already exists. - To find the matching existing issue, `search_issues` for: `is:issue is:open label:area- label:os- in:title ""` and `is:issue is:open label:area- ""`. Match on test name + stress mode + arch. If multiple candidates, pick the one with the most recent activity. + To find the matching existing issue, `search_issues` for: `is:issue is:open label:area- in:title ""` and `is:issue is:open label:area- ""`. Match on test name + stress mode + arch. If multiple candidates, pick the one with the most recent activity. If you found such an issue earlier in the run while deduplicating, **do not stop there — proceed to file the muting PR**. + + After every run, you should be able to answer YES to: "for each still-failing stress-mode product bug whose tracking issue already exists, did I emit a `create_pull_request` output?" If the answer is NO for any of them, you have not done the job. - **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. From 745f130365324568c4c25e37667efaee6a2ed20a Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 21:22:03 +0200 Subject: [PATCH 10/19] Rename to ci-failure-scan to reflect platform-agnostic scope Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...le-scan.lock.yml => ci-failure-scan.lock.yml} | 16 ++++++++-------- .../{mobile-scan.md => ci-failure-scan.md} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{mobile-scan.lock.yml => ci-failure-scan.lock.yml} (99%) rename .github/workflows/{mobile-scan.md => ci-failure-scan.md} (100%) diff --git a/.github/workflows/mobile-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml similarity index 99% rename from .github/workflows/mobile-scan.lock.yml rename to .github/workflows/ci-failure-scan.lock.yml index 96e23512a2c337..b1db68a0d2c55b 100644 --- a/.github/workflows/mobile-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -55,7 +55,7 @@ name: "CI Outer-Loop Failure Scanner" # - maintainer # Roles processed as role check in pre-activation job # - write # Roles processed as role check in pre-activation job schedule: - - cron: "53 */6 * * *" + - cron: "34 */6 * * *" # Friendly format: every 6h (scattered) # steps: # Steps injected into pre-activation job # - name: Checkout the select-copilot-pat action folder @@ -163,7 +163,7 @@ jobs: id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: - GH_AW_WORKFLOW_FILE: "mobile-scan.lock.yml" + GH_AW_WORKFLOW_FILE: "ci-failure-scan.lock.yml" GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | @@ -246,7 +246,7 @@ jobs: cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' - {{#runtime-import .github/workflows/mobile-scan.md}} + {{#runtime-import .github/workflows/ci-failure-scan.md}} GH_AW_PROMPT_bd8e55f06b00486c_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates @@ -331,7 +331,7 @@ jobs: GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_WORKFLOW_ID_SANITIZED: mobilescan + GH_AW_WORKFLOW_ID_SANITIZED: cifailurescan outputs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} @@ -954,7 +954,7 @@ jobs: issues: write pull-requests: write concurrency: - group: "gh-aw-conclusion-mobile-scan" + group: "gh-aw-conclusion-ci-failure-scan" cancel-in-progress: false outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} @@ -1037,7 +1037,7 @@ jobs: GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "mobile-scan" + GH_AW_WORKFLOW_ID: "ci-failure-scan" GH_AW_ENGINE_ID: "copilot" GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} @@ -1270,11 +1270,11 @@ jobs: pull-requests: write timeout-minutes: 15 env: - GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/mobile-scan" + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/ci-failure-scan" GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" GH_AW_ENGINE_MODEL: "claude-sonnet-4.6" - GH_AW_WORKFLOW_ID: "mobile-scan" + GH_AW_WORKFLOW_ID: "ci-failure-scan" GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" outputs: code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/ci-failure-scan.md similarity index 100% rename from .github/workflows/mobile-scan.md rename to .github/workflows/ci-failure-scan.md From 521872f44d6d5dc486604b7813fc84d1ad2a4bc5 Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 21:53:58 +0200 Subject: [PATCH 11/19] Iterate: [ci-scan] prefix on all outputs, no Mute, fix KBE fence, coverage discipline Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.md | 31 ++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 17c869591ad679..a24d4a71ffbfa3 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -221,6 +221,15 @@ Use this when a PR is not the right tool — product regression, native crash, m Same `os-*`, `area-*`, `arch-*` labels. +## Outputs: title and labels + +- **All issues and PRs MUST have title prefix `[ci-scan] `**, including tracking issues, Known Build Error issues, and muting PRs. Examples: + - `[ci-scan] Test failure: on ` + - `[ci-scan] Known Build Error: ` + - `[ci-scan] Skip under (refs #)` +- **Do not use the word "Mute" or "Muting"** in titles. Use "Skip", "Disable", "Suppress", or "Exclude" depending on the mechanism. Examples: "Skip … under GCStress", "Disable … on tvOS", "Suppress … in MiniFull AOT mode". +- Labels are unchanged from the per-outcome rules above. + ## Known Build Error issue A Known Build Error is a tracking issue that Arcade Build Analysis (https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssueJsonStepByStep.md) automatically matches against future failures so PRs aren't blocked by an already-tracked flake. @@ -231,9 +240,9 @@ File one when **all** of the following hold: - No fix PR is currently open (verify via `search_pull_requests`). - The failure is **not** a build break — only test failures, hangs, or infra issues. Build breaks must use a regular issue. -Required structure (Build Analysis is strict — match the headings exactly): +Required structure (Build Analysis is strict — match the headings exactly, and use **exactly three backticks** for the JSON code fence; never four. The opening and closing fence must be the same length, otherwise the fence is broken and Build Analysis silently skips the issue): -````markdown +``` ## Build Information Build: Build error leg or test failing: - @@ -243,15 +252,17 @@ Pull request: -```json +(open three backticks, then `json` on the same line) { "ErrorMessage": "", "ErrorPattern": "", "BuildRetry": false, "ExcludeConsoleLog": false } +(close three backticks) ``` -```` + +The pseudo-instructions `(open three backticks, then ...)` and `(close three backticks)` above are **placeholders** in this prompt because nesting fenced code blocks in the prompt itself is fragile; in the actual issue body emit literal ```` ``` `` (three backticks) on each side of the JSON object. Verify the open and close fences both consist of exactly three backticks before submitting. If you are uncertain, count them. Choose `ErrorMessage` (substring) by default. Use `ErrorPattern` only when a regex is genuinely needed and confirm it has no catastrophic backtracking. Set `BuildRetry: true` **only** for confirmed infra/queue-side flakes (dead-letter, device-lost, agent disconnect) where retrying is safe. @@ -276,6 +287,18 @@ These look like permission errors but are physical: - OData `$top` must be encoded as `%24top` in URLs. - Bash allowlist: `dotnet`, `git`, `find`, `ls`, `cat`, `grep`, `head`, `tail`, `wc`, `curl`, `jq`, `tee`, `sed`, `awk`, `tr`, `cut`, `sort`, `uniq`, `xargs`, `echo`, `date`, `mkdir`, `test`, `env`, `basename`, `dirname`, `bash`, `sh`, `chmod`. No `gh`, no `pwsh`, no `python`. Each call runs in a fresh subshell — persist intermediate state to files under `/tmp/gh-aw/agent/`. +## Coverage discipline (avoid arbitrary selection) + +Failure selection must be **systematic, not opportunistic**. Process pipelines in the order listed in the "Pipelines to scan" table. For each pipeline: + +1. List every failed signature in the latest scanned build (sorted by occurrence count in the window, descending). +2. For each signature, decide and record one of: `→ filed-issue #aw_`, `→ filed-PR #aw_`, `→ existing-issue #`, `→ existing-PR #`, `→ skipped: `. A skipped signature MUST have a reason (e.g., "build canceled, not a test failure", "less than 2 occurrences and not blocking", "owned by area-Infrastructure rota and already triaged"). +3. Keep a per-pipeline tally on disk under `/tmp/gh-aw/agent/coverage/.txt`. At the end, print a summary table to the agent log: `pipeline | total-signatures | issues-filed | prs-filed | reused-existing | skipped-with-reason`. + +Caps still apply (10 PRs / 5 issues / run); when the cap is hit, fall back to "skipped: cap reached" rather than dropping signatures silently. Subsequent runs will pick them up. + +Do not jump between pipelines mid-investigation. Finish all classifications for pipeline N before moving to pipeline N+1. + ## Submit Search existing issues and PRs (`search_issues`, `search_pull_requests`) before creating anything new — never duplicate. Cross-check against issues filed by the existing JIT failure-tracking bot (e.g. open issues authored by `JulieLeeMSFT` for JIT pipelines) and reference rather than re-file them. When using `search_pull_requests`, filter to `is:merged OR review:approved` so the integrity filter does not silently drop low-trust results. If an issue already tracks the failure, **prefer opening a PR that references it via `[ActiveIssue("https://github.com/dotnet/runtime/issues/")]`** rather than filing another issue. If `search_issues` returns no matches, proceed to file the issue. From 44e81be0da62bd86da11e297bffda9a34a3f9548 Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 21:54:32 +0200 Subject: [PATCH 12/19] Temporarily revert filename for branch dispatch testing Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...ailure-scan.lock.yml => mobile-scan.lock.yml} | 16 ++++++++-------- .../{ci-failure-scan.md => mobile-scan.md} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{ci-failure-scan.lock.yml => mobile-scan.lock.yml} (99%) rename .github/workflows/{ci-failure-scan.md => mobile-scan.md} (100%) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/mobile-scan.lock.yml similarity index 99% rename from .github/workflows/ci-failure-scan.lock.yml rename to .github/workflows/mobile-scan.lock.yml index b1db68a0d2c55b..96e23512a2c337 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/mobile-scan.lock.yml @@ -55,7 +55,7 @@ name: "CI Outer-Loop Failure Scanner" # - maintainer # Roles processed as role check in pre-activation job # - write # Roles processed as role check in pre-activation job schedule: - - cron: "34 */6 * * *" + - cron: "53 */6 * * *" # Friendly format: every 6h (scattered) # steps: # Steps injected into pre-activation job # - name: Checkout the select-copilot-pat action folder @@ -163,7 +163,7 @@ jobs: id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: - GH_AW_WORKFLOW_FILE: "ci-failure-scan.lock.yml" + GH_AW_WORKFLOW_FILE: "mobile-scan.lock.yml" GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | @@ -246,7 +246,7 @@ jobs: cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' - {{#runtime-import .github/workflows/ci-failure-scan.md}} + {{#runtime-import .github/workflows/mobile-scan.md}} GH_AW_PROMPT_bd8e55f06b00486c_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates @@ -331,7 +331,7 @@ jobs: GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_WORKFLOW_ID_SANITIZED: cifailurescan + GH_AW_WORKFLOW_ID_SANITIZED: mobilescan outputs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} @@ -954,7 +954,7 @@ jobs: issues: write pull-requests: write concurrency: - group: "gh-aw-conclusion-ci-failure-scan" + group: "gh-aw-conclusion-mobile-scan" cancel-in-progress: false outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} @@ -1037,7 +1037,7 @@ jobs: GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "ci-failure-scan" + GH_AW_WORKFLOW_ID: "mobile-scan" GH_AW_ENGINE_ID: "copilot" GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} @@ -1270,11 +1270,11 @@ jobs: pull-requests: write timeout-minutes: 15 env: - GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/ci-failure-scan" + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/mobile-scan" GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" GH_AW_ENGINE_MODEL: "claude-sonnet-4.6" - GH_AW_WORKFLOW_ID: "ci-failure-scan" + GH_AW_WORKFLOW_ID: "mobile-scan" GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" outputs: code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/mobile-scan.md similarity index 100% rename from .github/workflows/ci-failure-scan.md rename to .github/workflows/mobile-scan.md From 43479b23333f7a6ae27b6f49f0266624239a3f7e Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 22:27:24 +0200 Subject: [PATCH 13/19] Rename to ci-failure-scan.* (final state) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...le-scan.lock.yml => ci-failure-scan.lock.yml} | 16 ++++++++-------- .../{mobile-scan.md => ci-failure-scan.md} | 0 2 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{mobile-scan.lock.yml => ci-failure-scan.lock.yml} (99%) rename .github/workflows/{mobile-scan.md => ci-failure-scan.md} (100%) diff --git a/.github/workflows/mobile-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml similarity index 99% rename from .github/workflows/mobile-scan.lock.yml rename to .github/workflows/ci-failure-scan.lock.yml index 96e23512a2c337..b1db68a0d2c55b 100644 --- a/.github/workflows/mobile-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -55,7 +55,7 @@ name: "CI Outer-Loop Failure Scanner" # - maintainer # Roles processed as role check in pre-activation job # - write # Roles processed as role check in pre-activation job schedule: - - cron: "53 */6 * * *" + - cron: "34 */6 * * *" # Friendly format: every 6h (scattered) # steps: # Steps injected into pre-activation job # - name: Checkout the select-copilot-pat action folder @@ -163,7 +163,7 @@ jobs: id: check-lock-file uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: - GH_AW_WORKFLOW_FILE: "mobile-scan.lock.yml" + GH_AW_WORKFLOW_FILE: "ci-failure-scan.lock.yml" GH_AW_CONTEXT_WORKFLOW_REF: "${{ github.workflow_ref }}" with: script: | @@ -246,7 +246,7 @@ jobs: cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' - {{#runtime-import .github/workflows/mobile-scan.md}} + {{#runtime-import .github/workflows/ci-failure-scan.md}} GH_AW_PROMPT_bd8e55f06b00486c_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates @@ -331,7 +331,7 @@ jobs: GH_AW_ASSETS_BRANCH: "" GH_AW_ASSETS_MAX_SIZE_KB: 0 GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs - GH_AW_WORKFLOW_ID_SANITIZED: mobilescan + GH_AW_WORKFLOW_ID_SANITIZED: cifailurescan outputs: checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} effective_tokens: ${{ steps.parse-mcp-gateway.outputs.effective_tokens }} @@ -954,7 +954,7 @@ jobs: issues: write pull-requests: write concurrency: - group: "gh-aw-conclusion-mobile-scan" + group: "gh-aw-conclusion-ci-failure-scan" cancel-in-progress: false outputs: incomplete_count: ${{ steps.report_incomplete.outputs.incomplete_count }} @@ -1037,7 +1037,7 @@ jobs: GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} - GH_AW_WORKFLOW_ID: "mobile-scan" + GH_AW_WORKFLOW_ID: "ci-failure-scan" GH_AW_ENGINE_ID: "copilot" GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} @@ -1270,11 +1270,11 @@ jobs: pull-requests: write timeout-minutes: 15 env: - GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/mobile-scan" + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/ci-failure-scan" GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" GH_AW_ENGINE_MODEL: "claude-sonnet-4.6" - GH_AW_WORKFLOW_ID: "mobile-scan" + GH_AW_WORKFLOW_ID: "ci-failure-scan" GH_AW_WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" outputs: code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} diff --git a/.github/workflows/mobile-scan.md b/.github/workflows/ci-failure-scan.md similarity index 100% rename from .github/workflows/mobile-scan.md rename to .github/workflows/ci-failure-scan.md From ce1557e2d55e0aaab545192f729faf369c08fc1b Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 22:36:15 +0200 Subject: [PATCH 14/19] Drop production-source paths from allowed-files (test-only scope) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.lock.yml | 30 +++++++++++----------- .github/workflows/ci-failure-scan.md | 2 -- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index b1db68a0d2c55b..f5bd6c6bd23f72 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"1183df8a729f5d21fe0a5e0f691050677c0d45a07527f1ed5e4f045eee0a7c77","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"fd3979b0d6d2e0582a73bcf0677e0174d7a07cdacefd3b046d926c6713f2b19c","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc","version":"v0.68.1"}]} # ___ _ _ # / _ \ | | (_) @@ -197,19 +197,19 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' + cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' - GH_AW_PROMPT_bd8e55f06b00486c_EOF + GH_AW_PROMPT_1705160421966dcc_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' + cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' Tools: create_issue(max:5), create_pull_request(max:10), missing_tool, missing_data, noop - GH_AW_PROMPT_bd8e55f06b00486c_EOF + GH_AW_PROMPT_1705160421966dcc_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' + cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' The following GitHub context information is available for this workflow: @@ -242,12 +242,12 @@ jobs: - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). - GH_AW_PROMPT_bd8e55f06b00486c_EOF + GH_AW_PROMPT_1705160421966dcc_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_bd8e55f06b00486c_EOF' + cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' {{#runtime-import .github/workflows/ci-failure-scan.md}} - GH_AW_PROMPT_bd8e55f06b00486c_EOF + GH_AW_PROMPT_1705160421966dcc_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -413,9 +413,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_8c92f388958dbf74_EOF' - {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**/tests/**","src/libraries/**/src/**","src/libraries/Common/tests/**","src/libraries/Common/src/**","src/coreclr/**/tests/**","src/mono/**/tests/**","src/tests/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_8c92f388958dbf74_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_f855a461e7e8f6dc_EOF' + {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**/tests/**","src/libraries/Common/tests/**","src/coreclr/**/tests/**","src/mono/**/tests/**","src/tests/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_f855a461e7e8f6dc_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -645,7 +645,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_6263a0955267e9f0_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_7f9449a6cdffe668_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -689,7 +689,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_6263a0955267e9f0_EOF + GH_AW_MCP_CONFIG_7f9449a6cdffe668_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -1354,7 +1354,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dev.azure.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,helix.dot.net,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**/tests/**\",\"src/libraries/**/src/**\",\"src/libraries/Common/tests/**\",\"src/libraries/Common/src/**\",\"src/coreclr/**/tests/**\",\"src/mono/**/tests/**\",\"src/tests/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**/tests/**\",\"src/libraries/Common/tests/**\",\"src/coreclr/**/tests/**\",\"src/mono/**/tests/**\",\"src/tests/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index a24d4a71ffbfa3..8a9c81b5378e94 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -85,9 +85,7 @@ safe-outputs: protected-files: blocked allowed-files: - "src/libraries/**/tests/**" - - "src/libraries/**/src/**" - "src/libraries/Common/tests/**" - - "src/libraries/Common/src/**" - "src/coreclr/**/tests/**" - "src/mono/**/tests/**" - "src/tests/**" From 09f0a28a22913d78aec0b1b9594e955056cae119 Mon Sep 17 00:00:00 2001 From: Copilot Date: Tue, 5 May 2026 22:41:00 +0200 Subject: [PATCH 15/19] Address review: widen allowed-files to product/runtime sources, allow small product fixes, schedule every 12h Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.lock.yml | 34 +++++++++++----------- .github/workflows/ci-failure-scan.md | 13 +++++---- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index f5bd6c6bd23f72..02c556b57245bd 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"fd3979b0d6d2e0582a73bcf0677e0174d7a07cdacefd3b046d926c6713f2b19c","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"0fd237d1bd4990a30b09f3925ffc4767695576788469b2f10089d5bef508d29c","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc","version":"v0.68.1"}]} # ___ _ _ # / _ \ | | (_) @@ -55,8 +55,8 @@ name: "CI Outer-Loop Failure Scanner" # - maintainer # Roles processed as role check in pre-activation job # - write # Roles processed as role check in pre-activation job schedule: - - cron: "34 */6 * * *" - # Friendly format: every 6h (scattered) + - cron: "34 */12 * * *" + # Friendly format: every 12h (scattered) # steps: # Steps injected into pre-activation job # - name: Checkout the select-copilot-pat action folder # uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -197,19 +197,19 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' + cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' - GH_AW_PROMPT_1705160421966dcc_EOF + GH_AW_PROMPT_a0e100c6041a3947_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' + cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' Tools: create_issue(max:5), create_pull_request(max:10), missing_tool, missing_data, noop - GH_AW_PROMPT_1705160421966dcc_EOF + GH_AW_PROMPT_a0e100c6041a3947_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' + cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' The following GitHub context information is available for this workflow: @@ -242,12 +242,12 @@ jobs: - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). - GH_AW_PROMPT_1705160421966dcc_EOF + GH_AW_PROMPT_a0e100c6041a3947_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_1705160421966dcc_EOF' + cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' {{#runtime-import .github/workflows/ci-failure-scan.md}} - GH_AW_PROMPT_1705160421966dcc_EOF + GH_AW_PROMPT_a0e100c6041a3947_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -413,9 +413,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_f855a461e7e8f6dc_EOF' - {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**/tests/**","src/libraries/Common/tests/**","src/coreclr/**/tests/**","src/mono/**/tests/**","src/tests/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_f855a461e7e8f6dc_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_7fb9ae1419858410_EOF' + {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**","src/coreclr/**","src/mono/**","src/tests/**","src/native/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_7fb9ae1419858410_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -645,7 +645,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_7f9449a6cdffe668_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_dbd370a9d32d3516_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -689,7 +689,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_7f9449a6cdffe668_EOF + GH_AW_MCP_CONFIG_dbd370a9d32d3516_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -1354,7 +1354,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dev.azure.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,helix.dot.net,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**/tests/**\",\"src/libraries/Common/tests/**\",\"src/coreclr/**/tests/**\",\"src/mono/**/tests/**\",\"src/tests/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**\",\"src/coreclr/**\",\"src/mono/**\",\"src/tests/**\",\"src/native/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 8a9c81b5378e94..717ddc47df44d8 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -8,7 +8,7 @@ permissions: pull-requests: read on: - schedule: every 6h + schedule: every 12h workflow_dispatch: roles: [admin, maintainer, write] @@ -84,11 +84,11 @@ safe-outputs: max: 10 protected-files: blocked allowed-files: - - "src/libraries/**/tests/**" - - "src/libraries/Common/tests/**" - - "src/coreclr/**/tests/**" - - "src/mono/**/tests/**" + - "src/libraries/**" + - "src/coreclr/**" + - "src/mono/**" - "src/tests/**" + - "src/native/**" - "eng/testing/**" labels: [agentic-workflows] create-issue: @@ -159,7 +159,8 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing an **existing** issue. - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. - **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. -- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file or compile error from the log. Do not attempt an `allowed-files` PR for product code unless the fix is one-line and clearly limited to test infrastructure under `eng/testing/**`. +- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → if the compile error has a clear, mechanical root cause and the fix is **≤ 20 lines in a single file** (e.g., obvious typo, missing `#if`, wrong type cast, missing `using`), open a draft PR with the fix and reference the failing build. Otherwise file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file and compile error from the log. +- **Small product fix opportunity** — if a test failure has a clear, well-localized root cause in product code (≤ 20 lines, single file, behavioral fix verifiable by the failing test), and the fix is in `src/libraries/**`, `src/coreclr/**`, `src/mono/**`, or `src/native/**`, open a draft PR with the fix. The PR body must (a) cite the failing test as evidence, (b) explain the root cause, and (c) state explicitly why the fix is safe (e.g., "doesn't change public API", "only affects the failing path"). Do not attempt this for changes touching: public API surface, JIT codegen, GC, threading, security primitives, or anything > 20 lines. When in doubt, file an issue instead — a small wrong fix is worse than a tracking issue. - **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **two-pass resolution across runs**: 1. First run (no existing tracking issue found): file the tracking issue only. Do **not** open a PR — the issue number isn't available inside a patch (safe-outputs only resolve `aw_` tokens in body text, never inside diffs). 2. Subsequent run (a matching tracking issue with `area-*` + `arch-*` + `os-*` + stress-mode signature already exists, AND the failure is still occurring in the latest scanned build): you **MUST** open a draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s). This is the convergence step — without it, the pipeline never goes green. Skipping the PR because "the issue is already tracked" is incorrect; the existence of the issue is precisely what enables the PR. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the existing tracking issue. Hardcode the issue number in the diff because the issue already exists. From c4bcd070a97bb5ad4a89ff23bf5d894adb3276ac Mon Sep 17 00:00:00 2001 From: Copilot Date: Wed, 6 May 2026 12:45:47 +0200 Subject: [PATCH 16/19] Address review: KBE-first two-pass flow, JIT issue template, label discipline, more pipelines Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.lock.yml | 30 +++--- .github/workflows/ci-failure-scan.md | 109 +++++++++++++++++---- 2 files changed, 107 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index 02c556b57245bd..004a47cc0c47fb 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"0fd237d1bd4990a30b09f3925ffc4767695576788469b2f10089d5bef508d29c","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"905a4e2ef7b0e251ef41933431f6ab4d8eb831025a06ea68bfa2dd8db6c17c4d","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc","version":"v0.68.1"}]} # ___ _ _ # / _ \ | | (_) @@ -22,7 +22,7 @@ # # For more information: https://github.github.com/gh-aw/introduction/overview/ # -# Periodic platform-agnostic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Fixes per-test failures via PR; files an actionable tracking issue otherwise. +# Periodic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Files Known Build Errors so failures are immediately ignorable in PR CI; opens companion skip PRs to remove the failure permanently after human review. # # Secrets used: # - COPILOT_GITHUB_TOKEN @@ -197,19 +197,19 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' + cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' - GH_AW_PROMPT_a0e100c6041a3947_EOF + GH_AW_PROMPT_92419be4a38f6bda_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' + cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' Tools: create_issue(max:5), create_pull_request(max:10), missing_tool, missing_data, noop - GH_AW_PROMPT_a0e100c6041a3947_EOF + GH_AW_PROMPT_92419be4a38f6bda_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' + cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' The following GitHub context information is available for this workflow: @@ -242,12 +242,12 @@ jobs: - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). - GH_AW_PROMPT_a0e100c6041a3947_EOF + GH_AW_PROMPT_92419be4a38f6bda_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_a0e100c6041a3947_EOF' + cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' {{#runtime-import .github/workflows/ci-failure-scan.md}} - GH_AW_PROMPT_a0e100c6041a3947_EOF + GH_AW_PROMPT_92419be4a38f6bda_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -413,9 +413,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_7fb9ae1419858410_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_233364f05b5f8ced_EOF' {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**","src/coreclr/**","src/mono/**","src/tests/**","src/native/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_7fb9ae1419858410_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_233364f05b5f8ced_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -645,7 +645,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_dbd370a9d32d3516_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_642e68d7c7f69051_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -689,7 +689,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_dbd370a9d32d3516_EOF + GH_AW_MCP_CONFIG_642e68d7c7f69051_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -1138,7 +1138,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 env: WORKFLOW_NAME: "CI Outer-Loop Failure Scanner" - WORKFLOW_DESCRIPTION: "Periodic platform-agnostic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Fixes per-test failures via PR; files an actionable tracking issue otherwise." + WORKFLOW_DESCRIPTION: "Periodic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Files Known Build Errors so failures are immediately ignorable in PR CI; opens companion skip PRs to remove the failure permanently after human review." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} with: script: | diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 717ddc47df44d8..78c52288bfc0ad 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -1,6 +1,6 @@ --- name: "CI Outer-Loop Failure Scanner" -description: "Periodic platform-agnostic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Fixes per-test failures via PR; files an actionable tracking issue otherwise." +description: "Periodic scan of runtime-extra-platforms and outer-loop CI pipelines (JIT/GC stress, PGO, libraries-jitstress, etc.). Files Known Build Errors so failures are immediately ignorable in PR CI; opens companion skip PRs to remove the failure permanently after human review." permissions: contents: read @@ -117,27 +117,38 @@ Iterate over every pipeline in this list. For each, fetch builds on branch `main | Pipeline | Definition ID | Notes | |----------|---------------|-------| | runtime-extra-platforms | 154 | Apple mobile, Android, browser, wasi, NativeAOT outer loop | +| runtime-coreclr outerloop | 108 | | | runtime-coreclr jitstress | 109 | JIT stress modes | | runtime-coreclr jitstressregs | 110 | | | runtime-coreclr jitstress2-jitstressregs | 111 | | | runtime-coreclr gcstress0x3-gcstress0xc | 112 | | | runtime-coreclr gcstress-extra | 113 | | +| runtime-coreclr r2r-extra | 114 | | | runtime-coreclr jitstress-isas-x86 | 115 | | | runtime-coreclr jitstress-isas-arm | 116 | | | runtime-coreclr jitstressregs-x86 | 117 | | | runtime-coreclr libraries-jitstressregs | 118 | | | runtime-coreclr libraries-jitstress2-jitstressregs | 119 | | +| runtime-coreclr r2r | 120 | | +| runtime-coreclr gc-simulator | 123 | | +| runtime-coreclr crossgen2 | 124 | | | runtime-jit-experimental | 137 | OSR / partial compilation | | runtime-coreclr libraries-jitstress | 138 | | | runtime-coreclr ilasm | 140 | | | runtime-coreclr pgo | 144 | | | runtime-coreclr libraries-pgo | 145 | | +| gc-standalone | 146 | ADO name differs from display name | | runtime-coreclr superpmi-replay | 150 | | +| runtime-coreclr superpmi-asmdiffs-checked-release | 153 | | | runtime-coreclr jit-cfg | 155 | Control flow guard | | runtime-coreclr jitstress-random | 159 | Stress mode value comes from logs | | runtime-coreclr libraries-jitstress-random | 160 | Stress mode value comes from logs | | runtime-coreclr pgostress | 230 | | | runtime-coreclr jitstress-isas-avx512 | 235 | | +| runtime-nativeaot-outerloop | 265 | | +| runtime-diagnostics | 309 | | +| runtime-interpreter | 316 | ADO name differs from display name | +| runtime-libraries-interpreter | 330 | ADO name differs from display name | If a pipeline has no completed build in the last 7 days, skip it silently. @@ -153,26 +164,55 @@ Read the relevant skill before classifying / fixing. Skills live under `.github/ ## Outcome (per actionable failure) -- **Per-test platform / configuration incompatibility** (e.g., test fails only under `jitstress=2`, `gcstress=0xC`, on a single mobile arch, on browser, on NativeAOT) → open a draft PR with a per-test attribute change: - - `[SkipOnPlatform(TestPlatforms., "")]` for platform-specific failures. - - `[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.))]` narrowed via existing helpers. - - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing an **existing** issue. - - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or wrap with a guard helper that checks `DOTNET_JitStress`/`DOTNET_GCStress` env vars where the existing test infra supports it. -- **Recurring flaky failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window, no obvious product fix in flight, blocking unrelated PRs) → file a **Known Build Error** issue (see "Known Build Error issue" section below). This lets Arcade Build Analysis auto-match future hits and unblock PRs. -- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → if the compile error has a clear, mechanical root cause and the fix is **≤ 20 lines in a single file** (e.g., obvious typo, missing `#if`, wrong type cast, missing `using`), open a draft PR with the fix and reference the failing build. Otherwise file a regular tracking issue (NOT a Known Build Error — Build Analysis explicitly forbids that for build breaks). Reference the failing source file and compile error from the log. -- **Small product fix opportunity** — if a test failure has a clear, well-localized root cause in product code (≤ 20 lines, single file, behavioral fix verifiable by the failing test), and the fix is in `src/libraries/**`, `src/coreclr/**`, `src/mono/**`, or `src/native/**`, open a draft PR with the fix. The PR body must (a) cite the failing test as evidence, (b) explain the root cause, and (c) state explicitly why the fix is safe (e.g., "doesn't change public API", "only affects the failing path"). Do not attempt this for changes touching: public API surface, JIT codegen, GC, threading, security primitives, or anything > 20 lines. When in doubt, file an issue instead — a small wrong fix is worse than a tracking issue. -- **Product regression — JIT/GC/runtime bug surfaced by a stress mode** (native crash, GC assertion, JIT-emitted bad code) → **two-pass resolution across runs**: - 1. First run (no existing tracking issue found): file the tracking issue only. Do **not** open a PR — the issue number isn't available inside a patch (safe-outputs only resolve `aw_` tokens in body text, never inside diffs). - 2. Subsequent run (a matching tracking issue with `area-*` + `arch-*` + `os-*` + stress-mode signature already exists, AND the failure is still occurring in the latest scanned build): you **MUST** open a draft PR adding `[ActiveIssue("https://github.com/dotnet/runtime/issues/", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` (or the `TestPlatforms.` overload for platform-specific stress) to the failing test(s). This is the convergence step — without it, the pipeline never goes green. Skipping the PR because "the issue is already tracked" is incorrect; the existence of the issue is precisely what enables the PR. The PR title prefix is `[ci-scan]`. The PR body's "Linked issue" section must link the existing tracking issue. Hardcode the issue number in the diff because the issue already exists. +The primary purpose of this workflow is to keep PR CI green. **KBE** = Known Build Error: an issue tagged `Known Build Error` whose body contains a JSON `ErrorMessage`/`ErrorPattern` block that Arcade Build Analysis matches against future failure logs to mark them as already-tracked, so unrelated PRs aren't blocked. KBEs are immediately effective for PR CI; muting PRs are not effective until merged by a human (latency ≥ 12h, often days). The workflow runs every 12h and converges on **two artifacts per failure across two runs**: KBE in run N (immediate), muting PR in run N+1 (permanent after merge), with a small-fix PR added in run N+1 when scope allows. + +### Per-failure deliverables + +For each actionable failure, produce **up to three artifacts**: + +1. **KBE** — immediate Build Analysis signal so PR CI is unblocked right away. Always produced (or reused if one already exists) for stable-signature failures. +2. **Muting PR** — small, clean, mergeable PR that just adds `[ActiveIssue(...)]` / `` referencing the KBE. No diagnosis logic, no product code. Designed to be merge-without-thinking by any maintainer who agrees the failure should be silenced. Always produced when (1) is produced. +3. **Fix PR** — actual product/test code fix. Produced **only when** (a) the root cause is clear from the failure log, (b) the change fits the "small product fix opportunity" bounds (≤ 20 lines, single file, non-API, non-JIT-codegen, non-GC, non-threading, non-security), and (c) the failing test verifies the fix. Otherwise the deeper investigation is left to the area owner via the KBE — do NOT attempt a speculative fix PR. + +The muting PR and the fix PR are independent: a maintainer can merge the muting PR immediately (CI goes green) and then iterate on the fix PR at human pace. If the fix PR lands first, the muting PR becomes a no-op and can be closed; if the muting PR lands first, the fix PR removes the `[ActiveIssue]` annotation. + +### Two-pass KBE → PR flow (across runs) + +Same-run KBE + PR is not possible: gh-aw strict mode forbids `issues: write` on the agent job, so the agent cannot create issues at runtime — it can only emit safe-outputs `create_issue` directives that are processed by a separate post-agent job after the agent finishes. Issue numbers are therefore never visible to the agent during execution. Patches cannot reference an issue number that doesn't exist yet. + +The agent must accept this constraint and produce KBEs in run N, then companion PRs in run N+1. The 12-hour cadence makes this acceptable: the KBE alone unblocks PR CI immediately (the moment the safe-outputs job processes it, ~1 min after the agent finishes), and the muting PR follows within 12h. + +For each actionable failure, in this order: + +1. **Search for existing artifacts** before creating anything new: + - `search_issues` for an open KBE: `is:issue is:open label:"Known Build Error" in:body ""`. Try variations on the signature (full `[FAIL]` line, assertion text, exception type + test name). + - `search_pull_requests` for an open muting PR that already silences this test: `is:pr is:open in:title "" "[ci-scan]"` and `is:pr is:open "" ActiveIssue`. + - `search_pull_requests` for an open small-fix PR: `is:pr is:open in:title "" "[ci-scan]"`. + - If a KBE + muting PR already cover this failure, **skip** — record it in the coverage tally as `→ already-covered: KBE # + PR #` and move on. Do not duplicate. +2. **No existing KBE → file one via safe-outputs `create_issue`**. Apply labels: `Known Build Error`, `blocking-clean-ci`, plus any verified `area-*` / `os-*` / `arch-*` (see "Never invent labels" below). Title prefix: `[ci-scan] `. Body: the KBE format described in "Known Build Error issue" below. The safe-outputs handler will create the issue ~1 minute after the agent finishes; the issue number is not available to the agent during this run. +3. **Existing KBE found AND failure still occurring AND no muting PR exists yet → open the muting PR via safe-outputs `create_pull_request`** with the existing KBE issue number hardcoded in the diff: `[ActiveIssue("https://github.com/dotnet/runtime/issues/", ...)]` for unit tests, `true` (with an inline `` comment) for stress-incompatible JIT csproj families. PR title prefix `[ci-scan] `; PR body's "Linked KBE" section links the issue. This PR must change **only test annotations / csproj test-config flags** — no product code, no diagnosis, no logic. Aim for ≤ 5 lines of diff. +4. **(Optional, alongside step 3) Open a small-fix PR via safe-outputs `create_pull_request`** if the failure satisfies the "small product fix opportunity" criteria above. Separate PR, separate branch, separate diff. PR body must (a) cite the failing test as evidence, (b) explain the root cause, (c) state explicitly why the fix is safe, and (d) note "If this lands before #, that PR can be closed". Do not bundle the fix into the muting PR — keep them separate so a maintainer can take one without the other. - To find the matching existing issue, `search_issues` for: `is:issue is:open label:area- in:title ""` and `is:issue is:open label:area- ""`. Match on test name + stress mode + arch. If multiple candidates, pick the one with the most recent activity. If you found such an issue earlier in the run while deduplicating, **do not stop there — proceed to file the muting PR**. +Caps: safe-outputs `create_issue` max 5/run, `create_pull_request` max 10/run. When a cap is hit, fall back to "skipped: cap reached" rather than silently dropping signatures — subsequent runs will pick them up. - After every run, you should be able to answer YES to: "for each still-failing stress-mode product bug whose tracking issue already exists, did I emit a `create_pull_request` output?" If the answer is NO for any of them, you have not done the job. -- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue. Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. +After every run, you should be able to answer YES to: "for each actionable failure I encountered, did I either (a) confirm a KBE+PR pair already covers it, or (b) file the missing piece (KBE or PR or both, depending on what was missing)?" If the answer is NO for any of them, you have not done the job. + +### Per-failure-class rules + +- **Recurring failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window) → KBE + muting PR (mandatory) + fix PR (optional, only if criteria met). +- **Per-test platform / configuration incompatibility** (e.g., test fails only under `jitstress=2`, `gcstress=0xC`, on a single mobile arch, on browser, on NativeAOT) — KBE + muting PR (mandatory). Allowed muting PR mechanisms: + - `[SkipOnPlatform(TestPlatforms., "")]` for platform-specific failures. + - `[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.))]` narrowed via existing helpers. + - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing the KBE. + - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or `true` at the csproj level. **Tradeoff**: stress-guarded skips remove the test signal from the stress pipelines, so the bug becomes invisible in those pipelines until the JIT fix lands. The KBE you filed in step 2 is what keeps the JIT team aware; without that KBE, the muting PR alone would silently lose the signal. +- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → if the compile error has a clear, mechanical root cause and the fix is **≤ 20 lines in a single file** (e.g., obvious typo, missing `#if`, wrong type cast, missing `using`), open a fix PR. **No KBE for build breaks** — Build Analysis explicitly forbids that. If the fix is non-trivial, file a regular tracking issue and reference the failing source file and compile error. +- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue (not a KBE). Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. -**Convergence target**: leave each pipeline green-on-next-run by either (a) muting the failing test via PR or (b) filing a Known Build Error issue that Build Analysis can auto-match. A tracking-issue-only outcome is acceptable only for build breaks and infra failures where there is nothing to mute on the test side. +**Convergence target**: for every actionable test/runtime failure, leave the run with both (a) a KBE filed (immediate effect on PR CI via Build Analysis) and (b) a clean muting PR open against that KBE (permanent effect after merge, low review cost). The fix PR is a bonus when the root cause is obviously small. A tracking-issue-only outcome is acceptable only for build breaks (which Build Analysis cannot match) and infra failures. + +After every run, you should be able to answer YES to: "for each actionable failure I encountered, did I emit (or already find) both a KBE and a muting PR?" If the answer is NO for any of them, you have not done the job. Do not emit `noop`. Either a PR or an issue must come out of every actionable failure. @@ -220,6 +260,40 @@ Use this when a PR is not the right tool — product regression, native crash, m Same `os-*`, `area-*`, `arch-*` labels. +### JIT pipeline issue template (definitions 109–160, 230, 235, 108, 137, 144–145, 150, 153) + +For tracking issues filed against a JIT, GC, PGO, or stress pipeline, use this body layout instead of the generic "five sections" above (matches the in-repo convention; see #125685 for the canonical example): + +``` +**Summary:** + + +**Failed in ():** +- [ ]() +- [ ]() +- ... + +**Console Log:** [Console Log]() + +**Failed tests:** +(use a fenced code block; per-pipeline, list the failing legs and tests) + +- + - + - + +- + - + +**Error Message:** +(fenced code block with the canonical error line) + +**Stack Trace:** +(fenced code block with the relevant stack trace; trim noise but keep the failing frame) +``` + +This format makes the issue immediately actionable for JIT/GC owners (@JulieLeeMSFT, @BruceForstall, @jakobbotsch, @dotnet/jit-contrib) without further drilldown. Apply the appropriate `area-CodeGen-coreclr` / `area-GC-coreclr` / `area-PGO-coreclr` / `area-Tools-ILVerification` label per the failing pipeline. + ## Outputs: title and labels - **All issues and PRs MUST have title prefix `[ci-scan] `**, including tracking issues, Known Build Error issues, and muting PRs. Examples: @@ -227,6 +301,7 @@ Same `os-*`, `area-*`, `arch-*` labels. - `[ci-scan] Known Build Error: ` - `[ci-scan] Skip under (refs #)` - **Do not use the word "Mute" or "Muting"** in titles. Use "Skip", "Disable", "Suppress", or "Exclude" depending on the mechanism. Examples: "Skip … under GCStress", "Disable … on tvOS", "Suppress … in MiniFull AOT mode". +- **Never invent labels.** Only apply labels that already exist on dotnet/runtime. Before applying any `area-*` label, verify it exists by listing repository labels via `curl -s 'https://api.github.com/repos/dotnet/runtime/labels?per_page=100&page=N'` (with pre-bound URL variable) and grep for the candidate name. If the canonical owner label cannot be confirmed, omit the `area-*` label entirely and let triage assign it — do not invent a plausible-looking label like `area-Extensions-FileProviders` that does not exist. The same rule applies to `os-*`, `arch-*`, and any other label families. Stick to labels you have verified. - Labels are unchanged from the per-outcome rules above. ## Known Build Error issue @@ -269,7 +344,7 @@ Title: `Test failure: ` for test failures, or `Known B Labels: `Known Build Error`, `blocking-clean-ci`, plus the test's `area-*` label and any `os-*` / `arch-*` labels that apply. -Before filing, search for an existing Known Build Error issue with a matching `ErrorMessage` (`label:"Known Build Error" in:body ""`). If one exists and is open, do not duplicate — instead append the new build to the existing issue's body via an issue comment with the build link, leg, and timestamp. +Before filing, search for an existing Known Build Error issue with a matching `ErrorMessage` (`label:"Known Build Error" in:body ""`). If one exists and is open, **skip silently — do not duplicate, do not append a comment**. Build Analysis already counts the new occurrence in its hit-count summary on the issue body; piling on issue comments per occurrence creates noise on already-noisy KBEs (some have tens of hits per run). If `search_issues` returns no matches, proceed to file the new KBE. ## Hard environment constraints From e42c9e918696926f81417f89ef22875c902986c4 Mon Sep 17 00:00:00 2001 From: Copilot Date: Wed, 6 May 2026 13:18:15 +0200 Subject: [PATCH 17/19] Address review: explicit Linked KBE line, per-pass YES check, [ci-scan] on KBE title, dedup convergence statement Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.md | 30 +++++++++++++++++----------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 78c52288bfc0ad..84011ad7c80f95 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -190,29 +190,35 @@ For each actionable failure, in this order: - `search_pull_requests` for an open small-fix PR: `is:pr is:open in:title "" "[ci-scan]"`. - If a KBE + muting PR already cover this failure, **skip** — record it in the coverage tally as `→ already-covered: KBE # + PR #` and move on. Do not duplicate. 2. **No existing KBE → file one via safe-outputs `create_issue`**. Apply labels: `Known Build Error`, `blocking-clean-ci`, plus any verified `area-*` / `os-*` / `arch-*` (see "Never invent labels" below). Title prefix: `[ci-scan] `. Body: the KBE format described in "Known Build Error issue" below. The safe-outputs handler will create the issue ~1 minute after the agent finishes; the issue number is not available to the agent during this run. -3. **Existing KBE found AND failure still occurring AND no muting PR exists yet → open the muting PR via safe-outputs `create_pull_request`** with the existing KBE issue number hardcoded in the diff: `[ActiveIssue("https://github.com/dotnet/runtime/issues/", ...)]` for unit tests, `true` (with an inline `` comment) for stress-incompatible JIT csproj families. PR title prefix `[ci-scan] `; PR body's "Linked KBE" section links the issue. This PR must change **only test annotations / csproj test-config flags** — no product code, no diagnosis, no logic. Aim for ≤ 5 lines of diff. -4. **(Optional, alongside step 3) Open a small-fix PR via safe-outputs `create_pull_request`** if the failure satisfies the "small product fix opportunity" criteria above. Separate PR, separate branch, separate diff. PR body must (a) cite the failing test as evidence, (b) explain the root cause, (c) state explicitly why the fix is safe, and (d) note "If this lands before #, that PR can be closed". Do not bundle the fix into the muting PR — keep them separate so a maintainer can take one without the other. +3. **Existing KBE found AND failure still occurring AND no muting PR exists yet → open the muting PR via safe-outputs `create_pull_request`** with the existing KBE issue number hardcoded in the diff: `[ActiveIssue("https://github.com/dotnet/runtime/issues/", ...)]` for unit tests, `true` (with an inline `` comment) for stress-incompatible JIT csproj families. PR title prefix `[ci-scan] `; **the PR body MUST include a top-level "Linked KBE" line of the form `Linked KBE: #` so the link is unambiguous and machine-readable**, in addition to the prose "Linked KBE" section. This PR must change **only test annotations / csproj test-config flags** — no product code, no diagnosis, no logic. Aim for ≤ 5 lines of diff. +4. **(Optional, alongside step 3) Open a small-fix PR via safe-outputs `create_pull_request`** if the failure satisfies the "small product fix opportunity" criteria above. Separate PR, separate branch, separate diff. PR body must (a) cite the failing test as evidence, (b) explain the root cause, (c) state explicitly why the fix is safe, (d) include `Linked KBE: #` as a top-level line, and (e) note "If this lands before #, that PR can be closed". Do not bundle the fix into the muting PR — keep them separate so a maintainer can take one without the other. Caps: safe-outputs `create_issue` max 5/run, `create_pull_request` max 10/run. When a cap is hit, fall back to "skipped: cap reached" rather than silently dropping signatures — subsequent runs will pick them up. -After every run, you should be able to answer YES to: "for each actionable failure I encountered, did I either (a) confirm a KBE+PR pair already covers it, or (b) file the missing piece (KBE or PR or both, depending on what was missing)?" If the answer is NO for any of them, you have not done the job. +After every run, you should be able to answer YES to **whichever of these applies to each failure**: + +- **First-encounter failure (no existing KBE):** "Did I file the KBE?" Muting/fix PRs are deferred to the next run — they cannot reference an issue number that doesn't exist yet at agent runtime. +- **Existing KBE, no muting PR yet:** "Did I open the muting PR (and, if criteria met, the small-fix PR)?" +- **Existing KBE + existing muting PR:** "Did I confirm both, skip silently, and record `→ already-covered: KBE # + PR #` in the coverage tally?" + +If the answer is NO for any failure, you have not done the job. ### Per-failure-class rules -- **Recurring failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window) → KBE + muting PR (mandatory) + fix PR (optional, only if criteria met). -- **Per-test platform / configuration incompatibility** (e.g., test fails only under `jitstress=2`, `gcstress=0xC`, on a single mobile arch, on browser, on NativeAOT) — KBE + muting PR (mandatory). Allowed muting PR mechanisms: +The two-pass flow above applies to all classes below. "KBE + muting PR" means: KBE in the run that first encounters the failure, muting PR in the next run that finds the KBE already exists. + +- **Recurring failure with a stable error signature** (≥ 2 occurrences on `main` in the scanned window) → KBE (run N) + muting PR (run N+1) + fix PR (optional, run N+1, only if criteria met). +- **Per-test platform / configuration incompatibility** (e.g., test fails only under `jitstress=2`, `gcstress=0xC`, on a single mobile arch, on browser, on NativeAOT) → KBE (run N) + muting PR (run N+1). Allowed muting PR mechanisms: - `[SkipOnPlatform(TestPlatforms., "")]` for platform-specific failures. - `[ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.))]` narrowed via existing helpers. - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing the KBE. - - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or `true` at the csproj level. **Tradeoff**: stress-guarded skips remove the test signal from the stress pipelines, so the bug becomes invisible in those pipelines until the JIT fix lands. The KBE you filed in step 2 is what keeps the JIT team aware; without that KBE, the muting PR alone would silently lose the signal. -- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → if the compile error has a clear, mechanical root cause and the fix is **≤ 20 lines in a single file** (e.g., obvious typo, missing `#if`, wrong type cast, missing `using`), open a fix PR. **No KBE for build breaks** — Build Analysis explicitly forbids that. If the fix is non-trivial, file a regular tracking issue and reference the failing source file and compile error. -- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue (not a KBE). Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and update its description in place rather than duplicating. + - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or `true` at the csproj level. **Tradeoff**: stress-guarded skips remove the test signal from the stress pipelines, so the bug becomes invisible in those pipelines until the JIT fix lands. The KBE filed in run N is what keeps the JIT team aware; without that KBE, the muting PR alone would silently lose the signal. +- **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → if the compile error has a clear, mechanical root cause and the fix is **≤ 20 lines in a single file** (e.g., obvious typo, missing `#if`, wrong type cast, missing `using`), open a fix PR (no KBE — Build Analysis explicitly forbids KBEs for build breaks). If the fix is non-trivial, file a regular tracking issue and reference the failing source file and compile error. +- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue (not a KBE). Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and skip silently if one already exists (do not duplicate, do not append a comment — the agent only has read permission on existing issues). For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. -**Convergence target**: for every actionable test/runtime failure, leave the run with both (a) a KBE filed (immediate effect on PR CI via Build Analysis) and (b) a clean muting PR open against that KBE (permanent effect after merge, low review cost). The fix PR is a bonus when the root cause is obviously small. A tracking-issue-only outcome is acceptable only for build breaks (which Build Analysis cannot match) and infra failures. - -After every run, you should be able to answer YES to: "for each actionable failure I encountered, did I emit (or already find) both a KBE and a muting PR?" If the answer is NO for any of them, you have not done the job. +**Convergence target**: across two consecutive runs, every actionable test/runtime failure ends up with both (a) a KBE filed (immediate effect on PR CI via Build Analysis) and (b) a clean muting PR open against that KBE (permanent effect after merge, low review cost). The fix PR is a bonus when the root cause is obviously small. A tracking-issue-only outcome is acceptable only for build breaks (which Build Analysis cannot match) and infra failures. Do not emit `noop`. Either a PR or an issue must come out of every actionable failure. @@ -340,7 +346,7 @@ The pseudo-instructions `(open three backticks, then ...)` and `(close three bac Choose `ErrorMessage` (substring) by default. Use `ErrorPattern` only when a regex is genuinely needed and confirm it has no catastrophic backtracking. Set `BuildRetry: true` **only** for confirmed infra/queue-side flakes (dead-letter, device-lost, agent disconnect) where retrying is safe. -Title: `Test failure: ` for test failures, or `Known Build Error: ` for non-test build errors. +Title: `[ci-scan] Test failure: ` for test failures, or `[ci-scan] Known Build Error: ` for non-test build errors. The `[ci-scan] ` prefix is mandatory on every issue and PR this workflow files (see "Outputs: title and labels" above). Labels: `Known Build Error`, `blocking-clean-ci`, plus the test's `area-*` label and any `os-*` / `arch-*` labels that apply. From 784fbb6d0d2d9929660b069f06641e4f43a8dcc1 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 6 May 2026 13:27:13 +0200 Subject: [PATCH 18/19] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 84011ad7c80f95..96a1e66c0ea71e 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -318,7 +318,7 @@ File one when **all** of the following hold: - The failure has occurred ≥ 2 times in the scanned window on `main`. - The error has a stable substring or regex signature that uniquely identifies it. - No fix PR is currently open (verify via `search_pull_requests`). -- The failure is **not** a build break — only test failures, hangs, or infra issues. Build breaks must use a regular issue. +- The failure is **not** a build break or an infrastructure failure — only test failures or hangs are eligible for a KBE. Build breaks and infra failures (for example dead-letter, device-lost, or agent-disconnect issues) must use a regular tracking issue. Required structure (Build Analysis is strict — match the headings exactly, and use **exactly three backticks** for the JSON code fence; never four. The opening and closing fence must be the same length, otherwise the fence is broken and Build Analysis silently skips the issue): From fc17e7a496c8f89629042018cb8d578e700ff55f Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Wed, 6 May 2026 18:24:46 +0200 Subject: [PATCH 19/19] Address review: hard label allowlist + KBE signature specificity rules - safe-outputs.create-issue.allowed-labels restricted to ["Known Build Error", "blocking-clean-ci"] - safe-outputs.create-pull-request.allowed-labels restricted to [agentic-workflows] - Stripped prose telling the model to add os-*/area-*/arch-* labels; added a single 'Labels (hard restriction)' clause under 'Outputs: title and labels'; KBE label line lists only the two permitted labels - Added 'Signature specificity (mandatory)' subsection: rejects bare exit codes / generic tool-failed verbs / bare exception types; requires assertion text or test+exception message; forbids padding ErrorMessage arrays with generic tokens; instructs model to file a regular issue (not a KBE) when no specific signature exists Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci-failure-scan.lock.yml | 34 +++++++++--------- .github/workflows/ci-failure-scan.md | 42 +++++++++++++++++----- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index 004a47cc0c47fb..45c464a68d5a98 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"905a4e2ef7b0e251ef41933431f6ab4d8eb831025a06ea68bfa2dd8db6c17c4d","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} +# gh-aw-metadata: {"schema_version":"v3","frontmatter_hash":"9f73c4276266e2b227107862cdbff63e464a2f91e9c3cb0a5a749b407cea2b5b","compiler_version":"v0.68.1","strict":true,"agent_id":"copilot","agent_model":"claude-sonnet-4.6"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9"},{"repo":"actions/upload-artifact","sha":"bbbca2ddaa5d8feaa63e36b76fdaad77386f024f","version":"v7"},{"repo":"github/gh-aw-actions/setup","sha":"2fe53acc038ba01c3bbdc767d4b25df31ca5bdfc","version":"v0.68.1"}]} # ___ _ _ # / _ \ | | (_) @@ -197,19 +197,19 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' + cat << 'GH_AW_PROMPT_416b8e1b41d3a8c5_EOF' - GH_AW_PROMPT_92419be4a38f6bda_EOF + GH_AW_PROMPT_416b8e1b41d3a8c5_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' + cat << 'GH_AW_PROMPT_416b8e1b41d3a8c5_EOF' Tools: create_issue(max:5), create_pull_request(max:10), missing_tool, missing_data, noop - GH_AW_PROMPT_92419be4a38f6bda_EOF + GH_AW_PROMPT_416b8e1b41d3a8c5_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' + cat << 'GH_AW_PROMPT_416b8e1b41d3a8c5_EOF' The following GitHub context information is available for this workflow: @@ -242,12 +242,12 @@ jobs: - **Note**: If a branch you need is not in the list above and is not listed as an additional fetched ref, it has NOT been checked out. For private repositories you cannot fetch it without proper authentication. If the branch is required and not available, exit with an error and ask the user to add it to the `fetch:` option of the `checkout:` configuration (e.g., `fetch: ["refs/pulls/open/*"]` for all open PR refs, or `fetch: ["main", "feature/my-branch"]` for specific branches). - GH_AW_PROMPT_92419be4a38f6bda_EOF + GH_AW_PROMPT_416b8e1b41d3a8c5_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/github_mcp_tools_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_92419be4a38f6bda_EOF' + cat << 'GH_AW_PROMPT_416b8e1b41d3a8c5_EOF' {{#runtime-import .github/workflows/ci-failure-scan.md}} - GH_AW_PROMPT_92419be4a38f6bda_EOF + GH_AW_PROMPT_416b8e1b41d3a8c5_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 @@ -413,16 +413,16 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_233364f05b5f8ced_EOF' - {"create_issue":{"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**","src/coreclr/**","src/mono/**","src/tests/**","src/native/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_233364f05b5f8ced_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_93f4d7e8f71e4c3f_EOF' + {"create_issue":{"allowed_labels":["Known Build Error","blocking-clean-ci"],"labels":["agentic-workflows"],"max":5},"create_pull_request":{"allowed_files":["src/libraries/**","src/coreclr/**","src/mono/**","src/tests/**","src/native/**","eng/testing/**"],"draft":true,"labels":["agentic-workflows"],"max":10,"max_patch_size":1024,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS"],"protected_files_policy":"blocked","protected_path_prefixes":[".github/",".agents/"],"title_prefix":"[ci-scan] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"true"},"report_incomplete":{}} + GH_AW_SAFE_OUTPUTS_CONFIG_93f4d7e8f71e4c3f_EOF - name: Write Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | { "description_suffixes": { - "create_issue": " CONSTRAINTS: Maximum 5 issue(s) can be created. Labels [\"agentic-workflows\"] will be automatically added.", - "create_pull_request": " CONSTRAINTS: Maximum 10 pull request(s) can be created. Title will be prefixed with \"[ci-scan] \". Labels [\"agentic-workflows\"] will be automatically added. PRs will be created as drafts." + "create_issue": " CONSTRAINTS: Maximum 5 issue(s) can be created. Labels [\"agentic-workflows\"] will be automatically added. Only these labels are allowed: [\"Known Build Error\" \"blocking-clean-ci\"].", + "create_pull_request": " CONSTRAINTS: Maximum 10 pull request(s) can be created. Title will be prefixed with \"[ci-scan] \". Labels [\"agentic-workflows\"] will be automatically added. Only these labels are allowed: [\"agentic-workflows\"]. PRs will be created as drafts." }, "repo_params": {}, "dynamic_tools": [] @@ -645,7 +645,7 @@ jobs: export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_MCP_GUARD_MIN_INTEGRITY -e GITHUB_MCP_GUARD_REPOS -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.2.17' mkdir -p /home/runner/.copilot - cat << GH_AW_MCP_CONFIG_642e68d7c7f69051_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" + cat << GH_AW_MCP_CONFIG_f1265124db2abfb7_EOF | bash "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.sh" { "mcpServers": { "github": { @@ -689,7 +689,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_642e68d7c7f69051_EOF + GH_AW_MCP_CONFIG_f1265124db2abfb7_EOF - name: Download activation artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -1354,7 +1354,7 @@ jobs: GH_AW_ALLOWED_DOMAINS: "*.blob.core.windows.net,*.githubusercontent.com,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,codeload.github.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dev.azure.com,docs.github.com,github-cloud.githubusercontent.com,github-cloud.s3.amazonaws.com,github.blog,github.com,github.githubassets.com,helix.dot.net,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,lfs.github.com,objects.githubusercontent.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} - GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**\",\"src/coreclr/**\",\"src/mono/**\",\"src/tests/**\",\"src/native/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"allowed_labels\":[\"Known Build Error\",\"blocking-clean-ci\"],\"labels\":[\"agentic-workflows\"],\"max\":5},\"create_pull_request\":{\"allowed_files\":[\"src/libraries/**\",\"src/coreclr/**\",\"src/mono/**\",\"src/tests/**\",\"src/native/**\",\"eng/testing/**\"],\"draft\":true,\"labels\":[\"agentic-workflows\"],\"max\":10,\"max_patch_size\":1024,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"AGENTS.md\"],\"protected_files_policy\":\"blocked\",\"protected_path_prefixes\":[\".github/\",\".agents/\"],\"title_prefix\":\"[ci-scan] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"true\"},\"report_incomplete\":{}}" GH_AW_CI_TRIGGER_TOKEN: ${{ secrets.GH_AW_CI_TRIGGER_TOKEN }} with: github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 96a1e66c0ea71e..c1aec2f41a41c8 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -91,9 +91,11 @@ safe-outputs: - "src/native/**" - "eng/testing/**" labels: [agentic-workflows] + allowed-labels: [agentic-workflows] create-issue: max: 5 labels: [agentic-workflows] + allowed-labels: ["Known Build Error", "blocking-clean-ci"] timeout-minutes: 90 @@ -189,7 +191,7 @@ For each actionable failure, in this order: - `search_pull_requests` for an open muting PR that already silences this test: `is:pr is:open in:title "" "[ci-scan]"` and `is:pr is:open "" ActiveIssue`. - `search_pull_requests` for an open small-fix PR: `is:pr is:open in:title "" "[ci-scan]"`. - If a KBE + muting PR already cover this failure, **skip** — record it in the coverage tally as `→ already-covered: KBE # + PR #` and move on. Do not duplicate. -2. **No existing KBE → file one via safe-outputs `create_issue`**. Apply labels: `Known Build Error`, `blocking-clean-ci`, plus any verified `area-*` / `os-*` / `arch-*` (see "Never invent labels" below). Title prefix: `[ci-scan] `. Body: the KBE format described in "Known Build Error issue" below. The safe-outputs handler will create the issue ~1 minute after the agent finishes; the issue number is not available to the agent during this run. +2. **No existing KBE → file one via safe-outputs `create_issue`**. The only labels permitted on KBE issues are `Known Build Error` and `blocking-clean-ci` (see "Outputs: title and labels" below). Title prefix: `[ci-scan] `. Body: the KBE format described in "Known Build Error issue" below. The safe-outputs handler will create the issue ~1 minute after the agent finishes; the issue number is not available to the agent during this run. 3. **Existing KBE found AND failure still occurring AND no muting PR exists yet → open the muting PR via safe-outputs `create_pull_request`** with the existing KBE issue number hardcoded in the diff: `[ActiveIssue("https://github.com/dotnet/runtime/issues/", ...)]` for unit tests, `true` (with an inline `` comment) for stress-incompatible JIT csproj families. PR title prefix `[ci-scan] `; **the PR body MUST include a top-level "Linked KBE" line of the form `Linked KBE: #` so the link is unambiguous and machine-readable**, in addition to the prose "Linked KBE" section. This PR must change **only test annotations / csproj test-config flags** — no product code, no diagnosis, no logic. Aim for ≤ 5 lines of diff. 4. **(Optional, alongside step 3) Open a small-fix PR via safe-outputs `create_pull_request`** if the failure satisfies the "small product fix opportunity" criteria above. Separate PR, separate branch, separate diff. PR body must (a) cite the failing test as evidence, (b) explain the root cause, (c) state explicitly why the fix is safe, (d) include `Linked KBE: #` as a top-level line, and (e) note "If this lands before #, that PR can be closed". Do not bundle the fix into the muting PR — keep them separate so a maintainer can take one without the other. @@ -214,7 +216,7 @@ The two-pass flow above applies to all classes below. "KBE + muting PR" means: K - `[ActiveIssue("https://github.com/dotnet/runtime/issues/", TestPlatforms.)]` referencing the KBE. - For JIT/GC stress: `[ActiveIssue("...", typeof(TestLibrary.PlatformDetection), nameof(TestLibrary.PlatformDetection.IsStressTest))]` or `true` at the csproj level. **Tradeoff**: stress-guarded skips remove the test signal from the stress pipelines, so the bug becomes invisible in those pipelines until the JIT fix lands. The KBE filed in run N is what keeps the JIT team aware; without that KBE, the muting PR alone would silently lose the signal. - **Build break on a single leg** (`Build product` or similar failed; `Send to Helix` skipped) → if the compile error has a clear, mechanical root cause and the fix is **≤ 20 lines in a single file** (e.g., obvious typo, missing `#if`, wrong type cast, missing `using`), open a fix PR (no KBE — Build Analysis explicitly forbids KBEs for build breaks). If the fix is non-trivial, file a regular tracking issue and reference the failing source file and compile error. -- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue (not a KBE). Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue with the matching `area-*` + `os-*` label and skip silently if one already exists (do not duplicate, do not append a comment — the agent only has read permission on existing issues). +- **Anything else** — multi-assembly cluster, infrastructure (queue exhaustion / dead-letter / device-lost) — file a tracking issue (not a KBE). Group all infra failures from one run into a single issue. Before filing, `search_issues` for an open issue whose title or body matches the same failure signature and skip silently if one already exists (do not duplicate, do not append a comment — the agent only has read permission on existing issues). For each failure compute a `(definition_id, work_item_or_phase, queue, stress_mode, [FAIL] or compile-error signature)` signature. Look back through ~10 completed builds in the same definition to build first-seen-in-window timestamp and occurrence count. @@ -254,9 +256,9 @@ Five H2 sections, in this exact order: 2. **Impact on platforms** — bullet list of `(pipeline + platform/arch + Helix queue + stress mode + exit code)` per affected occurrence. 3. **Errors log** — sanitized excerpt from the Helix console log (the `[FAIL]` line, the assertion or exception, and the `Failed tests:` summary). Strip JWTs, bearer tokens, `ApplicationGatewayAffinity*=`, and per-user paths. 4. **First build it occurred** — first build in the scanned window where this signature appeared: build link, finish time, commit SHA, occurrences-in-window count. State explicitly that this is computed within the scanned window and may not be the true origin. -5. **Linked issue** (optional) — if an `ActiveIssue` reference is used, link the issue and quote the matching label set. +5. **Linked issue** (optional) — if an `ActiveIssue` reference is used, link the issue. -Branch from `origin/main`. Stage only the files you intend to change with `git add `; never `git add -A`. Verify with `git diff --name-only --cached` before committing. Labels: at least one `os-*` (`os-android`, `os-ios`, `os-tvos`, `os-maccatalyst`, `os-browser`, `os-wasi`, `os-windows`, `os-linux`, `os-osx`) where applicable, plus the test's `area-*` label, plus `arch-*` for arch-specific failures, plus the relevant configuration label (`disabled-test`, `jit-stress`, `gc-stress`, `pgo`, `nativeaot`, …) when present. +Branch from `origin/main`. Stage only the files you intend to change with `git add `; never `git add -A`. Verify with `git diff --name-only --cached` before committing. Do not include any labels in the PR (see "Outputs: title and labels" below). ## Issue body @@ -264,7 +266,7 @@ Use this when a PR is not the right tool — product regression, native crash, m 5. **Recommended action** — concrete next step: which area owner, which file likely needs the fix, or what investigation would localize the root cause. For JIT/GC issues include the exact stress mode env vars and the JIT method-name from the log. Reference any related PR or issue you found via `search_issues`. The issue must be actionable — a checkbox-ready task list, not just "FYI". -Same `os-*`, `area-*`, `arch-*` labels. +Do not include any labels in the issue creation request (see "Outputs: title and labels" below). ### JIT pipeline issue template (definitions 109–160, 230, 235, 108, 137, 144–145, 150, 153) @@ -298,7 +300,7 @@ For tracking issues filed against a JIT, GC, PGO, or stress pipeline, use this b (fenced code block with the relevant stack trace; trim noise but keep the failing frame) ``` -This format makes the issue immediately actionable for JIT/GC owners (@JulieLeeMSFT, @BruceForstall, @jakobbotsch, @dotnet/jit-contrib) without further drilldown. Apply the appropriate `area-CodeGen-coreclr` / `area-GC-coreclr` / `area-PGO-coreclr` / `area-Tools-ILVerification` label per the failing pipeline. +This format makes the issue immediately actionable for JIT/GC owners (@JulieLeeMSFT, @BruceForstall, @jakobbotsch, @dotnet/jit-contrib) without further drilldown. Area triage (`area-CodeGen-coreclr` / `area-GC-coreclr` / `area-PGO-coreclr` / `area-Tools-ILVerification`) is added later by a human reviewer — do not propose any `area-*` label yourself. ## Outputs: title and labels @@ -307,8 +309,7 @@ This format makes the issue immediately actionable for JIT/GC owners (@JulieLeeM - `[ci-scan] Known Build Error: ` - `[ci-scan] Skip under (refs #)` - **Do not use the word "Mute" or "Muting"** in titles. Use "Skip", "Disable", "Suppress", or "Exclude" depending on the mechanism. Examples: "Skip … under GCStress", "Disable … on tvOS", "Suppress … in MiniFull AOT mode". -- **Never invent labels.** Only apply labels that already exist on dotnet/runtime. Before applying any `area-*` label, verify it exists by listing repository labels via `curl -s 'https://api.github.com/repos/dotnet/runtime/labels?per_page=100&page=N'` (with pre-bound URL variable) and grep for the candidate name. If the canonical owner label cannot be confirmed, omit the `area-*` label entirely and let triage assign it — do not invent a plausible-looking label like `area-Extensions-FileProviders` that does not exist. The same rule applies to `os-*`, `arch-*`, and any other label families. Stick to labels you have verified. -- Labels are unchanged from the per-outcome rules above. +- **Labels (hard restriction).** You **MUST NOT** propose any labels in your output. The workflow auto-applies `agentic-workflows` to every issue and PR, and additionally permits **only** `Known Build Error` and `blocking-clean-ci` on Known Build Error issues. Any other label — `os-*`, `area-*`, `arch-*`, `disabled-test`, `jit-stress`, `gc-stress`, `pgo`, `nativeaot`, `untriaged`, etc. — is rejected by `safe-outputs.allowed-labels` and **will be dropped**. Do not invent new labels under any name. Area, OS, and arch triage is performed by a human reviewer after the issue/PR is filed; do not attempt to pre-apply or guess them. ## Known Build Error issue @@ -346,9 +347,32 @@ The pseudo-instructions `(open three backticks, then ...)` and `(close three bac Choose `ErrorMessage` (substring) by default. Use `ErrorPattern` only when a regex is genuinely needed and confirm it has no catastrophic backtracking. Set `BuildRetry: true` **only** for confirmed infra/queue-side flakes (dead-letter, device-lost, agent disconnect) where retrying is safe. +### Signature specificity (mandatory) + +The `ErrorMessage` / `ErrorPattern` MUST uniquely identify **this specific failure mode**, not an entire category of crashes or build errors. A signature that would match unrelated future regressions is wrong and will mute legitimate failures. + +**Reject** signatures that consist only of: + +- A bare exit code or signal: `exitcode: 139`, `exit code 1`, `Segmentation fault`, `Aborted`, `SIGSEGV`, `SIGABRT`. +- A generic tool name + failure verb: `Crossgen2 failed`, `ilasm failed`, `dotnet build failed`, `xharness exited`. +- A bare exception type with no message: `BadImageFormatException`, `NullReferenceException`, `Fatal error. Invalid Program`, `Assertion failed`. +- A bare `[FAIL]` line with only the test class name and no exception/assertion text. +- Common infra strings: `Connection reset`, `Operation timed out`, `Resource temporarily unavailable`, `No space left on device`. + +**Prefer** signatures built from the most specific stable token in the log. In order of preference: + +1. The exact assertion text or exception **message** (not just the type), e.g. `Assertion failed 'comp->compHndBBtabCount == 0' in 'X' during 'Y'`. +2. The fully-qualified failing test name combined with a specific exception message, e.g. `System.Text.Json.Tests.Utf8JsonReaderTests.TestFoo … System.InvalidOperationException: Cannot read value of type X`. +3. A unique native stack frame or symbol from the crash dump excerpt, e.g. `coreclr!Compiler::fgMorphCall + 0x`. +4. A specific JIT method-being-compiled marker plus the specific stress mode, when the crash is JIT/GC stress only. + +**Combining signature parts** — a JSON array in `ErrorMessage` is AND-matched (all substrings must be present in the failure log). Do not pad an array with generic tokens like `exitcode: 139` or `Crash` alongside the specific message — those tokens add no specificity and only risk false negatives if the log format changes. Include at most one supplementary token, and only when it is itself non-generic (e.g. a specific assembly name or test name). + +If you cannot produce a signature that meets the bar above, **do not file a Known Build Error**. File a regular tracking issue instead and call out in "Recommended action" that the failure needs a stable signature before it can be muted. + Title: `[ci-scan] Test failure: ` for test failures, or `[ci-scan] Known Build Error: ` for non-test build errors. The `[ci-scan] ` prefix is mandatory on every issue and PR this workflow files (see "Outputs: title and labels" above). -Labels: `Known Build Error`, `blocking-clean-ci`, plus the test's `area-*` label and any `os-*` / `arch-*` labels that apply. +Labels: only `Known Build Error` and `blocking-clean-ci` are permitted on Known Build Error issues. Do not include any other label (no `area-*`, `os-*`, `arch-*`, etc.) — they will be rejected by `safe-outputs.allowed-labels`. Area and platform triage is added later by a human reviewer. Before filing, search for an existing Known Build Error issue with a matching `ErrorMessage` (`label:"Known Build Error" in:body ""`). If one exists and is open, **skip silently — do not duplicate, do not append a comment**. Build Analysis already counts the new occurrence in its hit-count summary on the issue body; piling on issue comments per occurrence creates noise on already-noisy KBEs (some have tens of hits per run). If `search_issues` returns no matches, proceed to file the new KBE.