From e8b71004e11fa90169b6a5394100ab8257c94cc7 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 06:22:30 +0100 Subject: [PATCH 1/2] fix(agentic-engineering): keep failed scheduled workflows red after main moves on The default-branch classifier read only the runs of the current head, so a scheduled workflow's failure disappeared at the next merge. It now also reads the branch's red scheduled runs of the last 31 days on any head (three complete reads) and each such workflow's newest scheduled success (one first-page read each), and joins them with the current-head runs. A workflow stays red until a later run of it succeeds; incomplete, capped or contradictory evidence still means unknown. Part of #669 Co-Authored-By: Claude Opus 5.5 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- .../.claude-plugin/plugin.json | 2 +- .../.codex-plugin/plugin.json | 2 +- plugins/agentic-engineering/CHANGELOG.md | 11 ++ plugins/agentic-engineering/README.md | 2 + .../agents/portfolio-surveyor.agent.md | 2 + plugins/agentic-engineering/plugin.json | 2 +- .../provider-neutral.desired-state.json | 4 +- .../classify-default-branch-ci-runs.sh | 163 ++++++++++++++++-- .../classify-default-branch-ci-runs.test.sh | 159 +++++++++++++++++ .../scripts/json-stream-performance.test.sh | 2 + .../scripts/json-stream.test.sh | 2 + .../scripts/surveyor-forge-readonly.test.sh | 7 + 14 files changed, 344 insertions(+), 18 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 351a332c..26168dfc 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -20,7 +20,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 351a332c..26168dfc 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -20,7 +20,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index 5df3b2d7..5802c1b5 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/.codex-plugin/plugin.json b/plugins/agentic-engineering/.codex-plugin/plugin.json index f605816d..6c4ea78a 100644 --- a/plugins/agentic-engineering/.codex-plugin/plugin.json +++ b/plugins/agentic-engineering/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/CHANGELOG.md b/plugins/agentic-engineering/CHANGELOG.md index e4ff8f07..4b5ff56f 100644 --- a/plugins/agentic-engineering/CHANGELOG.md +++ b/plugins/agentic-engineering/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog — `agentic-engineering` +## 7.6.1 — 2026-11-11 + +**Fixed** — the default-branch CI classifier no longer loses a failed scheduled workflow when +`main` moves on. A scheduled run uses whatever head `main` had at its cron time, so the next merge +used to hide its failure. The classifier now also reads the branch's failed, timed-out and +startup-failed scheduled runs of the last 31 days on any head, plus each such workflow's newest +scheduled success, and keeps the workflow red until a later run of it succeeds. That costs three +reads per repository plus one per failing scheduled workflow, and an incomplete or capped answer +still means unknown. A scheduled workflow whose last failure is older than 31 days is not seen, and +manually dispatched or managed runs keep the current-head rule (#669). + ## 7.6.0 — 2026-11-11 **Added** — two first-party skills for the rules around a product, both written from the diff --git a/plugins/agentic-engineering/README.md b/plugins/agentic-engineering/README.md index a3d39828..24ba1f21 100644 --- a/plugins/agentic-engineering/README.md +++ b/plugins/agentic-engineering/README.md @@ -611,6 +611,8 @@ under the guard: `--repo`, `--branch`, and a full `--head-sha`, and refuses the helper's offline `--input` mode. The helper rejects unrecognized outcomes, contradictory status/conclusion pairs, repeated run identities and invalid numeric identifiers before selecting the latest workflow result. + It also reads the branch's red scheduled runs of the last 31 days on any head, and each such + workflow's newest scheduled success, so a scheduled failure is not hidden by a later merge. Each selector is accepted once; repeated or empty values refuse before collection. - `scripts/count-unresolved-review-threads.sh` supplies the surveyor's unresolved-thread count. The guard accepts only `--repo` and a positive `--pr`, and only when the helper runs alone: its diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index 60b23c0b..cfd5e397 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -548,6 +548,8 @@ itself exits 2 on it, so every executable invocation must carry the resolved pat flags. **Do not reimplement the helper** inline. It owns the paginated API call in memory as well as classification, so a later-page API failure cannot be masked by a successful consumer of partial output and the read-only role never writes an intermediate file. +The helper also reads the branch's red scheduled runs of the last 31 days, whatever head they ran +on, so a failed scheduled workflow stays red after later merges until a later run of it succeeds. The bundled `forge-readonly-guard.sh` recognises the classifier only as this exact installed sibling; offline `--input` remains denied. Exit 0 is a complete classification; exit 2 means `unknown`, never green. diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index 5df3b2d7..5802c1b5 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 3209954d..2e7b994e 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -18,7 +18,7 @@ }, { "path": "scripts/classify-default-branch-ci-runs.sh", - "sha256": "9bb0ac5ae3705b7581e1da4a64d962e85a8e9de5a3c87cdc761d333625149de2", + "sha256": "7fea2c285ce47564b381007245f721fb5e46ec050eecb988bec8edffec700e8f", "executable": true }, { @@ -76,7 +76,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "e97a38eb100d87e852614cd87abf8eeee60501ee28b0eac323a538b8abaee99b" + "definitionSha256": "53936e0566f036cb7d59fb8f21f67efb29e70ac9201ce16ac06e5bd44bb5ccb2" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh index e6fe1546..03feb58c 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh @@ -5,8 +5,20 @@ # consumer and the read-only surveyor never writes a response file: # classify-default-branch-ci-runs.sh --repo OWNER/REPO --branch BRANCH --head-sha FULL_SHA # +# Remote mode also reads scheduled runs that are no longer on the current head. A scheduled +# workflow runs against whatever head the branch had at its cron time, so the next merge would +# otherwise hide its failure. Three complete reads list the branch's failed, timed-out and +# startup-failed scheduled runs of the last SCHEDULE_WINDOW_DAYS days; for each workflow found +# there, one first-page read fetches its newest successful scheduled run. Those runs join the +# current-head runs, so a scheduled workflow stays red until a later run of it succeeds. +# Cost per repository: the head pages + 3 + one per workflow with a red scheduled run. +# Blind spots: a scheduled workflow whose latest red run is older than the window, and +# non-push events other than schedule (workflow_dispatch, dynamic), which keep current-head +# semantics. +# # Fixture/offline mode accepts a flat run array, raw page-envelope stream, or --slurp envelope array: # classify-default-branch-ci-runs.sh --input PATH +# Offline mode classifies only the runs it is given; it reads no scheduled-run evidence. # # Stdout: one TSV line per current red identity — # workflow_id conclusion html_url name event path created_at run_id run_attempt @@ -63,6 +75,134 @@ observer_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) || exit 2 . "$observer_dir/json-stream.lib.sh" || exit 2 payload_path="" payload="" +schedule_runs='[]' +readonly SCHEDULE_WINDOW_DAYS=31 + +# shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq +jq_defs=' + def identifier: type == "number" and isfinite and . >= 1 + and . <= 9007199254740991 and floor == .; + + def valid_github_timestamp: + . as $timestamp + | type == "string" + and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") + and (try ((fromdateiso8601 | strftime("%Y-%m-%dT%H:%M:%SZ")) == $timestamp) + catch false); + + def valid_run_shape: + type == "object" + and (.workflow_id | identifier) + and (.id | identifier) + and (.run_attempt | identifier) + and (.created_at | valid_github_timestamp) + and ((.run_started_at // null) == null or (.run_started_at | valid_github_timestamp)); + + # Keep only the fields the classifier reads, so evidence stays small on argv. + def evidence_record: + {id, run_attempt, workflow_id, event, status, conclusion, created_at, + run_started_at, html_url, name, path, head_branch, head_sha}; + + # A complete paginated read: every page is an envelope, all pages agree on total_count, + # and together they hold exactly that many runs. A capped result never satisfies this. + def complete_envelope_runs: + if length == 0 then error("empty scheduled runs input") else . end + | [.[] | if type == "object" then . elif type == "array" then .[] + else error("invalid scheduled runs payload") end] + | if length == 0 then error("scheduled runs result requires page envelopes") + elif any(.[]; type != "object" or (has("workflow_runs") | not) + or (.workflow_runs | type != "array")) then + error("invalid scheduled runs page") + elif any(.[]; .total_count | type != "number" or . < 0 or floor != .) then + error("scheduled runs page is missing a valid total_count") + elif (map(.total_count) | unique | length) != 1 then + error("scheduled runs pages disagree on total_count") + elif ([.[].workflow_runs[]] | length) != .[0].total_count then + error("scheduled runs result is incomplete or capped") + else [.[].workflow_runs[]] end; +' + +# Input: every page of one red-conclusion scheduled read. Output: its validated runs. +# shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq +red_schedule_filter=' + complete_envelope_runs + | if any(.[]; (valid_run_shape | not) + or .event != "schedule" + or .head_branch != $expected_branch + or .status != "completed" + or .conclusion != $expected_conclusion + or ((.created_at | fromdateiso8601) < ($since | fromdateiso8601))) + then error("scheduled red run does not match the requested branch, conclusion or window") + elif (map([.id, .run_attempt]) | unique | length) != length then + error("duplicate scheduled run identity") + else map(evidence_record) end +' + +# Input: the first page of one workflow newest successful scheduled runs (one run at most). +# shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq +success_schedule_filter=' + if length != 1 or (.[0] | type != "object") + or (.[0] | has("workflow_runs") | not) or (.[0].workflow_runs | type != "array") + or (.[0].total_count | type != "number" or . < 0 or floor != .) + then error("invalid newest scheduled success page") + else .[0] end + | if (.workflow_runs | length) != ([.total_count, 1] | min) then + error("newest scheduled success page is incomplete") + else .workflow_runs end + | if any(.[]; (valid_run_shape | not) + or .workflow_id != $expected_workflow_id + or .event != "schedule" + or .head_branch != $expected_branch + or .status != "completed" + or .conclusion != "success") + then error("newest scheduled success does not match the requested workflow") + else map(evidence_record) end +' + +# Read one GitHub API result into memory, refusing undecodable or ambiguous bytes. +retained_read() { + local raw + raw=$(gh api "$@" | json_stream_retain_raw) || return 2 + printf '%s\n' "$raw" | json_stream_unique || return 2 + printf '%s\n' "$raw" +} + +collect_schedule_evidence() { + local since conclusion page found reds='[]' successes='[]' workflow_ids workflow_id + since=$(jq -nr --argjson days "$SCHEDULE_WINDOW_DAYS" 'now | floor - ($days * 86400) | todate') || + return 2 + for conclusion in failure timed_out startup_failure; do + page=$(retained_read --paginate --slurp --method GET "repos/${repo}/actions/runs" \ + -f branch="$branch" \ + -f event=schedule \ + -f status="$conclusion" \ + -f created=">=$since" \ + -F per_page=100) || return 2 + found=$(printf '%s\n' "$page" | jq -cs \ + --arg expected_branch "$branch" \ + --arg expected_conclusion "$conclusion" \ + --arg since "$since" \ + "$jq_defs $red_schedule_filter") || return 2 + reds=$(jq -cn --argjson a "$reds" --argjson b "$found" '$a + $b') || return 2 + done + workflow_ids=$(jq -rn --argjson runs "$reds" '$runs | map(.workflow_id) | unique | .[]') || + return 2 + while IFS= read -r workflow_id; do + [ -n "$workflow_id" ] || continue + [[ "$workflow_id" =~ ^[1-9][0-9]*$ ]] || return 2 + page=$(retained_read --method GET "repos/${repo}/actions/workflows/${workflow_id}/runs" \ + -f branch="$branch" \ + -f event=schedule \ + -f status=success \ + -F per_page=1) || return 2 + found=$(printf '%s\n' "$page" | jq -cs \ + --arg expected_branch "$branch" \ + --argjson expected_workflow_id "$workflow_id" \ + "$jq_defs $success_schedule_filter") || return 2 + successes=$(jq -cn --argjson a "$successes" --argjson b "$found" '$a + $b') || return 2 + done <<<"$workflow_ids" + jq -cn --argjson a "$reds" --argjson b "$successes" '$a + $b' +} if [ -n "$input_path" ]; then if [ -n "$repo" ] || [ -n "$branch" ] || [ -n "$head_sha" ]; then @@ -97,6 +237,10 @@ else echo "classify-default-branch-ci-runs: GitHub Actions pagination failed; health is unknown" >&2 exit 2 fi + if ! schedule_runs=$(collect_schedule_evidence); then + echo "classify-default-branch-ci-runs: scheduled-run evidence is incomplete or malformed; health is unknown" >&2 + exit 2 + fi fi # shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq @@ -151,9 +295,6 @@ jq_filter=' or .conclusion == "timed_out" or .conclusion == "startup_failure"; - def identifier: type == "number" and isfinite and . >= 1 - and . <= 9007199254740991 and floor == .; - def valid_outcome: has("conclusion") and (if .conclusion == null then @@ -167,13 +308,6 @@ jq_filter=' (if .conclusion == null then .status != "completed" else .status == "completed" end) else true end); - def valid_github_timestamp: - . as $timestamp - | type == "string" - and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") - and (try ((fromdateiso8601 | strftime("%Y-%m-%dT%H:%M:%SZ")) == $timestamp) - catch false); - if length == 0 then error("empty input") else . end | [.[] | page_envelopes[]] as $envelopes | [.[] | page_runs[]] as $runs @@ -228,6 +362,12 @@ jq_filter=' elif any(.[]; (valid_outcome | not)) then error("invalid workflow run outcome") else . end + # Scheduled runs off the current head join here. A run seen by both reads must agree. + | (. + $schedule_runs) + | group_by([.id, .run_attempt]) + | map(if (map([.status, .conclusion]) | unique | length) != 1 then + error("contradictory observations of one workflow run attempt") + else .[0] end) | group_by(run_identity) | map( sort_by([(.run_started_at // .created_at), .id, .run_attempt]) @@ -256,7 +396,8 @@ fi if ! classification=$(printf '%s\n' "$payload" | jq -rs \ --arg expected_head_sha "$head_sha" \ --arg expected_branch "$branch" \ - "$jq_filter"); then + --argjson schedule_runs "$schedule_runs" \ + "$jq_defs $jq_filter"); then echo "classify-default-branch-ci-runs: malformed or incomplete runs payload; health is unknown" >&2 exit 2 fi diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh index 43c2572d..787687e6 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh @@ -72,6 +72,7 @@ expect_remote_head_mismatch() { mkdir -p "$stub_dir" cat >"$stub_dir/gh" <<'STUB' #!/usr/bin/env bash +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"main","head_sha":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/stale-fail","name":"CI"}]}' STUB chmod +x "$stub_dir/gh" @@ -96,6 +97,7 @@ expect_remote_branch_mismatch() { mkdir -p "$stub_dir" cat >"$stub_dir/gh" <<'STUB' #!/usr/bin/env bash +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"release","head_sha":"0123456789abcdef0123456789abcdef01234567","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/wrong-branch-fail","name":"CI"}]}' STUB chmod +x "$stub_dir/gh" @@ -119,6 +121,7 @@ expect_remote_mixed_case_head() { mkdir -p "$stub_dir" cat >"$stub_dir/gh" <<'STUB' #!/usr/bin/env bash +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"main","head_sha":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/current-fail","name":"CI"}]}' STUB chmod +x "$stub_dir/gh" @@ -305,6 +308,161 @@ expect_remote_head_mismatch expect_remote_branch_mismatch expect_remote_mixed_case_head +# Scheduled runs off the current head (#669). A dispatching forge stub answers each read from +# the case directory; a read with no file finds nothing. Every call is logged for the cost test. +schedule_bin="$TEST_TMP/schedule-bin" +mkdir -p "$schedule_bin" +cat >"$schedule_bin/gh" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$SCHEDULE_CASE/calls" +for s in failure timed_out startup_failure; do + case "$*" in *status=$s\ *) [ ! -e "$SCHEDULE_CASE/fail-$s" ] || exit 1 ;; esac +done +answer() { + if [ -e "$SCHEDULE_CASE/$1" ]; then cat "$SCHEDULE_CASE/$1"; else printf '%s\n' "$2"; fi +} +case "$*" in + *head_sha=*) answer head.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *'actions/workflows/'*'/runs'*) + args="$*"; id=${args#*actions/workflows/}; id=${id%%/runs*} + answer "success-$id.json" '{"total_count":0,"workflow_runs":[]}' ;; + *status=failure*) answer failure.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *status=timed_out*) answer timed_out.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *status=startup_failure*) answer startup_failure.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *) exit 1 ;; +esac +STUB +chmod +x "$schedule_bin/gh" + +head_sha=0123456789abcdef0123456789abcdef01234567 +old_sha=abcdefabcdefabcdefabcdefabcdefabcdefabcd +days_ago() { jq -nr --argjson d "$1" 'now | floor - ($d * 86400) | todate'; } +t_old=$(days_ago 3) +t_new=$(days_ago 1) +t_ancient=$(days_ago 40) + +# One run record. Args: id workflow_id event conclusion created_at head_sha [name] +run_json() { + jq -cn --argjson id "$1" --argjson wf "$2" --arg event "$3" --arg conclusion "$4" \ + --arg created "$5" --arg sha "$6" --arg name "${7:-Nightly}" '{ + id: $id, run_attempt: 1, workflow_id: $wf, event: $event, status: "completed", + conclusion: $conclusion, created_at: $created, head_branch: "main", head_sha: $sha, + html_url: ("https://example.test/run-\($id)"), name: $name}' +} +# A slurped single-page envelope of the given runs. +pages() { jq -cs '[{total_count: length, workflow_runs: .}]'; } +page() { jq -cs '{total_count: length, workflow_runs: .}'; } + +new_case() { + SCHEDULE_CASE="$TEST_TMP/schedule-case-$1" + rm -rf "$SCHEDULE_CASE" + mkdir -p "$SCHEDULE_CASE" + export SCHEDULE_CASE +} + +run_schedule_case() { + schedule_out='' schedule_status=0 + schedule_out=$(PATH="$schedule_bin:$PATH" "$CLASSIFIER" --repo devantler-tech/example \ + --branch main --head-sha "$head_sha" 2>"$TEST_TMP/stderr") || schedule_status=$? +} + +expect_schedule() { + local label=$1 expected_status=$2 expected=$3 + run_schedule_case + if [ "$schedule_status" -eq "$expected_status" ] && [ "$schedule_out" = "$expected" ]; then + pass=$((pass + 1)) + else + record_failure "$label" + printf ' expected exit %s and output:\n%s\n got exit %s and output:\n%s\n' \ + "$expected_status" "$expected" "$schedule_status" "$schedule_out" >&2 + sed 's/^/ stderr: /' "$TEST_TMP/stderr" >&2 + fi +} + +nightly_red=$'21\tfailure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_old"$'\t500\t1' + +new_case advanced +run_json 600 31 push success "$t_new" "$head_sha" CI | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a scheduled failure on an older head stays red after main advances' 0 "$nightly_red" +if [ "$(grep -c . "$SCHEDULE_CASE/calls")" -eq 5 ] && + grep -Fq 'repos/devantler-tech/example/actions/workflows/21/runs -f branch=main -f event=schedule -f status=success -F per_page=1' "$SCHEDULE_CASE/calls" && + [ "$(grep -c -- '-f event=schedule -f status=[a-z_]* -f created=>=' "$SCHEDULE_CASE/calls")" -eq 3 ]; then + pass=$((pass + 1)) +else + record_failure 'scheduled evidence costs three window reads plus one read per red workflow' + sed 's/^/ call: /' "$SCHEDULE_CASE/calls" >&2 +fi + +new_case cleared +run_json 600 31 push success "$t_new" "$head_sha" CI | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +run_json 501 21 schedule success "$t_new" "$old_sha" | page > "$SCHEDULE_CASE/success-21.json" +expect_schedule 'a later successful run of the same scheduled workflow clears it' 0 '' + +new_case older-success +run_json 500 21 schedule failure "$t_new" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +run_json 499 21 schedule success "$t_old" "$old_sha" | page > "$SCHEDULE_CASE/success-21.json" +expect_schedule 'an older success does not clear a newer scheduled failure' 0 \ + $'21\tfailure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_new"$'\t500\t1' + +new_case head-success +run_json 600 21 workflow_dispatch success "$t_new" "$head_sha" | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a later success of the same workflow on the current head clears it' 0 '' + +new_case startup +run_json 500 21 schedule startup_failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/startup_failure.json" +expect_schedule 'a startup-failed scheduled run is red' 0 \ + $'21\tstartup_failure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_old"$'\t500\t1' + +new_case on-head +run_json 500 21 schedule failure "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a scheduled failure seen by both reads is reported once' 0 \ + $'21\tfailure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_old"$'\t500\t1' + +new_case push-unchanged +run_json 600 31 push failure "$t_old" "$old_sha" CI | pages > "$SCHEDULE_CASE/failure.json.unused" +run_json 601 31 push success "$t_new" "$head_sha" CI | pages > "$SCHEDULE_CASE/head.json" +expect_schedule 'a push failure on an older head keeps current-head semantics' 0 '' + +# Incomplete, mismatched or contradictory scheduled evidence makes health unknown. +new_case capped +run_json 500 21 schedule failure "$t_old" "$old_sha" | jq -cs '[{total_count: 1001, workflow_runs: .}]' > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a capped scheduled-failure read makes health unknown' 2 '' + +new_case pages-disagree +{ run_json 500 21 schedule failure "$t_old" "$old_sha"; run_json 501 22 schedule failure "$t_old" "$old_sha"; } | + jq -cs '[{total_count: 2, workflow_runs: [.[0]]}, {total_count: 3, workflow_runs: [.[1]]}]' > "$SCHEDULE_CASE/failure.json" +expect_schedule 'scheduled pages that disagree on total_count make health unknown' 2 '' + +for mutation in '.event="push"' '.head_branch="feature"' '.conclusion="success"' \ + '.status="in_progress" | .conclusion=null' ".created_at=\"$t_ancient\"" '.id=0' 'del(.run_attempt)'; do + new_case mutation + run_json 500 21 schedule failure "$t_old" "$old_sha" | jq -c "$mutation" | pages > "$SCHEDULE_CASE/failure.json" + expect_schedule "a scheduled red run that is not what was asked for makes health unknown: $mutation" 2 '' +done + +for mutation in '.workflow_runs[0].workflow_id=22' '.workflow_runs[0].event="push"' \ + '.workflow_runs[0].conclusion="failure"' '.workflow_runs=[]' '.total_count=-1' '[.]'; do + new_case success-mutation + run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" + run_json 501 21 schedule success "$t_new" "$old_sha" | page | jq -c "$mutation" > "$SCHEDULE_CASE/success-21.json" + expect_schedule "a newest-success page that does not answer the question makes health unknown: $mutation" 2 '' +done + +new_case contradiction +run_json 500 21 schedule success "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'two reads that disagree about one run attempt make health unknown' 2 '' + +for conclusion in failure timed_out startup_failure; do + new_case producer + : > "$SCHEDULE_CASE/fail-$conclusion" + expect_schedule "a failed scheduled $conclusion read makes health unknown" 2 '' +done + if grep -Fq '../scripts/classify-default-branch-ci-runs.sh' "$SURVEYOR" && grep -Fq 'Do not reimplement the helper' "$SURVEYOR" && grep -Fq 'exit 2 means' "$SURVEYOR" && @@ -454,6 +612,7 @@ printf '%s\n' "$base_run" | jq -c \ cat > "$stub_dir/gh" <<'STUB' #!/usr/bin/env bash [ "${GH_HOST:-}" = github.com ] && [ "${GH_TELEMETRY:-}" = 0 ] || exit 77 +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac cat "$STUB_CI_RESPONSE" STUB chmod +x "$stub_dir/gh" diff --git a/plugins/agentic-engineering/scripts/json-stream-performance.test.sh b/plugins/agentic-engineering/scripts/json-stream-performance.test.sh index 94b10126..75312fe3 100755 --- a/plugins/agentic-engineering/scripts/json-stream-performance.test.sh +++ b/plugins/agentic-engineering/scripts/json-stream-performance.test.sh @@ -30,6 +30,8 @@ done mkdir "$work/bin" cat > "$work/bin/gh" <<'STUB' #!/usr/bin/env bash +# The classifier's scheduled-run reads find nothing; only the head read carries the bytes. +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac cat "$FORGE_BYTES" exit "${FORGE_EXIT:-0}" STUB diff --git a/plugins/agentic-engineering/scripts/json-stream.test.sh b/plugins/agentic-engineering/scripts/json-stream.test.sh index 6fe8d47c..09622d14 100755 --- a/plugins/agentic-engineering/scripts/json-stream.test.sh +++ b/plugins/agentic-engineering/scripts/json-stream.test.sh @@ -21,6 +21,8 @@ done mkdir "$work/bin" cat > "$work/bin/gh" <<'STUB' #!/usr/bin/env bash +# The classifier's scheduled-run reads find nothing; only the head read carries the bytes. +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac cat "$FORGE_BYTES" exit "${FORGE_EXIT:-0}" STUB diff --git a/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh b/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh index 2e737ddd..cce4c63d 100755 --- a/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh +++ b/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh @@ -239,6 +239,13 @@ mkdir -p "$TMP/forge-bin" cat >"$TMP/forge-bin/gh" <<'EOF' #!/usr/bin/env bash set -euo pipefail +# The scheduled-run reads have their own fixed shapes and find nothing here. +case "$*" in + 'api --paginate --slurp --method GET repos/owner/repo/actions/runs -f branch=main -f event=schedule -f status='*' -f created=>='*' -F per_page=100') + printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]' + exit 0 + ;; +esac [ "$*" = 'api --paginate --slurp --method GET repos/owner/repo/actions/runs -f head_sha=1111111111111111111111111111111111111111 -f branch=main -F per_page=100' ] || exit 1 printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"main","head_sha":"1111111111111111111111111111111111111111","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/fail","name":"CI"}]}' EOF From bd9eaa8df39209aae3403350ddf0818899563285 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Wed, 11 Nov 2026 07:05:12 +0100 Subject: [PATCH 2/2] fix(agentic-engineering): pass scheduled evidence on stdin and align the surveyor contract Accumulated scheduled-run evidence now reaches jq on standard input and through --slurpfile from a process substitution, never argv, so a large result cannot hit the argument-size limit. A 1.6 MB fixture proves it. The surveyor's step 4 now names scheduled runs as the deliberate exception to the current-head rule, with their clearing rule and limits. Part of #669 Co-Authored-By: Claude Opus 5.5 --- .../agents/portfolio-surveyor.agent.md | 15 +++++++++++---- .../provider-neutral.desired-state.json | 4 ++-- .../scripts/classify-default-branch-ci-runs.sh | 16 +++++++++------- .../classify-default-branch-ci-runs.test.sh | 15 +++++++++++++++ 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index cfd5e397..d69e29de 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -529,8 +529,9 @@ and decides. ### 4. CI red on the default branch (bounded, per-repo) Judge the default branch by **its current head**, and only by runs that represent that branch's -health. Resolve the head first, using the **full-length sha** — a runs endpoint typically returns an -empty set for an abbreviated one, which reads exactly like "nothing failed". Then invoke the shipped +health. Scheduled runs are the one deliberate exception, handled inside the helper (below). +Resolve the head first, using the **full-length sha** — a runs endpoint typically returns an empty set +for an abbreviated one, which reads exactly like "nothing failed". Then invoke the shipped [`../scripts/classify-default-branch-ci-runs.sh`](../scripts/classify-default-branch-ci-runs.sh), using the literal absolute path supplied by preflight. If preflight supplied no path, submit **one** bare `classify-default-branch-ci-runs.sh` probe through the active read-only guard. The @@ -549,7 +550,11 @@ flags. **Do not reimplement the helper** inline. It owns the paginated API call in memory as well as classification, so a later-page API failure cannot be masked by a successful consumer of partial output and the read-only role never writes an intermediate file. The helper also reads the branch's red scheduled runs of the last 31 days, whatever head they ran -on, so a failed scheduled workflow stays red after later merges until a later run of it succeeds. +on, because a scheduled workflow runs against whatever head the branch had at its cron time. Such a +failure stays red after later merges until a later run of the same workflow succeeds: its next +successful scheduled run, or a successful branch-level run of it on the current head. A scheduled +failure older than 31 days is not seen, and `workflow_dispatch` and `dynamic` runs keep the +current-head rule. A scheduled row the helper returns is current breakage, not a stale run. The bundled `forge-readonly-guard.sh` recognises the classifier only as this exact installed sibling; offline `--input` remains denied. Exit 0 is a complete classification; exit 2 means `unknown`, never green. @@ -590,7 +595,9 @@ different run. Red conclusions are `failure`, `timed_out`, and `startup_failure` All four filters are load-bearing, each against a different false positive: - **Not keyed to head** — a failed run stays attached to the sha it ran against and lingers long - after the branch moved on, surfacing days-old failures as live breakage. + after the branch moved on, surfacing days-old failures as live breakage. Scheduled runs are the + exception: the helper keeps the latest result of each scheduled workflow, so a newer success + supersedes an old failure and an unanswered failure is not stale. - **Not a branch-level event** — a pull-request- or comment-triggered workflow can carry the same head sha and branch name while testing a PR. Do **not** instead de-duplicate check-runs by name: several independent comment-triggered runs coexist at one sha, so "newest per name" hides a genuine diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 2e7b994e..86ad3256 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -18,7 +18,7 @@ }, { "path": "scripts/classify-default-branch-ci-runs.sh", - "sha256": "7fea2c285ce47564b381007245f721fb5e46ec050eecb988bec8edffec700e8f", + "sha256": "0ad57f0653c45c12c8df82c1ab4e42c3787d90e656c3f0f497398b58f3dbd856", "executable": true }, { @@ -76,7 +76,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "53936e0566f036cb7d59fb8f21f67efb29e70ac9201ce16ac06e5bd44bb5ccb2" + "definitionSha256": "718e2058a2a547e108e807fabd53ccf7cf7ce7a21be37d7045ac4ba7c2d6f184" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh index 03feb58c..3bcc6524 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh @@ -168,7 +168,9 @@ retained_read() { } collect_schedule_evidence() { - local since conclusion page found reds='[]' successes='[]' workflow_ids workflow_id + local since conclusion page found reds='' successes='' workflow_ids workflow_id + # Evidence accumulates as one JSON array per line and reaches jq on stdin, never argv, + # so a large result cannot hit the argument-size limit. since=$(jq -nr --argjson days "$SCHEDULE_WINDOW_DAYS" 'now | floor - ($days * 86400) | todate') || return 2 for conclusion in failure timed_out startup_failure; do @@ -183,9 +185,9 @@ collect_schedule_evidence() { --arg expected_conclusion "$conclusion" \ --arg since "$since" \ "$jq_defs $red_schedule_filter") || return 2 - reds=$(jq -cn --argjson a "$reds" --argjson b "$found" '$a + $b') || return 2 + reds+="$found"$'\n' done - workflow_ids=$(jq -rn --argjson runs "$reds" '$runs | map(.workflow_id) | unique | .[]') || + workflow_ids=$(printf '%s' "$reds" | jq -rs 'add // [] | map(.workflow_id) | unique | .[]') || return 2 while IFS= read -r workflow_id; do [ -n "$workflow_id" ] || continue @@ -199,9 +201,9 @@ collect_schedule_evidence() { --arg expected_branch "$branch" \ --argjson expected_workflow_id "$workflow_id" \ "$jq_defs $success_schedule_filter") || return 2 - successes=$(jq -cn --argjson a "$successes" --argjson b "$found" '$a + $b') || return 2 + successes+="$found"$'\n' done <<<"$workflow_ids" - jq -cn --argjson a "$reds" --argjson b "$successes" '$a + $b' + printf '%s%s' "$reds" "$successes" | jq -cs 'add // []' } if [ -n "$input_path" ]; then @@ -363,7 +365,7 @@ jq_filter=' error("invalid workflow run outcome") else . end # Scheduled runs off the current head join here. A run seen by both reads must agree. - | (. + $schedule_runs) + | (. + $schedule_evidence[0]) | group_by([.id, .run_attempt]) | map(if (map([.status, .conclusion]) | unique | length) != 1 then error("contradictory observations of one workflow run attempt") @@ -396,7 +398,7 @@ fi if ! classification=$(printf '%s\n' "$payload" | jq -rs \ --arg expected_head_sha "$head_sha" \ --arg expected_branch "$branch" \ - --argjson schedule_runs "$schedule_runs" \ + --slurpfile schedule_evidence <(printf '%s\n' "$schedule_runs") \ "$jq_defs $jq_filter"); then echo "classify-default-branch-ci-runs: malformed or incomplete runs payload; health is unknown" >&2 exit 2 diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh index 787687e6..d34ce63a 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh @@ -463,6 +463,21 @@ for conclusion in failure timed_out startup_failure; do expect_schedule "a failed scheduled $conclusion read makes health unknown" 2 '' done +# Evidence larger than one argument (Linux caps a single argument at 128 KiB) must reach jq +# without argv, so a busy failing workflow cannot turn into unknown health. +new_case large +long_name=$(printf 'N%.0s' $(seq 1 900)) +for i in $(seq 1 1500); do + run_json $((1000 + i)) 21 schedule failure "$t_old" "$old_sha" "$long_name" +done | pages > "$SCHEDULE_CASE/failure.json" +run_schedule_case +if [ "$schedule_status" -eq 0 ] && [ "$(printf '%s\n' "$schedule_out" | cut -f1,2,8)" = $'21\tfailure\t2500' ]; then + pass=$((pass + 1)) +else + record_failure 'more than an argument of scheduled evidence is still classified' + sed 's/^/ stderr: /' "$TEST_TMP/stderr" >&2 +fi + if grep -Fq '../scripts/classify-default-branch-ci-runs.sh' "$SURVEYOR" && grep -Fq 'Do not reimplement the helper' "$SURVEYOR" && grep -Fq 'exit 2 means' "$SURVEYOR" &&