diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 351a332..26168df 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -20,7 +20,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 351a332..26168df 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -20,7 +20,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index 5df3b2d..5802c1b 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/.codex-plugin/plugin.json b/plugins/agentic-engineering/.codex-plugin/plugin.json index f605816..6c4ea78 100644 --- a/plugins/agentic-engineering/.codex-plugin/plugin.json +++ b/plugins/agentic-engineering/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/CHANGELOG.md b/plugins/agentic-engineering/CHANGELOG.md index e4ff8f0..4b5ff56 100644 --- a/plugins/agentic-engineering/CHANGELOG.md +++ b/plugins/agentic-engineering/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog — `agentic-engineering` +## 7.6.1 — 2026-11-11 + +**Fixed** — the default-branch CI classifier no longer loses a failed scheduled workflow when +`main` moves on. A scheduled run uses whatever head `main` had at its cron time, so the next merge +used to hide its failure. The classifier now also reads the branch's failed, timed-out and +startup-failed scheduled runs of the last 31 days on any head, plus each such workflow's newest +scheduled success, and keeps the workflow red until a later run of it succeeds. That costs three +reads per repository plus one per failing scheduled workflow, and an incomplete or capped answer +still means unknown. A scheduled workflow whose last failure is older than 31 days is not seen, and +manually dispatched or managed runs keep the current-head rule (#669). + ## 7.6.0 — 2026-11-11 **Added** — two first-party skills for the rules around a product, both written from the diff --git a/plugins/agentic-engineering/README.md b/plugins/agentic-engineering/README.md index a3d3982..24ba1f2 100644 --- a/plugins/agentic-engineering/README.md +++ b/plugins/agentic-engineering/README.md @@ -611,6 +611,8 @@ under the guard: `--repo`, `--branch`, and a full `--head-sha`, and refuses the helper's offline `--input` mode. The helper rejects unrecognized outcomes, contradictory status/conclusion pairs, repeated run identities and invalid numeric identifiers before selecting the latest workflow result. + It also reads the branch's red scheduled runs of the last 31 days on any head, and each such + workflow's newest scheduled success, so a scheduled failure is not hidden by a later merge. Each selector is accepted once; repeated or empty values refuse before collection. - `scripts/count-unresolved-review-threads.sh` supplies the surveyor's unresolved-thread count. The guard accepts only `--repo` and a positive `--pr`, and only when the helper runs alone: its diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index 60b23c0..d69e29d 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -529,8 +529,9 @@ and decides. ### 4. CI red on the default branch (bounded, per-repo) Judge the default branch by **its current head**, and only by runs that represent that branch's -health. Resolve the head first, using the **full-length sha** — a runs endpoint typically returns an -empty set for an abbreviated one, which reads exactly like "nothing failed". Then invoke the shipped +health. Scheduled runs are the one deliberate exception, handled inside the helper (below). +Resolve the head first, using the **full-length sha** — a runs endpoint typically returns an empty set +for an abbreviated one, which reads exactly like "nothing failed". Then invoke the shipped [`../scripts/classify-default-branch-ci-runs.sh`](../scripts/classify-default-branch-ci-runs.sh), using the literal absolute path supplied by preflight. If preflight supplied no path, submit **one** bare `classify-default-branch-ci-runs.sh` probe through the active read-only guard. The @@ -548,6 +549,12 @@ itself exits 2 on it, so every executable invocation must carry the resolved pat flags. **Do not reimplement the helper** inline. It owns the paginated API call in memory as well as classification, so a later-page API failure cannot be masked by a successful consumer of partial output and the read-only role never writes an intermediate file. +The helper also reads the branch's red scheduled runs of the last 31 days, whatever head they ran +on, because a scheduled workflow runs against whatever head the branch had at its cron time. Such a +failure stays red after later merges until a later run of the same workflow succeeds: its next +successful scheduled run, or a successful branch-level run of it on the current head. A scheduled +failure older than 31 days is not seen, and `workflow_dispatch` and `dynamic` runs keep the +current-head rule. A scheduled row the helper returns is current breakage, not a stale run. The bundled `forge-readonly-guard.sh` recognises the classifier only as this exact installed sibling; offline `--input` remains denied. Exit 0 is a complete classification; exit 2 means `unknown`, never green. @@ -588,7 +595,9 @@ different run. Red conclusions are `failure`, `timed_out`, and `startup_failure` All four filters are load-bearing, each against a different false positive: - **Not keyed to head** — a failed run stays attached to the sha it ran against and lingers long - after the branch moved on, surfacing days-old failures as live breakage. + after the branch moved on, surfacing days-old failures as live breakage. Scheduled runs are the + exception: the helper keeps the latest result of each scheduled workflow, so a newer success + supersedes an old failure and an unanswered failure is not stale. - **Not a branch-level event** — a pull-request- or comment-triggered workflow can carry the same head sha and branch name while testing a PR. Do **not** instead de-duplicate check-runs by name: several independent comment-triggered runs coexist at one sha, so "newest per name" hides a genuine diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index 5df3b2d..5802c1b 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios and everything its agents need — engineer, read-only surveyor, meta-engineer and Flux troubleshooter agents; portfolio, product, spend and improvement workflows; GitHub, CodeRabbit review, security, library documentation, Go, Kubernetes, Flux, Talos, SvelteKit frontend, Stripe payments, engineering-practice, design, product-management, marketing, finance and productivity skills; and the MCP servers they use; configured by the consumer AGENTS.md", - "version": "7.6.0", + "version": "7.6.1", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index 3209954..86ad325 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -18,7 +18,7 @@ }, { "path": "scripts/classify-default-branch-ci-runs.sh", - "sha256": "9bb0ac5ae3705b7581e1da4a64d962e85a8e9de5a3c87cdc761d333625149de2", + "sha256": "0ad57f0653c45c12c8df82c1ab4e42c3787d90e656c3f0f497398b58f3dbd856", "executable": true }, { @@ -76,7 +76,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "e97a38eb100d87e852614cd87abf8eeee60501ee28b0eac323a538b8abaee99b" + "definitionSha256": "718e2058a2a547e108e807fabd53ccf7cf7ce7a21be37d7045ac4ba7c2d6f184" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh index e6fe154..3bcc652 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.sh @@ -5,8 +5,20 @@ # consumer and the read-only surveyor never writes a response file: # classify-default-branch-ci-runs.sh --repo OWNER/REPO --branch BRANCH --head-sha FULL_SHA # +# Remote mode also reads scheduled runs that are no longer on the current head. A scheduled +# workflow runs against whatever head the branch had at its cron time, so the next merge would +# otherwise hide its failure. Three complete reads list the branch's failed, timed-out and +# startup-failed scheduled runs of the last SCHEDULE_WINDOW_DAYS days; for each workflow found +# there, one first-page read fetches its newest successful scheduled run. Those runs join the +# current-head runs, so a scheduled workflow stays red until a later run of it succeeds. +# Cost per repository: the head pages + 3 + one per workflow with a red scheduled run. +# Blind spots: a scheduled workflow whose latest red run is older than the window, and +# non-push events other than schedule (workflow_dispatch, dynamic), which keep current-head +# semantics. +# # Fixture/offline mode accepts a flat run array, raw page-envelope stream, or --slurp envelope array: # classify-default-branch-ci-runs.sh --input PATH +# Offline mode classifies only the runs it is given; it reads no scheduled-run evidence. # # Stdout: one TSV line per current red identity — # workflow_id conclusion html_url name event path created_at run_id run_attempt @@ -63,6 +75,136 @@ observer_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) || exit 2 . "$observer_dir/json-stream.lib.sh" || exit 2 payload_path="" payload="" +schedule_runs='[]' +readonly SCHEDULE_WINDOW_DAYS=31 + +# shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq +jq_defs=' + def identifier: type == "number" and isfinite and . >= 1 + and . <= 9007199254740991 and floor == .; + + def valid_github_timestamp: + . as $timestamp + | type == "string" + and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") + and (try ((fromdateiso8601 | strftime("%Y-%m-%dT%H:%M:%SZ")) == $timestamp) + catch false); + + def valid_run_shape: + type == "object" + and (.workflow_id | identifier) + and (.id | identifier) + and (.run_attempt | identifier) + and (.created_at | valid_github_timestamp) + and ((.run_started_at // null) == null or (.run_started_at | valid_github_timestamp)); + + # Keep only the fields the classifier reads, so evidence stays small on argv. + def evidence_record: + {id, run_attempt, workflow_id, event, status, conclusion, created_at, + run_started_at, html_url, name, path, head_branch, head_sha}; + + # A complete paginated read: every page is an envelope, all pages agree on total_count, + # and together they hold exactly that many runs. A capped result never satisfies this. + def complete_envelope_runs: + if length == 0 then error("empty scheduled runs input") else . end + | [.[] | if type == "object" then . elif type == "array" then .[] + else error("invalid scheduled runs payload") end] + | if length == 0 then error("scheduled runs result requires page envelopes") + elif any(.[]; type != "object" or (has("workflow_runs") | not) + or (.workflow_runs | type != "array")) then + error("invalid scheduled runs page") + elif any(.[]; .total_count | type != "number" or . < 0 or floor != .) then + error("scheduled runs page is missing a valid total_count") + elif (map(.total_count) | unique | length) != 1 then + error("scheduled runs pages disagree on total_count") + elif ([.[].workflow_runs[]] | length) != .[0].total_count then + error("scheduled runs result is incomplete or capped") + else [.[].workflow_runs[]] end; +' + +# Input: every page of one red-conclusion scheduled read. Output: its validated runs. +# shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq +red_schedule_filter=' + complete_envelope_runs + | if any(.[]; (valid_run_shape | not) + or .event != "schedule" + or .head_branch != $expected_branch + or .status != "completed" + or .conclusion != $expected_conclusion + or ((.created_at | fromdateiso8601) < ($since | fromdateiso8601))) + then error("scheduled red run does not match the requested branch, conclusion or window") + elif (map([.id, .run_attempt]) | unique | length) != length then + error("duplicate scheduled run identity") + else map(evidence_record) end +' + +# Input: the first page of one workflow newest successful scheduled runs (one run at most). +# shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq +success_schedule_filter=' + if length != 1 or (.[0] | type != "object") + or (.[0] | has("workflow_runs") | not) or (.[0].workflow_runs | type != "array") + or (.[0].total_count | type != "number" or . < 0 or floor != .) + then error("invalid newest scheduled success page") + else .[0] end + | if (.workflow_runs | length) != ([.total_count, 1] | min) then + error("newest scheduled success page is incomplete") + else .workflow_runs end + | if any(.[]; (valid_run_shape | not) + or .workflow_id != $expected_workflow_id + or .event != "schedule" + or .head_branch != $expected_branch + or .status != "completed" + or .conclusion != "success") + then error("newest scheduled success does not match the requested workflow") + else map(evidence_record) end +' + +# Read one GitHub API result into memory, refusing undecodable or ambiguous bytes. +retained_read() { + local raw + raw=$(gh api "$@" | json_stream_retain_raw) || return 2 + printf '%s\n' "$raw" | json_stream_unique || return 2 + printf '%s\n' "$raw" +} + +collect_schedule_evidence() { + local since conclusion page found reds='' successes='' workflow_ids workflow_id + # Evidence accumulates as one JSON array per line and reaches jq on stdin, never argv, + # so a large result cannot hit the argument-size limit. + since=$(jq -nr --argjson days "$SCHEDULE_WINDOW_DAYS" 'now | floor - ($days * 86400) | todate') || + return 2 + for conclusion in failure timed_out startup_failure; do + page=$(retained_read --paginate --slurp --method GET "repos/${repo}/actions/runs" \ + -f branch="$branch" \ + -f event=schedule \ + -f status="$conclusion" \ + -f created=">=$since" \ + -F per_page=100) || return 2 + found=$(printf '%s\n' "$page" | jq -cs \ + --arg expected_branch "$branch" \ + --arg expected_conclusion "$conclusion" \ + --arg since "$since" \ + "$jq_defs $red_schedule_filter") || return 2 + reds+="$found"$'\n' + done + workflow_ids=$(printf '%s' "$reds" | jq -rs 'add // [] | map(.workflow_id) | unique | .[]') || + return 2 + while IFS= read -r workflow_id; do + [ -n "$workflow_id" ] || continue + [[ "$workflow_id" =~ ^[1-9][0-9]*$ ]] || return 2 + page=$(retained_read --method GET "repos/${repo}/actions/workflows/${workflow_id}/runs" \ + -f branch="$branch" \ + -f event=schedule \ + -f status=success \ + -F per_page=1) || return 2 + found=$(printf '%s\n' "$page" | jq -cs \ + --arg expected_branch "$branch" \ + --argjson expected_workflow_id "$workflow_id" \ + "$jq_defs $success_schedule_filter") || return 2 + successes+="$found"$'\n' + done <<<"$workflow_ids" + printf '%s%s' "$reds" "$successes" | jq -cs 'add // []' +} if [ -n "$input_path" ]; then if [ -n "$repo" ] || [ -n "$branch" ] || [ -n "$head_sha" ]; then @@ -97,6 +239,10 @@ else echo "classify-default-branch-ci-runs: GitHub Actions pagination failed; health is unknown" >&2 exit 2 fi + if ! schedule_runs=$(collect_schedule_evidence); then + echo "classify-default-branch-ci-runs: scheduled-run evidence is incomplete or malformed; health is unknown" >&2 + exit 2 + fi fi # shellcheck disable=SC2016 # jq program; dollar-prefixed names belong to jq @@ -151,9 +297,6 @@ jq_filter=' or .conclusion == "timed_out" or .conclusion == "startup_failure"; - def identifier: type == "number" and isfinite and . >= 1 - and . <= 9007199254740991 and floor == .; - def valid_outcome: has("conclusion") and (if .conclusion == null then @@ -167,13 +310,6 @@ jq_filter=' (if .conclusion == null then .status != "completed" else .status == "completed" end) else true end); - def valid_github_timestamp: - . as $timestamp - | type == "string" - and test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$") - and (try ((fromdateiso8601 | strftime("%Y-%m-%dT%H:%M:%SZ")) == $timestamp) - catch false); - if length == 0 then error("empty input") else . end | [.[] | page_envelopes[]] as $envelopes | [.[] | page_runs[]] as $runs @@ -228,6 +364,12 @@ jq_filter=' elif any(.[]; (valid_outcome | not)) then error("invalid workflow run outcome") else . end + # Scheduled runs off the current head join here. A run seen by both reads must agree. + | (. + $schedule_evidence[0]) + | group_by([.id, .run_attempt]) + | map(if (map([.status, .conclusion]) | unique | length) != 1 then + error("contradictory observations of one workflow run attempt") + else .[0] end) | group_by(run_identity) | map( sort_by([(.run_started_at // .created_at), .id, .run_attempt]) @@ -256,7 +398,8 @@ fi if ! classification=$(printf '%s\n' "$payload" | jq -rs \ --arg expected_head_sha "$head_sha" \ --arg expected_branch "$branch" \ - "$jq_filter"); then + --slurpfile schedule_evidence <(printf '%s\n' "$schedule_runs") \ + "$jq_defs $jq_filter"); then echo "classify-default-branch-ci-runs: malformed or incomplete runs payload; health is unknown" >&2 exit 2 fi diff --git a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh index 43c2572..d34ce63 100755 --- a/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh +++ b/plugins/agentic-engineering/scripts/classify-default-branch-ci-runs.test.sh @@ -72,6 +72,7 @@ expect_remote_head_mismatch() { mkdir -p "$stub_dir" cat >"$stub_dir/gh" <<'STUB' #!/usr/bin/env bash +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"main","head_sha":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/stale-fail","name":"CI"}]}' STUB chmod +x "$stub_dir/gh" @@ -96,6 +97,7 @@ expect_remote_branch_mismatch() { mkdir -p "$stub_dir" cat >"$stub_dir/gh" <<'STUB' #!/usr/bin/env bash +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"release","head_sha":"0123456789abcdef0123456789abcdef01234567","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/wrong-branch-fail","name":"CI"}]}' STUB chmod +x "$stub_dir/gh" @@ -119,6 +121,7 @@ expect_remote_mixed_case_head() { mkdir -p "$stub_dir" cat >"$stub_dir/gh" <<'STUB' #!/usr/bin/env bash +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"main","head_sha":"abcdefabcdefabcdefabcdefabcdefabcdefabcd","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/current-fail","name":"CI"}]}' STUB chmod +x "$stub_dir/gh" @@ -305,6 +308,176 @@ expect_remote_head_mismatch expect_remote_branch_mismatch expect_remote_mixed_case_head +# Scheduled runs off the current head (#669). A dispatching forge stub answers each read from +# the case directory; a read with no file finds nothing. Every call is logged for the cost test. +schedule_bin="$TEST_TMP/schedule-bin" +mkdir -p "$schedule_bin" +cat >"$schedule_bin/gh" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$SCHEDULE_CASE/calls" +for s in failure timed_out startup_failure; do + case "$*" in *status=$s\ *) [ ! -e "$SCHEDULE_CASE/fail-$s" ] || exit 1 ;; esac +done +answer() { + if [ -e "$SCHEDULE_CASE/$1" ]; then cat "$SCHEDULE_CASE/$1"; else printf '%s\n' "$2"; fi +} +case "$*" in + *head_sha=*) answer head.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *'actions/workflows/'*'/runs'*) + args="$*"; id=${args#*actions/workflows/}; id=${id%%/runs*} + answer "success-$id.json" '{"total_count":0,"workflow_runs":[]}' ;; + *status=failure*) answer failure.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *status=timed_out*) answer timed_out.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *status=startup_failure*) answer startup_failure.json '[{"total_count":0,"workflow_runs":[]}]' ;; + *) exit 1 ;; +esac +STUB +chmod +x "$schedule_bin/gh" + +head_sha=0123456789abcdef0123456789abcdef01234567 +old_sha=abcdefabcdefabcdefabcdefabcdefabcdefabcd +days_ago() { jq -nr --argjson d "$1" 'now | floor - ($d * 86400) | todate'; } +t_old=$(days_ago 3) +t_new=$(days_ago 1) +t_ancient=$(days_ago 40) + +# One run record. Args: id workflow_id event conclusion created_at head_sha [name] +run_json() { + jq -cn --argjson id "$1" --argjson wf "$2" --arg event "$3" --arg conclusion "$4" \ + --arg created "$5" --arg sha "$6" --arg name "${7:-Nightly}" '{ + id: $id, run_attempt: 1, workflow_id: $wf, event: $event, status: "completed", + conclusion: $conclusion, created_at: $created, head_branch: "main", head_sha: $sha, + html_url: ("https://example.test/run-\($id)"), name: $name}' +} +# A slurped single-page envelope of the given runs. +pages() { jq -cs '[{total_count: length, workflow_runs: .}]'; } +page() { jq -cs '{total_count: length, workflow_runs: .}'; } + +new_case() { + SCHEDULE_CASE="$TEST_TMP/schedule-case-$1" + rm -rf "$SCHEDULE_CASE" + mkdir -p "$SCHEDULE_CASE" + export SCHEDULE_CASE +} + +run_schedule_case() { + schedule_out='' schedule_status=0 + schedule_out=$(PATH="$schedule_bin:$PATH" "$CLASSIFIER" --repo devantler-tech/example \ + --branch main --head-sha "$head_sha" 2>"$TEST_TMP/stderr") || schedule_status=$? +} + +expect_schedule() { + local label=$1 expected_status=$2 expected=$3 + run_schedule_case + if [ "$schedule_status" -eq "$expected_status" ] && [ "$schedule_out" = "$expected" ]; then + pass=$((pass + 1)) + else + record_failure "$label" + printf ' expected exit %s and output:\n%s\n got exit %s and output:\n%s\n' \ + "$expected_status" "$expected" "$schedule_status" "$schedule_out" >&2 + sed 's/^/ stderr: /' "$TEST_TMP/stderr" >&2 + fi +} + +nightly_red=$'21\tfailure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_old"$'\t500\t1' + +new_case advanced +run_json 600 31 push success "$t_new" "$head_sha" CI | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a scheduled failure on an older head stays red after main advances' 0 "$nightly_red" +if [ "$(grep -c . "$SCHEDULE_CASE/calls")" -eq 5 ] && + grep -Fq 'repos/devantler-tech/example/actions/workflows/21/runs -f branch=main -f event=schedule -f status=success -F per_page=1' "$SCHEDULE_CASE/calls" && + [ "$(grep -c -- '-f event=schedule -f status=[a-z_]* -f created=>=' "$SCHEDULE_CASE/calls")" -eq 3 ]; then + pass=$((pass + 1)) +else + record_failure 'scheduled evidence costs three window reads plus one read per red workflow' + sed 's/^/ call: /' "$SCHEDULE_CASE/calls" >&2 +fi + +new_case cleared +run_json 600 31 push success "$t_new" "$head_sha" CI | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +run_json 501 21 schedule success "$t_new" "$old_sha" | page > "$SCHEDULE_CASE/success-21.json" +expect_schedule 'a later successful run of the same scheduled workflow clears it' 0 '' + +new_case older-success +run_json 500 21 schedule failure "$t_new" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +run_json 499 21 schedule success "$t_old" "$old_sha" | page > "$SCHEDULE_CASE/success-21.json" +expect_schedule 'an older success does not clear a newer scheduled failure' 0 \ + $'21\tfailure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_new"$'\t500\t1' + +new_case head-success +run_json 600 21 workflow_dispatch success "$t_new" "$head_sha" | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a later success of the same workflow on the current head clears it' 0 '' + +new_case startup +run_json 500 21 schedule startup_failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/startup_failure.json" +expect_schedule 'a startup-failed scheduled run is red' 0 \ + $'21\tstartup_failure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_old"$'\t500\t1' + +new_case on-head +run_json 500 21 schedule failure "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a scheduled failure seen by both reads is reported once' 0 \ + $'21\tfailure\thttps://example.test/run-500\tNightly\tschedule\t\t'"$t_old"$'\t500\t1' + +new_case push-unchanged +run_json 600 31 push failure "$t_old" "$old_sha" CI | pages > "$SCHEDULE_CASE/failure.json.unused" +run_json 601 31 push success "$t_new" "$head_sha" CI | pages > "$SCHEDULE_CASE/head.json" +expect_schedule 'a push failure on an older head keeps current-head semantics' 0 '' + +# Incomplete, mismatched or contradictory scheduled evidence makes health unknown. +new_case capped +run_json 500 21 schedule failure "$t_old" "$old_sha" | jq -cs '[{total_count: 1001, workflow_runs: .}]' > "$SCHEDULE_CASE/failure.json" +expect_schedule 'a capped scheduled-failure read makes health unknown' 2 '' + +new_case pages-disagree +{ run_json 500 21 schedule failure "$t_old" "$old_sha"; run_json 501 22 schedule failure "$t_old" "$old_sha"; } | + jq -cs '[{total_count: 2, workflow_runs: [.[0]]}, {total_count: 3, workflow_runs: [.[1]]}]' > "$SCHEDULE_CASE/failure.json" +expect_schedule 'scheduled pages that disagree on total_count make health unknown' 2 '' + +for mutation in '.event="push"' '.head_branch="feature"' '.conclusion="success"' \ + '.status="in_progress" | .conclusion=null' ".created_at=\"$t_ancient\"" '.id=0' 'del(.run_attempt)'; do + new_case mutation + run_json 500 21 schedule failure "$t_old" "$old_sha" | jq -c "$mutation" | pages > "$SCHEDULE_CASE/failure.json" + expect_schedule "a scheduled red run that is not what was asked for makes health unknown: $mutation" 2 '' +done + +for mutation in '.workflow_runs[0].workflow_id=22' '.workflow_runs[0].event="push"' \ + '.workflow_runs[0].conclusion="failure"' '.workflow_runs=[]' '.total_count=-1' '[.]'; do + new_case success-mutation + run_json 500 21 schedule failure "$t_old" "$old_sha" | pages > "$SCHEDULE_CASE/failure.json" + run_json 501 21 schedule success "$t_new" "$old_sha" | page | jq -c "$mutation" > "$SCHEDULE_CASE/success-21.json" + expect_schedule "a newest-success page that does not answer the question makes health unknown: $mutation" 2 '' +done + +new_case contradiction +run_json 500 21 schedule success "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/head.json" +run_json 500 21 schedule failure "$t_old" "$head_sha" | pages > "$SCHEDULE_CASE/failure.json" +expect_schedule 'two reads that disagree about one run attempt make health unknown' 2 '' + +for conclusion in failure timed_out startup_failure; do + new_case producer + : > "$SCHEDULE_CASE/fail-$conclusion" + expect_schedule "a failed scheduled $conclusion read makes health unknown" 2 '' +done + +# Evidence larger than one argument (Linux caps a single argument at 128 KiB) must reach jq +# without argv, so a busy failing workflow cannot turn into unknown health. +new_case large +long_name=$(printf 'N%.0s' $(seq 1 900)) +for i in $(seq 1 1500); do + run_json $((1000 + i)) 21 schedule failure "$t_old" "$old_sha" "$long_name" +done | pages > "$SCHEDULE_CASE/failure.json" +run_schedule_case +if [ "$schedule_status" -eq 0 ] && [ "$(printf '%s\n' "$schedule_out" | cut -f1,2,8)" = $'21\tfailure\t2500' ]; then + pass=$((pass + 1)) +else + record_failure 'more than an argument of scheduled evidence is still classified' + sed 's/^/ stderr: /' "$TEST_TMP/stderr" >&2 +fi + if grep -Fq '../scripts/classify-default-branch-ci-runs.sh' "$SURVEYOR" && grep -Fq 'Do not reimplement the helper' "$SURVEYOR" && grep -Fq 'exit 2 means' "$SURVEYOR" && @@ -454,6 +627,7 @@ printf '%s\n' "$base_run" | jq -c \ cat > "$stub_dir/gh" <<'STUB' #!/usr/bin/env bash [ "${GH_HOST:-}" = github.com ] && [ "${GH_TELEMETRY:-}" = 0 ] || exit 77 +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac cat "$STUB_CI_RESPONSE" STUB chmod +x "$stub_dir/gh" diff --git a/plugins/agentic-engineering/scripts/json-stream-performance.test.sh b/plugins/agentic-engineering/scripts/json-stream-performance.test.sh index 94b1012..75312fe 100755 --- a/plugins/agentic-engineering/scripts/json-stream-performance.test.sh +++ b/plugins/agentic-engineering/scripts/json-stream-performance.test.sh @@ -30,6 +30,8 @@ done mkdir "$work/bin" cat > "$work/bin/gh" <<'STUB' #!/usr/bin/env bash +# The classifier's scheduled-run reads find nothing; only the head read carries the bytes. +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac cat "$FORGE_BYTES" exit "${FORGE_EXIT:-0}" STUB diff --git a/plugins/agentic-engineering/scripts/json-stream.test.sh b/plugins/agentic-engineering/scripts/json-stream.test.sh index 6fe8d47..09622d1 100755 --- a/plugins/agentic-engineering/scripts/json-stream.test.sh +++ b/plugins/agentic-engineering/scripts/json-stream.test.sh @@ -21,6 +21,8 @@ done mkdir "$work/bin" cat > "$work/bin/gh" <<'STUB' #!/usr/bin/env bash +# The classifier's scheduled-run reads find nothing; only the head read carries the bytes. +case "$*" in *event=schedule*) printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]'; exit 0 ;; esac cat "$FORGE_BYTES" exit "${FORGE_EXIT:-0}" STUB diff --git a/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh b/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh index 2e737dd..cce4c63 100755 --- a/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh +++ b/plugins/agentic-engineering/scripts/surveyor-forge-readonly.test.sh @@ -239,6 +239,13 @@ mkdir -p "$TMP/forge-bin" cat >"$TMP/forge-bin/gh" <<'EOF' #!/usr/bin/env bash set -euo pipefail +# The scheduled-run reads have their own fixed shapes and find nothing here. +case "$*" in + 'api --paginate --slurp --method GET repos/owner/repo/actions/runs -f branch=main -f event=schedule -f status='*' -f created=>='*' -F per_page=100') + printf '%s\n' '[{"total_count":0,"workflow_runs":[]}]' + exit 0 + ;; +esac [ "$*" = 'api --paginate --slurp --method GET repos/owner/repo/actions/runs -f head_sha=1111111111111111111111111111111111111111 -f branch=main -F per_page=100' ] || exit 1 printf '%s\n' '{"total_count":1,"workflow_runs":[{"id":10,"run_attempt":1,"workflow_id":11,"event":"push","head_branch":"main","head_sha":"1111111111111111111111111111111111111111","conclusion":"failure","created_at":"2026-07-14T09:00:00Z","html_url":"https://example.test/fail","name":"CI"}]}' EOF