diff --git a/.github/agent-guides/workflows.md b/.github/agent-guides/workflows.md index 41db87a1..06fc2c77 100644 --- a/.github/agent-guides/workflows.md +++ b/.github/agent-guides/workflows.md @@ -262,7 +262,19 @@ rename destinations keep full coverage. The authenticated caller's original event/author admission remains outside mandatory execution selection. Shared helpers, other workflows, tests, fixtures, root configuration and unknown paths retain full coverage; missing Git -evidence fails the selector. Classification runs from the immutable trusted base +evidence fails the selector. Root documents are the only exception and need ordinary +regular-file modifications: the root `README.md` selects exactly the gated jobs whose +executed scripts read it, and the always-on documentation gates still run. Root +`AGENTS.md` has no gated reader, so it adds nothing beside a recognized non-Markdown +change; alone, with the README or with only Markdown it keeps full coverage. There is +no general Markdown exemption. `ci-selection-document-readers.sh` (run by +`test-ci-selection.sh`) derives the reader set from the workflow and every script, +action and Go module a gated job executes, and fails when the inventory's `README.md` +owners differ from it, when a gated job reads `AGENTS.md`, or when one copies or walks +the whole checkout. A new README-consuming check joins routing by adding `README.md` +to its inventory owners and its job id to the selector's reviewed reader list; a line +that only mentions a document (fixture data, a message or a comment) joins that +script's reviewed non-reading list with its exact text. Classification runs from the immutable trusted base before candidate checkout and selection on the same read-only runner; there is no separate prerequisite runner assignment. A successful trusted-base scope decision for a deployment-only PR omits only the explicitly inventoried smoke jobs; the diff --git a/.github/scripts/ci-selection/dotnet_workflow_test.go b/.github/scripts/ci-selection/dotnet_workflow_test.go index 23dde99a..f63a9390 100644 --- a/.github/scripts/ci-selection/dotnet_workflow_test.go +++ b/.github/scripts/ci-selection/dotnet_workflow_test.go @@ -37,6 +37,10 @@ func dotnetSelectionFixture(t *testing.T) (inventory, workflow) { w.Jobs["test-go-cache-native-observer"] = job{If: selectionGuard("test-go-cache-native-observer"), Needs: []any{"select-ci-tests", "test-validate-go-postgresql-fixture"}} i.Jobs["test-go-workflow-guards"] = []string{"actions/upload-coverage/"} w.Jobs["test-go-workflow-guards"] = job{If: selectionGuard("test-go-workflow-guards"), Needs: []any{"select-ci-tests"}} + for id := range readmeReaders { + i.Jobs[id] = []string{rootReadme} + w.Jobs[id] = job{If: selectionGuard(id), Needs: []any{"select-ci-tests"}} + } return i, w } diff --git a/.github/scripts/ci-selection/inventory.json b/.github/scripts/ci-selection/inventory.json index 5a6890f5..08badffe 100644 --- a/.github/scripts/ci-selection/inventory.json +++ b/.github/scripts/ci-selection/inventory.json @@ -109,8 +109,8 @@ "test-enable-auto-merge": [], "test-enable-auto-merge-actor-trust": [], "test-enable-auto-merge-queued": [], - "test-enable-auto-merge-author-gate": [], - "test-enable-auto-merge-pentad-gate": [], + "test-enable-auto-merge-author-gate": ["README.md"], + "test-enable-auto-merge-pentad-gate": ["README.md"], "test-enable-auto-merge-queue": [], "test-create-release": [], "test-create-release-no-issue-side-effects": [], @@ -141,10 +141,10 @@ "test-validate-go-lint-blocks": ["actions/upload-coverage/"], "test-validate-go-test-blocks": ["actions/upload-coverage/"], "test-validate-go-project-concurrency-key": ["actions/upload-coverage/"], - "test-validate-go-gate-lockstep": ["actions/upload-coverage/", "actions/dependency-review/"], + "test-validate-go-gate-lockstep": ["actions/upload-coverage/", "actions/dependency-review/", "README.md"], "test-lint-blocks": [], "test-prepare-fixes": [], - "test-lint-gate-lockstep": ["actions/upload-coverage/"], + "test-lint-gate-lockstep": ["actions/upload-coverage/", "README.md"], "test-apply-signed-fixes-suppresses-bots": [], "test-apply-signed-fixes-verifies-without-a-patch": [], "test-check-ci-run-budget": ["actions/check-ci-run-budget/"] diff --git a/.github/scripts/ci-selection/main.go b/.github/scripts/ci-selection/main.go index df57b8f2..e512f4ad 100644 --- a/.github/scripts/ci-selection/main.go +++ b/.github/scripts/ci-selection/main.go @@ -23,8 +23,25 @@ const goCacheObserver = "test-go-cache-native-observer" const goCacheFixture = "test-validate-go-postgresql-fixture" const goWorkflowGuards = "test-go-workflow-guards" +// Root documents admitted only as ordinary regular-file modifications. +const rootReadme = "README.md" +const rootAgents = "AGENTS.md" + var ordinarySelectiveFiles = map[string]bool{ dotnetFixtureAction: true, + rootReadme: true, + rootAgents: true, +} + +// Every gated job whose executed tests read the root README. The tracked-reader +// guard (.github/tests/ci-selection-document-readers.sh) derives this set from +// the workflow and its scripts; the inventory must own README.md for exactly +// these jobs. Always-on and preserved jobs read it without needing selection. +var readmeReaders = map[string]bool{ + "test-enable-auto-merge-author-gate": true, + "test-enable-auto-merge-pentad-gate": true, + "test-lint-gate-lockstep": true, + "test-validate-go-gate-lockstep": true, } // Native fixture jobs and their hosted/credential controls share this local action. @@ -64,6 +81,9 @@ func validateInventory(i inventory, w workflow) ([]string, error) { if err := validateGoWorkflowGuards(i, w); err != nil { return nil, err } + if err := validateReadmeReaders(i, w); err != nil { + return nil, err + } seen := map[string]bool{"select-ci-tests": true, "ci-required-checks": true} for _, id := range i.Always { if seen[id] { @@ -87,7 +107,8 @@ func validateInventory(i inventory, w workflow) ([]string, error) { seen[id] = true for _, p := range paths { if !regexp.MustCompile(`^actions/[a-z0-9-]+/$`).MatchString(p) && - (p != dotnetFixtureAction || !hasDotnetOwner(id)) { + (p != dotnetFixtureAction || !hasDotnetOwner(id)) && + (p != rootReadme || !readmeReaders[id]) { return nil, fmt.Errorf("invalid selective owner %q", p) } } @@ -171,6 +192,24 @@ func validateGoCacheOwners(i inventory, w workflow) error { return nil } +// Each README reader must own the file exactly once and stay selection-gated, so +// a README change cannot pass without the tests that read it. +func validateReadmeReaders(i inventory, w workflow) error { + for id := range readmeReaders { + count := 0 + for _, p := range i.Jobs[id] { + if p == rootReadme { + count++ + } + } + _, preserved := i.Preserved[id] + if _, ok := w.Jobs[id]; !ok || count != 1 || preserved { + return fmt.Errorf("incomplete or unbound README reader %s", id) + } + } + return nil +} + func hasDotnetOwner(id string) bool { _, ok := dotnetOwners[id] return ok @@ -207,7 +246,14 @@ func selectJobs(i inventory, event string, paths []string) []string { return all } selected := map[string]bool{} + // Root AGENTS.md has no gated reader. It may only accompany a recognized + // non-documentation change; alone or with other documents it stays full. + companion, source := false, false for _, path := range paths { + if path == rootAgents { + companion = true + continue + } known := false for id, owners := range i.Jobs { for _, owner := range owners { @@ -219,6 +265,10 @@ func selectJobs(i inventory, event string, paths []string) []string { if !known { return all } + source = source || !isDocumentation(path) + } + if companion && !source { + return all } result := []string{} for id := range selected { @@ -228,6 +278,11 @@ func selectJobs(i inventory, event string, paths []string) []string { return result } +func isDocumentation(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + return ext == ".md" || ext == ".markdown" || ext == ".mdx" +} + func changedPaths(root, base, head string) ([]string, error) { sha := regexp.MustCompile(`^[a-f0-9]{40}$`) if !sha.MatchString(base) || !sha.MatchString(head) { diff --git a/.github/scripts/ci-selection/main_test.go b/.github/scripts/ci-selection/main_test.go index 06c11280..04f27854 100644 --- a/.github/scripts/ci-selection/main_test.go +++ b/.github/scripts/ci-selection/main_test.go @@ -302,7 +302,7 @@ func TestEntrypointBindsGitEvidenceAndSelectedOutput(t *testing.T) { fails bool }{ {"selective PR", "pull_request", "true", head, "selected=[\"test-one\",\"test-one-wrapper\"]\n", false}, - {"main retains complete inventory", "push", "true", "", "selected=[\"test-go-cache-native-observer\",\"test-go-workflow-guards\",\"test-one\",\"test-one-wrapper\",\"test-run-dotnet-tests-gate-lockstep\",\"test-run-dotnet-tests-mtp\",\"test-run-dotnet-tests-workflow\",\"test-two\",\"test-validate-go-postgresql-fixture\",\"test-workflow\"]\n", false}, + {"main retains complete inventory", "push", "true", "", "selected=[\"test-enable-auto-merge-author-gate\",\"test-enable-auto-merge-pentad-gate\",\"test-go-cache-native-observer\",\"test-go-workflow-guards\",\"test-lint-gate-lockstep\",\"test-one\",\"test-one-wrapper\",\"test-run-dotnet-tests-gate-lockstep\",\"test-run-dotnet-tests-mtp\",\"test-run-dotnet-tests-workflow\",\"test-two\",\"test-validate-go-gate-lockstep\",\"test-validate-go-postgresql-fixture\",\"test-workflow\"]\n", false}, {"excluded event", "merge_group", "false", "", "selected=[]\n", false}, {"missing immutable head", "pull_request", "true", strings.Repeat("a", 40), "", true}, {"unknown eligibility", "push", "", "", "", true}, @@ -370,8 +370,8 @@ func TestDeploymentCatalogueScopeComposesWithSelection(t *testing.T) { name, event, eligible, scope, want string fails bool }{ - {"deployment smoke omission only", "pull_request", "true", "false", "selected=[\"test-go-cache-native-observer\",\"test-go-workflow-guards\",\"test-one-wrapper\",\"test-run-dotnet-tests-gate-lockstep\",\"test-run-dotnet-tests-mtp\",\"test-run-dotnet-tests-workflow\",\"test-validate-go-postgresql-fixture\",\"test-workflow\"]\n", false}, - {"full scope", "pull_request", "true", "true", "selected=[\"test-go-cache-native-observer\",\"test-go-workflow-guards\",\"test-one\",\"test-one-wrapper\",\"test-run-dotnet-tests-gate-lockstep\",\"test-run-dotnet-tests-mtp\",\"test-run-dotnet-tests-workflow\",\"test-two\",\"test-validate-go-postgresql-fixture\",\"test-workflow\"]\n", false}, + {"deployment smoke omission only", "pull_request", "true", "false", "selected=[\"test-enable-auto-merge-author-gate\",\"test-enable-auto-merge-pentad-gate\",\"test-go-cache-native-observer\",\"test-go-workflow-guards\",\"test-lint-gate-lockstep\",\"test-one-wrapper\",\"test-run-dotnet-tests-gate-lockstep\",\"test-run-dotnet-tests-mtp\",\"test-run-dotnet-tests-workflow\",\"test-validate-go-gate-lockstep\",\"test-validate-go-postgresql-fixture\",\"test-workflow\"]\n", false}, + {"full scope", "pull_request", "true", "true", "selected=[\"test-enable-auto-merge-author-gate\",\"test-enable-auto-merge-pentad-gate\",\"test-go-cache-native-observer\",\"test-go-workflow-guards\",\"test-lint-gate-lockstep\",\"test-one\",\"test-one-wrapper\",\"test-run-dotnet-tests-gate-lockstep\",\"test-run-dotnet-tests-mtp\",\"test-run-dotnet-tests-workflow\",\"test-two\",\"test-validate-go-gate-lockstep\",\"test-validate-go-postgresql-fixture\",\"test-workflow\"]\n", false}, {"missing scope", "pull_request", "true", "", "", true}, {"malformed scope", "pull_request", "true", "unknown", "", true}, {"push cannot omit smoke", "push", "true", "false", "", true}, diff --git a/.github/scripts/ci-selection/root_documents_test.go b/.github/scripts/ci-selection/root_documents_test.go new file mode 100644 index 00000000..301ac527 --- /dev/null +++ b/.github/scripts/ci-selection/root_documents_test.go @@ -0,0 +1,221 @@ +package main + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "testing" +) + +func sortedReaders() []string { + readers := []string{} + for id := range readmeReaders { + readers = append(readers, id) + } + sort.Strings(readers) + return readers +} + +func allJobs(i inventory) []string { + all := []string{} + for id := range i.Jobs { + all = append(all, id) + } + sort.Strings(all) + return all +} + +func union(lists ...[]string) []string { + seen := map[string]bool{} + out := []string{} + for _, list := range lists { + for _, id := range list { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + } + sort.Strings(out) + return out +} + +// The root README selects exactly its gated readers. Root AGENTS.md only rides +// along with a recognized non-documentation change; everything else stays full. +func TestRootDocumentRouting(t *testing.T) { + i, _ := dotnetSelectionFixture(t) + all := allJobs(i) + one := []string{"test-one", "test-one-wrapper"} + readers := sortedReaders() + for _, tc := range []struct { + name string + paths []string + want []string + }{ + {"README only", []string{"README.md"}, readers}, + {"AGENTS only", []string{"AGENTS.md"}, all}, + {"README and AGENTS", []string{"README.md", "AGENTS.md"}, all}, + {"AGENTS with action README only", []string{"AGENTS.md", "actions/one/README.md"}, all}, + {"AGENTS with uppercase Markdown only", []string{"AGENTS.md", "actions/one/NOTES.MD"}, all}, + {"AGENTS with action source", []string{"actions/one/action.yaml", "AGENTS.md"}, one}, + {"README with action source", []string{"actions/one/action.yaml", "README.md"}, union(one, readers)}, + {"README AGENTS and action source", []string{"AGENTS.md", "README.md", "actions/one/main.go"}, union(one, readers)}, + {"README with unknown", []string{"README.md", "unclassified"}, all}, + {"AGENTS with source and unknown", []string{"actions/one/action.yaml", "AGENTS.md", "docs/guide.md"}, all}, + {"AGENTS with shared helper", []string{".scripts/retry.sh", "AGENTS.md"}, all}, + {"AGENTS with workflow", []string{".github/workflows/ci.yaml", "AGENTS.md"}, all}, + {"nested README", []string{"docs/README.md"}, all}, + {"lowercase readme", []string{"readme.md"}, all}, + {"nested AGENTS", []string{"actions/one/action.yaml", "actions/AGENTS.md"}, all}, + {"CLAUDE shim", []string{"actions/one/action.yaml", "CLAUDE.md"}, all}, + {"unproven marker", []string{"README.md", ".ci-selection-unproven-workflow-change"}, all}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := selectJobs(i, "pull_request", tc.paths); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %v; want %v", got, tc.want) + } + }) + } + for _, event := range []string{"push", "merge_group"} { + if got := selectJobs(i, event, []string{"README.md"}); !reflect.DeepEqual(got, all) { + t.Fatalf("%s narrowed a README change: %v", event, got) + } + } +} + +// The reviewed production inventory binds README to exactly its readers. +func TestProductionInventoryReadmeRoute(t *testing.T) { + var i inventory + if err := load("inventory.json", &i); err != nil { + t.Fatal(err) + } + owners := []string{} + for id, paths := range i.Jobs { + for _, p := range paths { + if p == rootReadme { + owners = append(owners, id) + } + if p == rootAgents { + t.Fatalf("AGENTS.md must have no gated owner, found %s", id) + } + } + } + sort.Strings(owners) + if !reflect.DeepEqual(owners, sortedReaders()) { + t.Fatalf("inventory README owners %v; reviewed readers %v", owners, sortedReaders()) + } + if got := selectJobs(i, "pull_request", []string{"README.md"}); !reflect.DeepEqual(got, sortedReaders()) { + t.Fatalf("README-only selection %v", got) + } + if got := selectJobs(i, "pull_request", []string{"AGENTS.md"}); len(got) != len(i.Jobs) { + t.Fatalf("AGENTS-only selection narrowed: %v", got) + } + action := selectJobs(i, "pull_request", []string{"actions/approve-pr/action.yaml"}) + if got := selectJobs(i, "pull_request", []string{"actions/approve-pr/action.yaml", "AGENTS.md"}); !reflect.DeepEqual(got, action) { + t.Fatalf("AGENTS companion changed an action selection: %v vs %v", got, action) + } +} + +func TestReadmeReaderOwnershipFailsClosed(t *testing.T) { + reader := sortedReaders()[0] + for _, mutation := range []string{"complete", "missing owner", "duplicate owner", "preserved reader", "missing job", "invented reader", "AGENTS owner"} { + t.Run(mutation, func(t *testing.T) { + i, w := dotnetSelectionFixture(t) + switch mutation { + case "missing owner": + i.Jobs[reader] = []string{} + case "duplicate owner": + i.Jobs[reader] = []string{rootReadme, rootReadme} + case "preserved reader": + i.Preserved[reader] = w.Jobs[reader] + case "missing job": + delete(i.Jobs, reader) + delete(w.Jobs, reader) + case "invented reader": + i.Jobs["test-one"] = append(i.Jobs["test-one"], rootReadme) + case "AGENTS owner": + i.Jobs["test-one"] = append(i.Jobs["test-one"], rootAgents) + } + _, err := validateInventory(i, w) + if mutation == "complete" { + if err != nil { + t.Fatalf("complete README readers rejected: %v", err) + } + } else if err == nil { + t.Fatalf("accepted %s", mutation) + } + }) + } +} + +// Additions, deletions, renames, symlinks and mode changes of a root document +// are not ordinary modifications and must keep full coverage. +func TestRootDocumentGitStatusRetainsFullFallback(t *testing.T) { + for _, doc := range []string{rootReadme, rootAgents} { + for _, kind := range []string{"modified", "added", "deleted", "symlink", "renamed", "executable"} { + t.Run(doc+" "+kind, func(t *testing.T) { + dir, _, git := gitFixture(t) + p := filepath.Join(dir, doc) + if kind != "added" { + if err := os.WriteFile(p, []byte("base\n"), 0600); err != nil { + t.Fatal(err) + } + git("add", "--", doc) + git("commit", "-qm", "document baseline") + } + base := git("rev-parse", "HEAD") + switch kind { + case "modified", "added": + if err := os.WriteFile(p, []byte("changed\n"), 0600); err != nil { + t.Fatal(err) + } + case "deleted": + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + case "symlink": + if err := os.Remove(p); err != nil { + t.Fatal(err) + } + if err := os.Symlink("actions/one/action.yaml", p); err != nil { + t.Fatal(err) + } + case "renamed": + if err := os.Rename(p, filepath.Join(dir, "actions/one/"+doc)); err != nil { + t.Fatal(err) + } + case "executable": + if err := os.Chmod(p, 0700); err != nil { + t.Fatal(err) + } + } + // AGENTS.md can only narrow beside a recognized source change. + if doc == rootAgents { + if err := os.WriteFile(filepath.Join(dir, "actions/one/action.yaml"), []byte("source\n"), 0600); err != nil { + t.Fatal(err) + } + } + git("add", "-A", "--", ".") + git("commit", "-qm", "document change") + paths, err := changedPaths(dir, base, git("rev-parse", "HEAD")) + if err != nil { + t.Fatal(err) + } + i, _ := dotnetSelectionFixture(t) + got := selectJobs(i, "pull_request", paths) + want := sortedReaders() + if doc == rootAgents { + want = []string{"test-one", "test-one-wrapper"} + } + if kind == "modified" { + if !reflect.DeepEqual(got, want) { + t.Fatalf("ordinary modification selected %v; want %v", got, want) + } + } else if len(got) != len(i.Jobs) { + t.Fatalf("%s incorrectly narrowed coverage: %v", kind, got) + } + }) + } + } +} diff --git a/.github/tests/ci-selection-document-readers.sh b/.github/tests/ci-selection-document-readers.sh new file mode 100644 index 00000000..2e3e6793 --- /dev/null +++ b/.github/tests/ci-selection-document-readers.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Derive which selection-gated CI jobs read the root README.md or AGENTS.md +# from the tracked workflow and the code those jobs execute, then require the +# selector inventory to route README.md to exactly that set and AGENTS.md to +# none. A new reader, an unreviewed mention or a whole-checkout read fails. +# +# Usage: ci-selection-document-readers.sh ROOT INVENTORY_JSON WORKFLOW_JSON +# +# A job's executed code is its inline run/with text, its reusable workflow and +# every tracked shell, jq or Go file and local action or Go module it names, +# followed transitively. Data files are leaves. Every line of that code that +# names README or AGENTS.md, or copies, archives or walks the whole checkout, +# either makes the job a reader or appears with its exact trimmed text in the +# reviewed non-reading list below. AGENTS.md and whole-checkout reads have no +# reviewed reader and always fail. +set -euo pipefail +(($# == 3)) || { + echo 'usage: ci-selection-document-readers.sh ROOT INVENTORY_JSON WORKFLOW_JSON' >&2 + exit 2 +} +root="$1" inventory="$2" workflow="$3" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Reviewed mentions that do not read the repository's root documents: +# fixture data, messages and comments. Format: |. +cat >"$work/non-reading" <<'LINES' +.github/scripts/inspect-release-recovery/internal/inspection/inspection_test.go|entry["path"] = "README.md" +.github/tests/test-govulncheck-main-coverage.sh|fail "govulncheck's gate has no default-branch clause, so the path filter is the only gate on the default branch and a non-Go push (e.g. a README edit) leaves it unscanned while reporting green. Add: || github.ref == format('refs/heads/{0}', github.event.repository.default_branch) — see ksail#6373." +.github/workflows/validate-go-project-readonly.yaml|# alone means a push touching no Go path (a README edit) skips the scan, so the branch +actions/validate-retired-repo-links/main_test.go|{"retired link blocks", basic, "See https://github.com/example/retired/blob/main/README.md", 1, "docs/guide.md:1"}, +actions/validate-shell-pipelines/cli_test.go|"README.md": "producer | grep -q not-code\n", +actions/validate-shell-pipelines/cli_test.go|for _, scope := range []string{"", "missing", "../escape", "/tmp", ":(glob)*", "*.sh", "safe.sh\nmissing", "README.md", "./safe.sh"} { +actions/validate-shell-pipelines/cli_test.go|root := fixture(t, map[string]string{"safe.sh": "true\n", "README.md": "text\n"}) +actions/validate-shell-pipelines/cli_test.go|root := fixture(t, map[string]string{"README.md": "text"}) +.github/tests/test-go-filter-nested-modules-blocks.sh|# test (AGENTS.md, "Failure-mode coverage for gating workflows"). +.github/tests/test-govulncheck-main-coverage-blocks.sh|# test (AGENTS.md, "Failure-mode coverage for gating workflows"). +.github/tests/test-suite-default-branch-coverage-blocks.sh|# blocks-on-bad-input test (AGENTS.md, "Failure-mode coverage for gating workflows"). +.github/tests/test-suite-default-branch-coverage.sh|# the opt-in input `test-default-branch` (AGENTS.md, *Shipping a new capability behind an +actions/run-dotnet-tests/action.yaml|config_path="$(find . -maxdepth 1 \( -type f -o -type l \) -iname 'nuget.config' -print -quit)" +.github/tests/test-govulncheck-main-coverage.sh|fail "the govulncheck job passes no 'allow-file' input, so a committed .govulncheck-allow.txt is ignored and every risk-accepted advisory blocks the scan — see AGENTS.md, *Tested invariants*." +.github/tests/test-suite-default-branch-coverage.sh|fail "the test job's gate does not reference '$flag_ref', so the default-branch run is not behind the opt-in input it is required to ship behind — a behaviour change landing on every consumer at once. See AGENTS.md, 'Shipping a new capability behind an opt-in flag'." +.github/tests/test-suite-default-branch-coverage.sh|fail "the test job references '$flag_ref' OUTSIDE the OR-group that holds the path filter, so it is AND-ed at the top level and a caller that does not opt in loses the diff-triggered suite it has today. The input must gate the default-branch arm only — see AGENTS.md, 'Shipping a new capability behind an opt-in flag'." +.github/tests/test-suite-default-branch-coverage.sh|fail "no OR-arm both references '$flag_ref' and carries the default-branch clause, so the default-branch run is not the thing the opt-in input actually gates. See AGENTS.md, 'Shipping a new capability behind an opt-in flag'." +.github/tests/test-suite-default-branch-coverage.sh|fail "an OR-arm of the test job's gate carries a ref/default-branch predicate without being gated by '$flag_ref', so the new default-branch behaviour reaches consumers that never opted in. Offending arm: ${arm}. See AGENTS.md, 'Shipping a new capability behind an opt-in flag'." +.github/workflows/validate-go-project-readonly.yaml|# reachable from more than one trigger receives it either way (AGENTS.md, *Gotchas*). +.github/tests/test-create-release-breaking-bang.sh|before="$(cd "$tmp/$name" && find . -type f -exec shasum {} \; | sort)" +.github/tests/test-create-release-breaking-bang.sh|after="$(cd "$tmp/$name" && find . -type f -exec shasum {} \; | sort)" +LINES + +git -C "$root" ls-files >"$work/tracked" +# A reviewed exemption must still name a real tracked line; a stale one could +# otherwise silently excuse a future reader with the same text. +while IFS= read -r exemption; do + file="${exemption%%|*}" text="${exemption#*|}" + grep -qxF -- "$file" "$work/tracked" || { echo "FAIL: untracked non-reading exemption: $exemption" >&2; exit 1; } + # Trim into a file first: grep -q closing a pipe early would fail pipefail. + sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//' "$root/$file" >"$work/trimmed" + grep -qxF -- "$text" "$work/trimmed" || { echo "FAIL: stale non-reading exemption: $exemption" >&2; exit 1; } +done <"$work/non-reading" + +jq -r '(.jobs | keys) - (.preserved | keys) | .[]' "$inventory" >"$work/gated" +[[ -s "$work/gated" ]] || { echo 'FAIL: no selection-gated jobs to inspect' >&2; exit 1; } +jq -r '[.jobs | to_entries[] | select(.value | index("README.md")) | .key] | sort | .[]' "$inventory" >"$work/routed" +if jq -e '[.jobs[] | .[] | select(. == "AGENTS.md")] | length > 0' "$inventory" >/dev/null; then + echo 'FAIL: AGENTS.md has no gated reader and must not own selection' >&2 + exit 1 +fi +# One record per gated job: a header, then its inline text with a fixed prefix. +jq -r --rawfile gated "$work/gated" ' + ($gated | split("\n") | map(select(length > 0))) as $ids | + $ids[] as $id | .jobs[$id] as $job | + if $job == null then error("gated job \($id) is missing from the workflow") else . end | + "@@job \($id) \(($job.uses // "-") | sub("^(\\$|\\.)/"; "") | sub("@.*$"; ""))", + ([(($job | .with, .env, .defaults) // empty | tostring), ($job.steps[]? | (.uses // empty), (.run // empty), ((.with // {}) | tostring), ((.env // {}) | tostring), (."working-directory" // empty))] | join("\n") | split("\n")[] | "| " + .) +' "$workflow" >"$work/jobs" + +# Walk every job's executed code in one pass; files are read and classified once. +awk -v root="$root" -v tracked="$work/tracked" -v exempt="$work/non-reading" ' + function code(p) { return p ~ /\.(sh|bash|jq|go)$/ || p ~ /(^|\/)action\.ya?ml$/ } + function trim(s) { sub(/^[[:space:]]+/, "", s); sub(/[[:space:]]+$/, "", s); return s } + # Executable tracked files a reference names: an exact code file, or every + # code file of a local action or Go module directory. + function expand(r, i, out) { + sub(/\/+$/, "", r); sub(/^\.\//, "", r) + if (r in memo) return memo[r] + out = "" + if ((r in tracked_set) && code(r)) out = r + if (r in unit) for (i = 1; i <= ntracked; i++) + if (index(paths[i], r "/") == 1 && code(paths[i]) && paths[i] != r) out = out "\n" paths[i] + memo[r] = out + return out + } + function scan(line, dir, source, s, tok) { + s = line + while (match(s, /(\.github|\.scripts|actions|scripts|tests)\/[A-Za-z0-9_.\/-]*[A-Za-z0-9_-]/)) { + refs[source] = refs[source] "\n" substr(s, RSTART, RLENGTH); s = substr(s, RSTART + RLENGTH) + } + s = line + while (match(s, /[A-Za-z0-9_-]+\.(sh|bash|go|jq)/)) { + refs[source] = refs[source] "\n" dir "/" substr(s, RSTART, RLENGTH); s = substr(s, RSTART + RLENGTH) + } + if (line ~ mentions || line ~ whole) { + tok = trim(line) + if ((source "|" tok) in exempted) return + if (tok ~ whole || index(tok, "AGENTS.md")) verdict[source] = verdict[source] "\nfail\t" source ": " tok + else verdict[source] = verdict[source] "\nreader" + } + } + function load(p, line, dir) { + if (p in loaded) return + loaded[p] = 1; refs[p] = ""; verdict[p] = "" + dir = p; if (!sub(/\/[^\/]*$/, "", dir)) dir = "." + while ((getline line < (root "/" p)) > 0) scan(line, dir, p) + close(root "/" p) + } + function visit(p, n, i, list, m, j, targets) { + if (p in seen) return + seen[p] = 1; load(p); emit(p) + n = split(refs[p], list, "\n") + for (i = 1; i <= n; i++) if (list[i] != "") { + m = split(expand(list[i]), targets, "\n") + for (j = 1; j <= m; j++) if (targets[j] != "") visit(targets[j]) + } + } + function emit(p, n, i, list) { + n = split(verdict[p], list, "\n") + for (i = 1; i <= n; i++) { + if (list[i] == "reader") reader = 1 + else if (index(list[i], "fail\t") == 1) print "FAIL\t" id ": " substr(list[i], 6) + } + } + function finish( n, i, list, m, j, targets) { + if (id == "") return + emit(inline) + n = split(refs[inline], list, "\n") + for (i = 1; i <= n; i++) if (list[i] != "") { + m = split(expand(list[i]), targets, "\n") + for (j = 1; j <= m; j++) if (targets[j] != "") visit(targets[j]) + } + if (reusable != "-") { + if (!(reusable in tracked_set)) { print "FAIL\t" id ": calls untracked " reusable; return } + visit(reusable) + } + if (reader) print "READER\t" id + } + BEGIN { + mentions = "README|AGENTS\\.md" + whole = "(^|[^A-Za-z0-9_])(cp|rsync|tar|find|archive)[[:space:]]([^|;&]*[[:space:]])?(\"?[$][{]?(root|repo_root|source_root|GITHUB_WORKSPACE)[}]?/?[.]?\"?|[.]|[.]/)([[:space:]]|$)" + while ((getline line < tracked) > 0) { + paths[++ntracked] = line; tracked_set[line] = 1 + if (line ~ /\/(action\.ya?ml|go\.mod)$/) { d = line; sub(/\/[^\/]*$/, "", d); unit[d] = 1 } + } + while ((getline line < exempt) > 0) exempted[line] = 1 + } + /^@@job / { + finish() + id = $2; reusable = $3; reader = 0; delete seen + inline = "ci.yaml#" id; refs[inline] = ""; verdict[inline] = "" + next + } + { scan(substr($0, 3), ".", inline) } + END { finish() } +' "$work/jobs" >"$work/verdicts" + +if grep -q '^FAIL' "$work/verdicts"; then + echo 'FAIL: unreviewed AGENTS.md or whole-checkout read in a selection-gated job:' >&2 + awk -F '\t' '$1 == "FAIL" { print $2 }' "$work/verdicts" >&2 + exit 1 +fi +awk -F '\t' '$1 == "READER" { print $2 }' "$work/verdicts" | sort >"$work/readers" +if ! diff -u "$work/readers" "$work/routed" >"$work/diff"; then + echo 'FAIL: README.md route differs from its tracked gated readers (- reader not routed, + routed non-reader):' >&2 + cat "$work/diff" >&2 + exit 1 +fi +printf 'PASS: README.md routes to its %s tracked gated readers among %s gated jobs; AGENTS.md has none.\n' \ + "$(wc -l <"$work/readers" | tr -d ' ')" "$(wc -l <"$work/gated" | tr -d ' ')" diff --git a/.github/tests/test-ci-selection.sh b/.github/tests/test-ci-selection.sh index fefb0e19..0a9ead2e 100644 --- a/.github/tests/test-ci-selection.sh +++ b/.github/tests/test-ci-selection.sh @@ -187,4 +187,115 @@ if JOB_RESULTS='success failure' CATALOGUE_REQUIRED=true SELECTOR_RESULT=success echo 'FAIL: independently admitted authenticated caller failure was concealed' >&2 exit 1 fi + +# Root documents route only to the gated jobs that actually read them. Derive +# that reader set from the tracked workflow and executed scripts, and prove the +# derivation rejects a missing, invented or new reader. +readers="$root/.github/tests/ci-selection-document-readers.sh" +inventory="$root/.github/scripts/ci-selection/inventory.json" +bash "$readers" "$root" "$inventory" "$work/workflow.json" +reject_readers() { # description expected-reason inventory workflow + if bash "$readers" "$root" "$3" "$4" >"$work/readers.log" 2>&1 || ! grep -qF -- "$2" "$work/readers.log"; then + echo "FAIL: document reader guard did not reject $1 with: $2" >&2 + cat "$work/readers.log" >&2 + exit 1 + fi +} +route='README.md route differs from its tracked gated readers' +unreviewed='unreviewed AGENTS.md or whole-checkout read' +jq '.jobs["test-lint-gate-lockstep"] -= ["README.md"]' "$inventory" >"$work/readers-inventory.json" +reject_readers 'a README reader without its route' "$route" "$work/readers-inventory.json" "$work/workflow.json" +jq '.jobs["test-upload-coverage"] += ["README.md"]' "$inventory" >"$work/readers-inventory.json" +reject_readers 'an invented README reader' "$route" "$work/readers-inventory.json" "$work/workflow.json" +jq '.jobs["test-approve-pr"] += ["AGENTS.md"]' "$inventory" >"$work/readers-inventory.json" +reject_readers 'an AGENTS.md owner' 'AGENTS.md has no gated reader' "$work/readers-inventory.json" "$work/workflow.json" +for step in 'cat README.md' 'grep -q Agents AGENTS.md' 'cp -R . /tmp/copy' 'tar -cf /tmp/all.tar .' 'find . -type f'; do + reason="$unreviewed" + [[ "$step" != *README* ]] || reason="$route" + jq --arg run "$step" '.jobs["test-approve-pr"].steps += [{"run": $run}]' "$work/workflow.json" >"$work/readers-workflow.json" + reject_readers "an unrouted new reader: $step" "$reason" "$inventory" "$work/readers-workflow.json" +done +# A document named only in a step or job environment is still read. +jq '.jobs["test-approve-pr"].steps += [{"env": {"DOC": "README.md"}, "run": "cat \"$DOC\""}]' "$work/workflow.json" >"$work/readers-workflow.json" +reject_readers 'a README reader through a step environment' "$route" "$inventory" "$work/readers-workflow.json" +jq '.jobs["test-approve-pr"].env = {"DOC": "AGENTS.md"}' "$work/workflow.json" >"$work/readers-workflow.json" +reject_readers 'an AGENTS.md reader through a job environment' "$unreviewed" "$inventory" "$work/readers-workflow.json" +# A reader reached only through an executed script is still a reader; a +# script that never reads the documents keeps the route unchanged. +jq '.jobs["test-approve-pr"].steps += [{"run": "bash .github/tests/test-catalogue-pin-guards-controls.sh"}]' "$work/workflow.json" >"$work/readers-workflow.json" +reject_readers 'a transitive README reader' "$route" "$inventory" "$work/readers-workflow.json" +jq '.jobs["test-approve-pr"].steps += [{"run": "bash .github/tests/test-approve-pr-credential-boundary.sh"}]' "$work/workflow.json" >"$work/readers-workflow.json" +bash "$readers" "$root" "$inventory" "$work/readers-workflow.json" >/dev/null + +# A reviewed exemption that no longer names a real line must not linger. +awk '{ print } /^cat >"\$work\/non-reading" <<'"'"'LINES'"'"'$/ { print "actions/approve-pr/action.yaml|cat README.md" }' "$readers" >"$work/stale-readers.sh" +if bash "$work/stale-readers.sh" "$root" "$inventory" "$work/workflow.json" >"$work/readers.log" 2>&1 || + ! grep -qF 'stale non-reading exemption' "$work/readers.log"; then + echo 'FAIL: document reader guard kept a stale exemption' >&2 + exit 1 +fi + +# Exercise real root-document change sets through the production selector and +# the required reducer. Narrow routes still require every selected success. +docs="$work/docs" +mkdir -p "$docs/actions/approve-pr" +git init -q "$docs" +git -C "$docs" config user.name 'CI fixture' +git -C "$docs" config user.email 'fixture@example.invalid' +git -C "$docs" config commit.gpgsign false +printf 'readme\n' >"$docs/README.md" +printf 'agents\n' >"$docs/AGENTS.md" +printf 'action\n' >"$docs/actions/approve-pr/action.yaml" +git -C "$docs" add -- README.md AGENTS.md actions/approve-pr/action.yaml +git -C "$docs" commit -qm 'test: documents baseline' +docs_base="$(git -C "$docs" rev-parse HEAD)" +select_docs() { # change-kind... ; prints the selected JSON + git -C "$docs" checkout -q --detach "$docs_base" + local change + for change in "$@"; do + case "$change" in + readme) printf 'readme change\n' >>"$docs/README.md" ;; + agents) printf 'agents change\n' >>"$docs/AGENTS.md" ;; + action) printf 'action change\n' >>"$docs/actions/approve-pr/action.yaml" ;; + unknown) printf 'unknown\n' >"$docs/unclassified.txt" ;; + readme-renamed) git -C "$docs" mv README.md docs-readme.md ;; + esac + done + git -C "$docs" add -A -- . + git -C "$docs" commit -qm "test: $*" + : >"$work/docs-output" + EVENT_NAME=pull_request RUN_CATALOGUE=true CATALOGUE_SCOPE=true BASE_SHA="$docs_base" \ + HEAD_SHA="$(git -C "$docs" rev-parse HEAD)" GITHUB_OUTPUT="$work/docs-output" \ + go -C "$root/.github/scripts/ci-selection" run . "$docs" "$inventory" "$work/workflow.json" >/dev/null + sed -n 's/^selected=//p' "$work/docs-output" +} +readme_readers='["test-enable-auto-merge-author-gate","test-enable-auto-merge-pentad-gate","test-lint-gate-lockstep","test-validate-go-gate-lockstep"]' +action_only="$(select_docs action)" +jq -e '. == ["test-approve-pr","test-approve-pr-credential-boundary"]' <<<"$action_only" >/dev/null +jq -e --argjson want "$readme_readers" '. == $want' <<<"$(select_docs readme)" >/dev/null || + { echo 'FAIL: README-only change did not select exactly its readers' >&2; exit 1; } +jq -e --argjson want "$action_only" '. == $want' <<<"$(select_docs action agents)" >/dev/null || + { echo 'FAIL: AGENTS.md companion changed a recognized action selection' >&2; exit 1; } +jq -e --argjson a "$action_only" --argjson r "$readme_readers" '. == ($a + $r | unique)' <<<"$(select_docs action readme agents)" >/dev/null || + { echo 'FAIL: README with a recognized action lost a reader or an owner' >&2; exit 1; } +for kinds in 'agents' 'readme agents' 'readme unknown' 'action agents unknown' 'readme-renamed'; do + # shellcheck disable=SC2086 # Each kinds word is one fixture change. + jq -e 'length == 61' <<<"$(select_docs $kinds)" >/dev/null || + { echo "FAIL: root-document change '$kinds' narrowed full coverage" >&2; exit 1; } +done +for kinds in 'readme' 'action readme agents'; do + # shellcheck disable=SC2086 # Each kinds word is one fixture change. + selected="$(select_docs $kinds)" + needs="$(jq -cn --argjson selected "$selected" 'reduce $selected[] as $id ({"test-enable-auto-merge-queue":{result:"skipped"},"test-go-workflow-guards":{result:"skipped"}}; .[$id] = {result:"success"})')" + JOB_RESULTS='success skipped' CATALOGUE_REQUIRED=true SELECTOR_RESULT=success SELECTED_JOBS="$selected" NEEDS_JSON="$needs" run_reducer >/dev/null + for caller in $(jq -r '.[]' <<<"$selected"); do + for outcome in skipped failure; do + bad_needs="$(jq --arg caller "$caller" --arg outcome "$outcome" '.[$caller].result=$outcome' <<<"$needs")" + if JOB_RESULTS="$(jq -r '[.[].result] | join(" ")' <<<"$bad_needs")" CATALOGUE_REQUIRED=true SELECTOR_RESULT=success SELECTED_JOBS="$selected" NEEDS_JSON="$bad_needs" run_reducer >/dev/null 2>&1; then + echo "FAIL: selected root-document reader $caller was $outcome without rejection" >&2 + exit 1 + fi + done + done +done echo 'PASS: complete job inventory, owner coverage and preserved scheduling'