From e982c7be7cf726b1b4bf70ce89d7f188bbd3a129 Mon Sep 17 00:00:00 2001 From: Iurii Trukhin Date: Fri, 6 Nov 2026 01:01:35 +0100 Subject: [PATCH 1/3] feat(safepush): verify native GitHub CI provenance Signed-off-by: Iurii Trukhin --- .github/safepush.json | 54 ++ cmd/cloudring-safepush/main.go | 130 +++++ cmd/cloudring-safepush/main_test.go | 90 +++ docs/ci-checks.md | 51 +- docs/safepush.md | 128 +++++ pkg/safepush/checks.go | 157 ++++++ pkg/safepush/checks_test.go | 330 +++++++++++ pkg/safepush/doc.go | 19 + pkg/safepush/github.go | 748 +++++++++++++++++++++++++ pkg/safepush/github_test.go | 718 ++++++++++++++++++++++++ pkg/safepush/policy.go | 155 +++++ pkg/safepush/policy_test.go | 79 +++ pkg/safepush/repository_policy_test.go | 117 ++++ pkg/safepush/workflow_source.go | 130 +++++ pkg/safepush/workflow_source_test.go | 129 +++++ 15 files changed, 3009 insertions(+), 26 deletions(-) create mode 100644 .github/safepush.json create mode 100644 cmd/cloudring-safepush/main.go create mode 100644 cmd/cloudring-safepush/main_test.go create mode 100644 docs/safepush.md create mode 100644 pkg/safepush/checks.go create mode 100644 pkg/safepush/checks_test.go create mode 100644 pkg/safepush/doc.go create mode 100644 pkg/safepush/github.go create mode 100644 pkg/safepush/github_test.go create mode 100644 pkg/safepush/policy.go create mode 100644 pkg/safepush/policy_test.go create mode 100644 pkg/safepush/repository_policy_test.go create mode 100644 pkg/safepush/workflow_source.go create mode 100644 pkg/safepush/workflow_source_test.go diff --git a/.github/safepush.json b/.github/safepush.json new file mode 100644 index 0000000..dbb5878 --- /dev/null +++ b/.github/safepush.json @@ -0,0 +1,54 @@ +{ + "schemaVersion": "cloudring.safepush.github/v1", + "repository": "opencloudtech/CloudRING", + "repositoryId": 1288130713, + "workflows": [ + { + "path": ".github/workflows/ci.yml", + "jobs": [ + "docscheck", + "go-build-module-integrity", + "go-test", + "go-test-race", + "go-vet", + "ocs-conformance", + "ocs-validate", + "platform-manifests", + "postgres-integration", + "source-safety", + "windows-go-test" + ] + }, + { + "path": ".github/workflows/cla-dco.yml", + "jobs": [ + "cla", + "dco" + ] + }, + { + "path": ".github/workflows/license.yml", + "jobs": [ + "license-check" + ] + }, + { + "path": ".github/workflows/security.yml", + "jobs": [ + "codeql", + "gosec", + "govulncheck", + "secret-scan" + ] + }, + { + "path": ".github/workflows/supply-chain.yml", + "jobs": [ + "supply-chain-policy" + ] + } + ], + "excludedWorkflows": [ + ".github/workflows/release-provenance.yml" + ] +} diff --git a/cmd/cloudring-safepush/main.go b/cmd/cloudring-safepush/main.go new file mode 100644 index 0000000..fb4aa4a --- /dev/null +++ b/cmd/cloudring-safepush/main.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "time" + + "github.com/opencloudtech/CloudRING/pkg/safepush" +) + +type githubVerifier func(context.Context, *http.Client, string, safepush.GitHubPolicy, safepush.GitHubCandidate) error +type githubPolicyReader func(context.Context, *http.Client, string, string, string) (safepush.GitHubPolicy, error) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + os.Exit(run(ctx, os.Args[1:], os.Getenv, os.Stdout, os.Stderr, safepush.ReadGitHubPolicy, safepush.VerifyGitHub)) +} + +// The native required workflow owns these arguments. There is deliberately no +// caller-supplied check-results file, success flag, API host, or policy override. +func run(ctx context.Context, args []string, getenv func(string) string, stdout, stderr io.Writer, read githubPolicyReader, verify githubVerifier) int { + if len(args) == 0 || args[0] != "github" { + fmt.Fprintln(stderr, "usage: cloudring-safepush github --repository OWNER/REPO --pr NUMBER --head SHA --base SHA --branch main|master") + return 2 + } + flags := flag.NewFlagSet("github", flag.ContinueOnError) + flags.SetOutput(io.Discard) + repository := flags.String("repository", "", "target GitHub repository") + pr := flags.Int("pr", 0, "pull request number") + head := flags.String("head", "", "exact proposed commit") + base := flags.String("base", "", "exact accepted target commit") + branch := flags.String("branch", "", "protected target branch") + timeout := flags.Duration("timeout", 20*time.Minute, "bounded wait for native CI, at most 30 minutes") + if flags.Parse(args[1:]) != nil || flags.NArg() != 0 || *repository == "" || *pr <= 0 || + *head == "" || *base == "" || (*branch != "main" && *branch != "master") || + *timeout <= 0 || *timeout > 30*time.Minute { + fmt.Fprintln(stderr, "SafePush request is invalid") + return 2 + } + candidate := safepush.GitHubCandidate{PullRequest: *pr, HeadSHA: *head, BaseSHA: *base, TargetBranch: *branch} + if safepush.ValidateGitHubCandidate(*repository, candidate) != nil { + fmt.Fprintln(stderr, "SafePush request is invalid") + return 2 + } + bearer := getenv("GITHUB_TOKEN") + if bearer == "" { + fmt.Fprintln(stderr, "SafePush requires the workflow's read-only GITHUB_TOKEN") + return 2 + } + ctx, cancel := context.WithTimeout(ctx, *timeout) + defer cancel() + client := &http.Client{Timeout: 30 * time.Second} + policy, err := read(ctx, client, bearer, *repository, *base) + if err != nil { + fmt.Fprintln(stderr, "SafePush blocked:", safeDiagnostic(err)) + return 1 + } + if policy.Repository != *repository { + fmt.Fprintln(stderr, "SafePush blocked: accepted policy belongs to another repository") + return 1 + } + err = waitForChecks(ctx, 30*time.Second, func() error { return verify(ctx, client, bearer, policy, candidate) }) + if err != nil { + fmt.Fprintln(stderr, "SafePush blocked:", safeDiagnostic(err)) + return 1 + } + if _, err := fmt.Fprintln(stdout, "SafePush: the exact required CI checks passed for the current candidate and base"); err != nil { + return 1 + } + return 0 +} + +func safeDiagnostic(err error) string { + switch { + case errors.Is(err, context.Canceled): + return "request canceled" + case errors.Is(err, context.DeadlineExceeded): + return "verification deadline exceeded" + case errors.Is(err, safepush.ErrPolicy): + return "accepted repository policy is invalid" + case errors.Is(err, safepush.ErrGitHubAPI): + return "GitHub API is unavailable or returned an invalid response" + case errors.Is(err, safepush.ErrGitHubSource): + return "workflow or policy source does not match the accepted revision" + case errors.Is(err, safepush.ErrGitHubState): + return "pull request or protected target is stale or changed during verification" + case errors.Is(err, safepush.ErrGitHubRun): + return "CI run, job, or step provenance is invalid" + case errors.Is(err, safepush.ErrMissingCheck): + return "a required CI job is missing" + case errors.Is(err, safepush.ErrUnexpectedCheck), errors.Is(err, safepush.ErrDuplicateObservation): + return "the CI job set is unexpected or ambiguous" + case errors.Is(err, safepush.ErrCheckNotSuccessful): + return "a required CI job did not complete successfully" + default: + return "verification failed" + } +} + +func waitForChecks(ctx context.Context, interval time.Duration, verify func() error) error { + for { + if err := ctx.Err(); err != nil { + return err + } + err := verify() + if contextErr := ctx.Err(); contextErr != nil { + return contextErr + } + if !errors.Is(err, safepush.ErrGitHubPending) { + return err + } + timer := time.NewTimer(interval) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} diff --git a/cmd/cloudring-safepush/main_test.go b/cmd/cloudring-safepush/main_test.go new file mode 100644 index 0000000..507953b --- /dev/null +++ b/cmd/cloudring-safepush/main_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "bytes" + "context" + "errors" + "net/http" + "strings" + "testing" + "time" + + "github.com/opencloudtech/CloudRING/pkg/safepush" +) + +func TestPendingIsTheOnlyRetriableAdmissionResult(t *testing.T) { + for _, terminal := range []error{nil, safepush.ErrCheckNotSuccessful, safepush.ErrPolicy, errors.New("unavailable")} { + calls := 0 + err := waitForChecks(context.Background(), time.Nanosecond, func() error { + calls++ + if calls == 1 { + return safepush.ErrGitHubPending + } + return terminal + }) + if calls != 2 || !errors.Is(err, terminal) { + t.Fatalf("calls=%d error=%v", calls, err) + } + } +} + +func TestCancellationStopsPendingAdmission(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + err := waitForChecks(ctx, time.Hour, func() error { calls++; cancel(); return safepush.ErrGitHubPending }) + if !errors.Is(err, context.Canceled) || calls != 1 { + t.Fatalf("calls=%d error=%v", calls, err) + } +} + +func TestSuccessAfterCancellationIsNotAccepted(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + err := waitForChecks(ctx, time.Nanosecond, func() error { cancel(); return nil }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("late success was accepted: %v", err) + } +} + +func TestCommandDoesNotEchoSensitiveProviderErrors(t *testing.T) { + args := []string{"github", "--repository", "synthetic/project", "--pr", "1", "--head", strings.Repeat("a", 40), "--base", strings.Repeat("b", 40), "--branch", "main"} + marker := "synthetic-confidential-marker" + read := func(context.Context, *http.Client, string, string, string) (safepush.GitHubPolicy, error) { + return safepush.GitHubPolicy{Repository: "synthetic/project"}, nil + } + for _, failure := range []error{nil, errors.New(marker)} { + var out, errout bytes.Buffer + code := run(context.Background(), args, func(string) string { return marker }, &out, &errout, read, + func(context.Context, *http.Client, string, safepush.GitHubPolicy, safepush.GitHubCandidate) error { + return failure + }) + if (code == 0) != (failure == nil) { + t.Fatalf("unexpected exit %d", code) + } + if strings.Contains(out.String()+errout.String(), marker) { + t.Fatal("provider error leaked sensitive input") + } + } +} + +func TestInvalidInvocationMakesNoProviderCall(t *testing.T) { + for _, args := range [][]string{ + {}, {"github", "--trusted=true"}, + {"github", "--repository", "synthetic/project", "--pr", "1", "--head", "a", "--base", "b", "--branch", "feature"}, + {"github", "--repository", "synthetic/project", "--pr", "1", "--head", "a", "--base", "b", "--branch", "main", "--timeout", "31m"}, + {"github", "--repository", "synthetic/project", "--pr", "1", "--head", "a", "--base", "main", "--branch", "main"}, + {"github", "--repository", "synthetic/../project", "--pr", "1", "--head", strings.Repeat("a", 40), "--base", strings.Repeat("b", 40), "--branch", "main"}, + {"github", "--repository", "synthetic/project", "--pr", "1", "--head", strings.Repeat("0", 40), "--base", strings.Repeat("b", 40), "--branch", "main"}, + } { + var out bytes.Buffer + read := func(context.Context, *http.Client, string, string, string) (safepush.GitHubPolicy, error) { + t.Fatal("provider called for invalid request") + return safepush.GitHubPolicy{}, nil + } + if code := run(context.Background(), args, func(string) string { return "synthetic-test-value" }, &out, &out, read, nil); code != 2 { + t.Fatalf("exit=%d", code) + } + } +} diff --git a/docs/ci-checks.md b/docs/ci-checks.md index 42d7e85..7dd648b 100644 --- a/docs/ci-checks.md +++ b/docs/ci-checks.md @@ -1,8 +1,7 @@ # CloudRING CI Checks -Public CI for a clean clone runs these read-only checks on the latest security -patch of the minimum supported Go release (1.25) and runs unit tests again on -the current supported release (1.26): +Public CI for a clean clone runs these read-only checks with the workflow's +pinned Go toolchain, currently Go 1.26.9: ```bash go mod download @@ -31,9 +30,9 @@ The public CI contract covers these checks: | Check | Contract | | --- | --- | -| Go tests | The public module must pass tests on supported minimum/current Go releases, plus race, vet, read-only module graph, and build checks. | +| Go tests | The complete public module must pass tests with the pinned CI toolchain, plus race, vet, read-only module graph, and build checks. | | PostgreSQL integration | Transactional-state CAS, migrations, concurrent writers, and the public runtime/session journey must pass against real digest-pinned PostgreSQL. The pinned Linux test container also exercises interrupted guest database directory setup and ownership rejection as UID 0; it asserts that identity before running these filesystem tests. | -| Windows | The same unit suite is run on `windows-latest` as a portability signal. Native Windows support is not a release-readiness blocker for the current goal. | +| Windows | The same unit suite runs on `windows-latest` and is included in the SafePush pre-merge policy. A passing suite does not establish native Windows release readiness. | | OCS validation | Every shipped connector package selected by the shared CI package list must pass `go run ./cmd/ocsctl validate`. | | OCS conformance | The same exact shipped connector packages must pass `go run ./cmd/ocsctl conformance`; validation cannot be green for an artifact that CI omits from conformance. | | Synthetic reference image | The digest-pinned `Containerfile` must build and its local mock-provider self-check must pass. | @@ -66,27 +65,27 @@ runtime image, credential context, conditional skip, and `continue-on-error`. Changing a reviewed workflow requires changing its recorded digest in the same review. -This repository check is defense in depth, not a self-authenticating policy: a -pull request controls the workflow revision that evaluates that pull request. -The server-side trust control for external contributors is protected `main` -with an up-to-date branch, one approving owner review, stale-review dismissal, -last-push approval, required conversations and checks, and no force-push or -branch deletion. The project founder and lead maintainer, `@trukhinyuri`, is -the final acceptance authority. Contributor-authored changes require that -owner review; automated or AI-assisted review does not replace the founder's -decision. - -GitHub does not let a pull-request author submit the approval required by the -same branch rule. While the repository has a single founder/administrator, a -founder-authored pull request may therefore use the administrator merge path -after a recorded founder review of the exact head SHA. This narrow owner path -still requires every configured status check to succeed on the current head, -an up-to-date base, resolved conversations, source-safety approval, and a -post-merge `main` verification. It does not authorize direct pushes, skipped -checks, force-pushes, branch deletion, or bypasses for other contributors. -Adding another administrator changes this trust assumption and requires an -explicit governance and branch-protection review before that access is -granted. +This repository check is defense in depth: a pull request controls the +workflow revision that evaluates that pull request. The separate +[SafePush verifier and deployment contract](safepush.md) bind CI observations +to the accepted policy, workflow sources, repository, pull request and commit. +The verifier must run from trusted immutable source and be required by the +hosting platform. Its presence in the repository does not enable that control. + +The required acceptance policy is an up-to-date protected target, every +required test passing and an owner-approved merge. Both `main` and `master` +must be covered; ordinary working branches remain unrestricted. The project +founder and lead maintainer, `@trukhinyuri`, is the final acceptance authority. +Automated or AI-assisted review does not replace that decision. Native review +rules must dismiss stale approvals and prevent unauthorized dismissal. + +Only the explicitly designated owner may bypass test and review requirements; +future delegates require an explicit policy change. Administrator or +Maintainer status alone must not grant that exception. This owner path also +handles the platform's prohibition on approving one's own pull request. +Force-push and deletion controls remain separate from the test/review +exception. The actual installed rules and both ordinary and bypass behavior +must be read back and tested before claiming SafePush is enabled. ## Release provenance diff --git a/docs/safepush.md b/docs/safepush.md new file mode 100644 index 0000000..22d7fff --- /dev/null +++ b/docs/safepush.md @@ -0,0 +1,128 @@ +# SafePush admission + +SafePush checks whether the exact proposed source has completed the full test +policy before a protected branch can change. It belongs in trusted CI and the +hosting platform's admission controls. A local hook is an optional early check; +disabling it must never permit a protected update. + +## Implementation and enforcement are separate + +`pkg/safepush` provides the check evaluator and a native GitHub CI adapter. +`cloudring-safepush github` reads `.github/safepush.json` from the exact accepted +target commit through the GitHub API, then verifies the current pull request +and its CI runs. It does not read a candidate-supplied success report, execute +candidate code, approve a review, publish a status, merge, or update a ref. + +The policy is a complete list of pre-merge workflows and jobs. In particular, +Windows tests are included even when a historical branch rule omitted them. +Excluded workflows must be explicitly listed and must not contain required +pre-merge tests. The release workflow runs after acceptance and is checked +separately as part of release delivery. + +Only actual completed-success jobs satisfy the policy. A missing job, an +unexpected job, a duplicate, a skipped or neutral result, an earlier successful +attempt hiding a later failure, an unrelated same-named workflow, or a changed +head/base must block the result. An API failure or incomplete response is not +evidence that there are no failing checks. + +The adapter also reads each accepted required workflow blob and checks its +Git object hash before parsing the YAML. A job or step may omit +`continue-on-error` or set it to the literal boolean `false`. Allowed failures +cannot be identified reliably from API conclusions alone. YAML aliases, +anchors, merge keys, duplicate keys and multiple documents are unsupported +and block verification. + +The native adapter owns pagination, repository/run/job identity, source +binding, selection of the current attempt and final state readback. The pure +`VerifyChecks` helper checks consistency of those observations; it is not an +authentication or enforcement boundary on its own. Passing a metadata check +also does not prove that a modified test still tests the right behavior. Human +review remains necessary. + +GitHub observations require an open, non-draft PR and the current target as +an ancestor of its head. Every selected run must be associated with that exact +PR, head, base and target. These API reads are not atomic with a later merge, +and the run's head and PR association alone do not prove the historical tested +merge commit. Native target freshness and owner review remain required; the +deployment must qualify its event and checkout behavior before activation. + +## GitHub deployment contract + +For repositories that require pull requests, the organization must require the +SafePush workflow from an immutable accepted CloudRING source commit. A +candidate-owned wrapper calling a reusable workflow is insufficient: the +candidate could delete or replace the wrapper. Requiring only the GitHub +Actions app and a check name also does not identify a particular workflow. + +The trusted workflow must check out only its own immutable source, run on a +GitHub-hosted runner, and use read-only contents, Actions, checks and pull +request access. It must not execute candidate scripts or actions, consume +candidate artifacts or shared build caches, or receive publishing credentials. +The CLI takes the current candidate identity from the native event and reads +the accepted policy itself: + +```sh +cloudring-safepush github \ + --repository "$TARGET_REPOSITORY" \ + --pr "$PULL_REQUEST_NUMBER" \ + --head "$CANDIDATE_SHA" \ + --base "$ACCEPTED_BASE_SHA" \ + --branch "$TARGET_BRANCH" +``` + +The job supplies its read-only `GITHUB_TOKEN`. Only pending CI is retried, for +a bounded time. Completed failures, mismatched identity, changed source, +authentication failure and malformed responses stop admission immediately. + +Native rules must independently enforce the owner's approval, dismiss stale +reviews and prevent unauthorized review dismissal. Both `main` and `master` +need protection. Test/review bypass identities must be explicit; assigning an +administrator role must not automatically add a SafePush exception. Force +push and deletion controls should remain separate from test/review exceptions. +Existing native code scanning checks remain required alongside SafePush; +generated provider workflows must not be silently relabeled as trusted +repository files. + +Keep every underlying required CI check in the native rule as well. An old +successful aggregate cannot revoke itself if a later attempt on the same head +fails after the verifier exits. Activation must test that a later failing +attempt blocks acceptance even when an earlier aggregate was successful. + +GitHub's required-workflow rule blocks direct pushes and supports PR and merge +queue events. It must not be presented as accepting arbitrary prechecked direct +pushes. A merge queue needs its own exact merge-group observation contract +before being enabled; a PR-only verifier cannot validate it. + +## GitLab deployment contract + +The tests must run on qualified nonpersonal Linux CI infrastructure with an +immutable trusted configuration. A candidate-local `.gitlab-ci.yml` include +cannot make a policy mandatory, because another candidate can remove or +override it. Pipeline and approval observations must come from GitLab, not +forwarded variables asserting that every gate passed. + +On GitLab Community Edition, successful-pipeline merge settings do not check +direct pushes, and role-based branch permissions cannot isolate one reviewer +when the project inherits other Maintainers and Owners. A green approval job +is also insufficient: the approval can be revoked after the pipeline passes. +Where native enforcement cannot express the policy, a server admission hook +must perform the final current-state decision before the ref changes. + +Such a hook must be installed and owned by the GitLab administrator, outside +candidate-controlled files. Its allowlist must scope it to the intended +CloudRING project IDs and protected refs. It must preserve unrestricted +ordinary branch pushes, reject absent/ambiguous/stale CI or review evidence, +verify the exact old/new commit relationship, and support only explicitly +authorized bypass identities. It must not use a personal workstation or +execute candidate code with its admission credentials. + +## Activation evidence + +Committing these sources or passing local tests does not enable server +enforcement. Record the actual installed immutable revision and re-read the +hosting platform's rules. Use an identity without bypass to verify that an +ordinary branch push succeeds while missing/failed/skipped tests, an +unapproved change, altered CI policy and stale commit evidence cannot advance +`main` or `master`. Verify a real approved all-green change, the intended owner +exception and unchanged out-of-scope repositories. Preserve an explicit +incomplete state until those checks have passed. diff --git a/pkg/safepush/checks.go b/pkg/safepush/checks.go new file mode 100644 index 0000000..d9a250d --- /dev/null +++ b/pkg/safepush/checks.go @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "errors" + "strings" + "unicode" + "unicode/utf8" +) + +// Verification errors contain fixed categories, never input values or CI logs. +// When several defects are present, VerifyChecks returns the first applicable +// category in this declaration order, independent of slice ordering. +var ( + ErrInvalidHeadSHA = errors.New("safepush: invalid head SHA") + ErrNoRequiredChecks = errors.New("safepush: no required checks") + ErrInvalidRequiredCheck = errors.New("safepush: invalid required check identity") + ErrDuplicateRequiredCheck = errors.New("safepush: duplicate required check identity") + ErrInvalidObservation = errors.New("safepush: invalid observation identity") + ErrHeadMismatch = errors.New("safepush: observation head does not match candidate") + ErrDuplicateObservation = errors.New("safepush: duplicate observation identity") + ErrUnexpectedCheck = errors.New("safepush: unexpected check identity") + ErrMissingCheck = errors.New("safepush: missing required check") + ErrCheckNotSuccessful = errors.New("safepush: check did not complete successfully") +) + +// CheckIdentity identifies one job in an adapter-verified immutable workflow. +// Source binds the workflow path and its trusted revision or content, rather +// than a display name. The adapter must establish that binding using native API +// records; setting Source alone is not evidence of authenticity. +// +// Both fields are compared exactly, without case folding, trimming, or Unicode +// normalization. They must be nonblank valid UTF-8 without control characters. +type CheckIdentity struct { + Source string `json:"source"` + Job string `json:"job"` +} + +// CheckObservation is one native CI job result for the selected run attempt. +// Status and Conclusion use the canonical pair "completed" and "success" only +// when the provider reports actual successful completion. The adapter must not +// translate skipped, neutral, allowed-failure, or absent results to that pair. +type CheckObservation struct { + Check CheckIdentity `json:"check"` + HeadSHA string `json:"headSHA"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` +} + +// VerifyChecks requires exactly one successful observation for every required +// identity and no other observations. The required set must not be empty. +// Every observation must match the exact candidate headSHA, which must be a +// nonzero 40- or 64-character lowercase hexadecimal Git object ID. +// +// Only Status == "completed" and Conclusion == "success" passes. Duplicate +// observations are ambiguous even when they agree; the adapter must select the +// authoritative run attempt before calling this function. Inputs are not +// modified, and nil means only that these consistency checks passed. See the +// package documentation for the adapter's remaining authorization obligations. +func VerifyChecks(headSHA string, required []CheckIdentity, observations []CheckObservation) error { + if !validHeadSHA(headSHA) { + return ErrInvalidHeadSHA + } + if len(required) == 0 { + return ErrNoRequiredChecks + } + + requiredSet := make(map[CheckIdentity]struct{}, len(required)) + var invalidRequired, duplicateRequired bool + for _, check := range required { + if !validCheckIdentity(check) { + invalidRequired = true + } + if _, exists := requiredSet[check]; exists { + duplicateRequired = true + } + requiredSet[check] = struct{}{} + } + if invalidRequired { + return ErrInvalidRequiredCheck + } + if duplicateRequired { + return ErrDuplicateRequiredCheck + } + + observedSet := make(map[CheckIdentity]struct{}, len(observations)) + var invalidObservation, headMismatch, duplicateObservation, unexpected, notSuccessful bool + for _, observation := range observations { + if !validCheckIdentity(observation.Check) { + invalidObservation = true + } + if observation.HeadSHA != headSHA { + headMismatch = true + } + if _, exists := observedSet[observation.Check]; exists { + duplicateObservation = true + } + observedSet[observation.Check] = struct{}{} + if _, exists := requiredSet[observation.Check]; !exists { + unexpected = true + } + if observation.Status != "completed" || observation.Conclusion != "success" { + notSuccessful = true + } + } + + switch { + case invalidObservation: + return ErrInvalidObservation + case headMismatch: + return ErrHeadMismatch + case duplicateObservation: + return ErrDuplicateObservation + case unexpected: + return ErrUnexpectedCheck + case len(observedSet) != len(requiredSet): + return ErrMissingCheck + case notSuccessful: + return ErrCheckNotSuccessful + default: + return nil + } +} + +func validHeadSHA(value string) bool { + if len(value) != 40 && len(value) != 64 { + return false + } + var nonzero bool + for _, char := range []byte(value) { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + if char != '0' { + nonzero = true + } + } + return nonzero +} + +func validCheckIdentity(identity CheckIdentity) bool { + return validIdentityPart(identity.Source) && validIdentityPart(identity.Job) +} + +func validIdentityPart(value string) bool { + if !utf8.ValidString(value) || strings.TrimSpace(value) == "" { + return false + } + for _, char := range value { + if unicode.IsControl(char) { + return false + } + } + return true +} diff --git a/pkg/safepush/checks_test.go b/pkg/safepush/checks_test.go new file mode 100644 index 0000000..c2aaff1 --- /dev/null +++ b/pkg/safepush/checks_test.go @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "errors" + "reflect" + "slices" + "strings" + "testing" +) + +func TestVerifyChecksAcceptsExactSuccessfulJobs(t *testing.T) { + t.Parallel() + for _, length := range []int{40, 64} { + head := strings.Repeat("ab", length/2) + t.Run(head, func(t *testing.T) { + t.Parallel() + required, observations := successfulChecks(head) + // API order is not evidence, and the same job name in two verified + // workflows denotes two distinct requirements. + slices.Reverse(observations) + beforeRequired := slices.Clone(required) + beforeObserved := slices.Clone(observations) + if err := VerifyChecks(head, required, observations); err != nil { + t.Fatalf("complete successful evidence rejected: %v", err) + } + if !reflect.DeepEqual(required, beforeRequired) || !reflect.DeepEqual(observations, beforeObserved) { + t.Fatal("verification mutated its inputs") + } + }) + } +} + +func TestVerifyChecksRejectsIncompleteOrAmbiguousEvidence(t *testing.T) { + t.Parallel() + tests := []struct { + name string + change func([]CheckIdentity, []CheckObservation) ([]CheckIdentity, []CheckObservation) + want error + }{ + { + name: "empty policy cannot disable admission", + change: func(_ []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + return nil, observations + }, + want: ErrNoRequiredChecks, + }, + { + name: "no observations", + change: func(required []CheckIdentity, _ []CheckObservation) ([]CheckIdentity, []CheckObservation) { + return required, nil + }, + want: ErrMissingCheck, + }, + { + name: "one missing job despite other green jobs", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + return required, observations[1:] + }, + want: ErrMissingCheck, + }, + { + name: "same job name from another workflow is not evidence", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + observations[0].Check.Source = ".github/workflows/other.yml@" + strings.Repeat("cd", 20) + return required, observations + }, + want: ErrUnexpectedCheck, + }, + { + name: "different workflow revision is not evidence", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + observations[0].Check.Source = ".github/workflows/go.yml@" + strings.Repeat("ef", 20) + return required, observations + }, + want: ErrUnexpectedCheck, + }, + { + name: "green result for stale head", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + observations[0].HeadSHA = strings.Repeat("cd", 20) + return required, observations + }, + want: ErrHeadMismatch, + }, + { + name: "observation lacks head", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + observations[0].HeadSHA = "" + return required, observations + }, + want: ErrHeadMismatch, + }, + { + name: "duplicate policy job", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + return append(required, required[0]), observations + }, + want: ErrDuplicateRequiredCheck, + }, + { + name: "duplicate successful observations", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + return required, append(observations, observations[0]) + }, + want: ErrDuplicateObservation, + }, + { + name: "green attempt cannot hide failed attempt", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + failed := observations[0] + failed.Conclusion = "failure" + return required, append(observations, failed) + }, + want: ErrDuplicateObservation, + }, + { + name: "unconfigured successful job", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + extra := observations[0] + extra.Check.Job = "new test suite" + return required, append(observations, extra) + }, + want: ErrUnexpectedCheck, + }, + { + name: "unconfigured failed job cannot be ignored", + change: func(required []CheckIdentity, observations []CheckObservation) ([]CheckIdentity, []CheckObservation) { + extra := observations[0] + extra.Check.Job = "new test suite" + extra.Conclusion = "failure" + return required, append(observations, extra) + }, + want: ErrUnexpectedCheck, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + head := strings.Repeat("ab", 20) + required, observations := successfulChecks(head) + required, observations = test.change(required, observations) + if err := VerifyChecks(head, required, observations); !errors.Is(err, test.want) { + t.Fatalf("got %v, want %v", err, test.want) + } + // Selecting whichever duplicate/defect the API returned first must + // not change the decision or its category. + slices.Reverse(required) + slices.Reverse(observations) + if err := VerifyChecks(head, required, observations); !errors.Is(err, test.want) { + t.Fatalf("reordered input: got %v, want %v", err, test.want) + } + }) + } +} + +func TestVerifyChecksRejectsEveryNonSuccessState(t *testing.T) { + t.Parallel() + tests := []struct { + status string + conclusion string + }{ + {status: "pending"}, + {status: "queued"}, + {status: "running"}, + {status: "in_progress"}, + {status: "waiting"}, + {status: "requested"}, + {status: "cancelled"}, + {status: "skipped"}, + {status: "success"}, + {status: "completed"}, + {status: "completed", conclusion: "failure"}, + {status: "completed", conclusion: "cancelled"}, + {status: "completed", conclusion: "skipped"}, + {status: "completed", conclusion: "neutral"}, + {status: "completed", conclusion: "timed_out"}, + {status: "completed", conclusion: "action_required"}, + {status: "completed", conclusion: "stale"}, + {status: "completed", conclusion: "startup_failure"}, + {status: "completed", conclusion: "unknown"}, + {status: "in_progress", conclusion: "success"}, + {status: "unknown", conclusion: "success"}, + {status: "Completed", conclusion: "success"}, + {status: "completed", conclusion: "Success"}, + {status: " completed", conclusion: "success"}, + {status: "completed", conclusion: "success "}, + {}, + } + for _, test := range tests { + t.Run(test.status+"/"+test.conclusion, func(t *testing.T) { + t.Parallel() + head := strings.Repeat("ab", 20) + required, observations := successfulChecks(head) + observations[1].Status = test.status + observations[1].Conclusion = test.conclusion + if err := VerifyChecks(head, required, observations); !errors.Is(err, ErrCheckNotSuccessful) { + t.Fatalf("non-success result: got %v, want %v", err, ErrCheckNotSuccessful) + } + }) + } +} + +func TestVerifyChecksRejectsMalformedHeadEvenWhenObservationsAgree(t *testing.T) { + t.Parallel() + for _, head := range []string{ + "", "main", strings.Repeat("a", 39), strings.Repeat("a", 41), + strings.Repeat("a", 63), strings.Repeat("a", 65), strings.Repeat("A", 40), + strings.Repeat("A", 64), strings.Repeat("g", 40), strings.Repeat("0", 40), + strings.Repeat("0", 64), strings.Repeat("a", 39) + "\n", strings.Repeat("a", 39) + " ", + } { + t.Run(head, func(t *testing.T) { + t.Parallel() + required, observations := successfulChecks(head) + if err := VerifyChecks(head, required, observations); !errors.Is(err, ErrInvalidHeadSHA) { + t.Fatalf("malformed candidate: got %v, want %v", err, ErrInvalidHeadSHA) + } + }) + } +} + +func TestVerifyChecksRejectsMalformedIdentities(t *testing.T) { + t.Parallel() + for _, value := range []string{"", " ", "\u00a0", "\x00job", "job\n", "job\tname", "job\x7f", "\xff"} { + t.Run(value, func(t *testing.T) { + t.Parallel() + for _, field := range []string{"source", "job"} { + for _, location := range []string{"required", "observation"} { + head := strings.Repeat("ab", 20) + required, observations := successfulChecks(head) + identity := &required[0] + want := ErrInvalidRequiredCheck + if location == "observation" { + identity = &observations[0].Check + want = ErrInvalidObservation + } + if field == "source" { + identity.Source = value + } else { + identity.Job = value + } + if err := VerifyChecks(head, required, observations); !errors.Is(err, want) { + t.Fatalf("%s %s: got %v, want %v", location, field, err, want) + } + } + } + }) + } +} + +func TestVerifyChecksDoesNotNormalizeIdentities(t *testing.T) { + t.Parallel() + head := strings.Repeat("ab", 20) + for _, pair := range [][2]string{ + {"unit", "Unit"}, + {"unit", " unit"}, + {"unit", "unit "}, + {"caf\u00e9", "cafe\u0301"}, + } { + for _, field := range []string{"source", "job"} { + required := []CheckIdentity{{Source: "workflow", Job: "unit"}} + observation := CheckObservation{Check: required[0], HeadSHA: head, Status: "completed", Conclusion: "success"} + if field == "source" { + required[0].Source = pair[0] + observation.Check.Source = pair[1] + } else { + required[0].Job = pair[0] + observation.Check.Job = pair[1] + } + if err := VerifyChecks(head, required, []CheckObservation{observation}); !errors.Is(err, ErrUnexpectedCheck) { + t.Fatalf("%s identity was normalized: got %v, want %v", field, err, ErrUnexpectedCheck) + } + // Meaningful spaces and Unicode are legal when their exact bytes + // agree; validation must not silently rewrite a native job name. + required[0] = observation.Check + if err := VerifyChecks(head, required, []CheckObservation{observation}); err != nil { + t.Fatalf("exact %s identity rejected: %v", field, err) + } + } + } +} + +func TestVerifyChecksPreservesIdentityFieldBoundaries(t *testing.T) { + t.Parallel() + head := strings.Repeat("ab", 20) + required := []CheckIdentity{{Source: "workflow/a", Job: "b"}, {Source: "workflow", Job: "a/b"}} + observations := []CheckObservation{ + {Check: required[0], HeadSHA: head, Status: "completed", Conclusion: "success"}, + {Check: required[1], HeadSHA: head, Status: "completed", Conclusion: "success"}, + } + if err := VerifyChecks(head, required, observations); err != nil { + t.Fatalf("distinct structured identities collided: %v", err) + } +} + +func TestVerifyChecksErrorsNeverEchoObservationValues(t *testing.T) { + t.Parallel() + head := strings.Repeat("ab", 20) + required, observations := successfulChecks(head) + const untrustedValue = "untrusted-response-marker" + observations[0].HeadSHA = untrustedValue + observations[0].Status = untrustedValue + observations[0].Conclusion = untrustedValue + observations[0].Check.Source = untrustedValue + observations[0].Check.Job = untrustedValue + err := VerifyChecks(head, required, observations) + if !errors.Is(err, ErrHeadMismatch) { + t.Fatalf("got %v, want %v", err, ErrHeadMismatch) + } + if strings.Contains(err.Error(), untrustedValue) { + t.Fatal("error exposed raw observation data") + } +} + +func successfulChecks(head string) ([]CheckIdentity, []CheckObservation) { + goWorkflow := ".github/workflows/go.yml@" + strings.Repeat("cd", 20) + securityWorkflow := ".github/workflows/security.yml@" + strings.Repeat("ef", 20) + required := []CheckIdentity{ + {Source: goWorkflow, Job: "unit (linux, go1.26.8)"}, + {Source: goWorkflow, Job: "race (linux, go1.26.8)"}, + {Source: securityWorkflow, Job: "unit (linux, go1.26.8)"}, + } + observations := make([]CheckObservation, len(required)) + for index, check := range required { + observations[index] = CheckObservation{Check: check, HeadSHA: head, Status: "completed", Conclusion: "success"} + } + return required, observations +} diff --git a/pkg/safepush/doc.go b/pkg/safepush/doc.go new file mode 100644 index 0000000..2e4cfc8 --- /dev/null +++ b/pkg/safepush/doc.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +// Package safepush provides deterministic checks over CI observations. +// +// VerifyChecks is a pure consistency check, not an enforcement boundary or a +// complete authorization decision. Its inputs cannot prove their own origin, +// freshness, or completeness. In particular, a source identity, caller assertion, +// or signature alone does not establish that a workflow actually ran. +// +// The native adapter must read the trusted required-job policy and complete, +// paginated run/job records from the owning CI service. It must verify the +// immutable workflow source and execution bindings before constructing source +// identities, and resolve run attempts without choosing an older green result. +// It must include every job in that scope, except exclusions explicitly permitted +// by the trusted policy. Branch enforcement, review approval, bypass authority, +// and revalidation when the candidate or target changes remain outside this +// helper. +package safepush diff --git a/pkg/safepush/github.go b/pkg/safepush/github.go new file mode 100644 index 0000000..c039cdd --- /dev/null +++ b/pkg/safepush/github.go @@ -0,0 +1,748 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "context" + "crypto/sha1" // #nosec G505 -- Git object identifiers, not signatures. + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "mime" + "net/http" + "net/url" + "reflect" + "strconv" + "strings" + "time" + + "github.com/opencloudtech/CloudRING/internal/strictjson" +) + +// Fixed error categories never expose credentials, remote response bodies, +// repository contents, URLs, or candidate-controlled names. +var ( + ErrGitHubPending = errors.New("safepush: required GitHub runs are pending") + ErrGitHubAPI = errors.New("safepush: GitHub API read failed") + ErrGitHubCandidate = errors.New("safepush: invalid GitHub candidate") + ErrGitHubState = errors.New("safepush: GitHub candidate or target state changed") + ErrGitHubSource = errors.New("safepush: GitHub workflow source is not trusted") + ErrGitHubRun = errors.New("safepush: GitHub run provenance is invalid") +) + +const ( + githubAPIOrigin = "https://api.github.com" + githubActionsID = int64(15368) + githubPageSize = 100 + githubListLimit = 1000 +) + +type githubAPI struct { + client *http.Client + bearerValue string + repo string +} + +type githubRepository struct { + ID int64 `json:"id"` + FullName string `json:"full_name"` + Archived bool `json:"archived"` + Disabled bool `json:"disabled"` +} + +type githubRef struct { + Ref string `json:"ref"` + SHA string `json:"sha"` + Repo githubRepository `json:"repo"` +} + +type githubPullRequest struct { + ID int64 `json:"id"` + Number int `json:"number"` + State string `json:"state"` + Draft bool `json:"draft"` + Merged bool `json:"merged"` + Head githubRef `json:"head"` + Base githubRef `json:"base"` +} + +type githubBranch struct { + Name string `json:"name"` + Protected bool `json:"protected"` + Commit struct { + SHA string `json:"sha"` + } `json:"commit"` +} + +type githubTreeEntry struct { + Path string `json:"path"` + Mode string `json:"mode"` + Type string `json:"type"` + SHA string `json:"sha"` +} + +type githubTree struct { + SHA string `json:"sha"` + Truncated *bool `json:"truncated"` + Tree []githubTreeEntry `json:"tree"` +} + +type githubWorkflow struct { + ID int64 `json:"id"` + Path string `json:"path"` + State string `json:"state"` +} + +type githubRun struct { + ID int64 `json:"id"` + WorkflowID int64 `json:"workflow_id"` + RunNumber int64 `json:"run_number"` + RunAttempt int `json:"run_attempt"` + CheckSuiteID int64 `json:"check_suite_id"` + Path string `json:"path"` + Event string `json:"event"` + HeadSHA string `json:"head_sha"` + HeadBranch string `json:"head_branch"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + Repository githubRepository `json:"repository"` + HeadRepository githubRepository `json:"head_repository"` + ReferencedWorkflows []json.RawMessage `json:"referenced_workflows"` + PullRequests []githubRunPR `json:"pull_requests"` +} + +type githubRunPR struct { + ID int64 `json:"id"` + Number int `json:"number"` + URL string `json:"url"` + Head githubRef `json:"head"` + Base githubRef `json:"base"` +} + +type githubStep struct { + Name string `json:"name"` + Number int `json:"number"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` +} + +type githubJob struct { + ID int64 `json:"id"` + RunID int64 `json:"run_id"` + RunAttempt int `json:"run_attempt"` + HeadSHA string `json:"head_sha"` + HeadBranch string `json:"head_branch"` + Name string `json:"name"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + CheckRunURL string `json:"check_run_url"` + Steps []githubStep `json:"steps"` +} + +type githubCheckRun struct { + ID int64 `json:"id"` + Name string `json:"name"` + HeadSHA string `json:"head_sha"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + CheckSuite struct { + ID int64 `json:"id"` + } `json:"check_suite"` + App struct { + ID int64 `json:"id"` + Slug string `json:"slug"` + } `json:"app"` +} + +// ReadGitHubPolicy reads the accepted base's regular policy blob without a +// checkout or contents-API symlink dereference. VerifyGitHub subsequently proves +// that this base is still the current protected target. The caller must obtain +// repository/baseSHA from its trusted workflow event, not candidate output. +func ReadGitHubPolicy(ctx context.Context, client *http.Client, token, repository, baseSHA string) (GitHubPolicy, error) { + if !repositoryName.MatchString(repository) || strings.HasSuffix(repository, "/.") || + strings.HasSuffix(repository, "/..") || !validHeadSHA(baseSHA) { + return GitHubPolicy{}, ErrPolicy + } + api, err := newGitHubAPI(client, token, repository) + if err != nil { + return GitHubPolicy{}, err + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + var repo githubRepository + if err = api.get(ctx, "", nil, &repo); err != nil { + return GitHubPolicy{}, err + } + if repo.ID <= 0 || repo.FullName != repository || repo.Archived || repo.Disabled { + return GitHubPolicy{}, ErrGitHubState + } + tree, err := api.readTree(ctx, baseSHA) + if err != nil { + return GitHubPolicy{}, err + } + entry, exists := tree[".github/safepush.json"] + if !exists || entry.Type != "blob" || entry.Mode != "100644" { + return GitHubPolicy{}, ErrGitHubSource + } + contents, err := api.readGitBlob(ctx, entry, strictjson.MaxDocumentBytes) + if err != nil { + return GitHubPolicy{}, err + } + policy, err := DecodeGitHubPolicy(contents) + if err != nil { + return GitHubPolicy{}, ErrPolicy + } + if policy.Repository != repo.FullName || policy.RepositoryID != repo.ID { + return GitHubPolicy{}, ErrGitHubSource + } + if ctx.Err() != nil { + return GitHubPolicy{}, ErrGitHubAPI + } + return policy, nil +} + +// VerifyGitHub verifies native PR/run/job provenance using the accepted policy. +// It executes no candidate code. Both required and excluded workflow files, and +// .github/scripts, must match the accepted base. This deliberately does not +// authenticate generated dynamic workflows or reusable workflow chains. +// +// Base must be an ancestor of head. With unchanged workflow sources this avoids +// claiming a stale PR test-merge tested newer candidate bytes. Native required +// reviews and branch rules remain necessary: these API reads are not an atomic +// ref-update transaction and do not prove the semantic adequacy of tests. +func VerifyGitHub(ctx context.Context, client *http.Client, token string, policy GitHubPolicy, candidate GitHubCandidate) error { + if err := ValidateGitHubPolicy(policy); err != nil { + return err + } + if ValidateGitHubCandidate(policy.Repository, candidate) != nil { + return ErrGitHubCandidate + } + api, err := newGitHubAPI(client, token, policy.Repository) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + var repo githubRepository + if err = api.get(ctx, "", nil, &repo); err != nil { + return err + } + if repo.ID != policy.RepositoryID || repo.FullName != policy.Repository || repo.Archived || repo.Disabled { + return ErrGitHubState + } + pr, err := api.readCandidate(ctx, policy, candidate) + if err != nil { + return err + } + if err = api.requireBaseAncestor(ctx, candidate); err != nil { + return err + } + baseTree, err := api.readTree(ctx, candidate.BaseSHA) + if err != nil { + return err + } + headTree, err := api.readTree(ctx, candidate.HeadSHA) + if err != nil { + return err + } + if err = verifyGitHubSources(policy, baseTree, headTree); err != nil { + return err + } + for _, workflow := range policy.Workflows { + contents, readErr := api.readGitBlob(ctx, baseTree[workflow.Path], maxWorkflowSourceBytes) + if readErr != nil { + return readErr + } + if err = verifyWorkflowSource(contents); err != nil { + return err + } + } + workflows, err := api.readWorkflows(ctx, policy) + if err != nil { + return err + } + runs, err := api.selectRuns(ctx, policy, pr, workflows) + if err != nil { + return err + } + var required []CheckIdentity + var observations []CheckObservation + for _, workflow := range policy.Workflows { + source := workflow.Path + "@" + baseTree[workflow.Path].SHA + for _, job := range workflow.Jobs { + required = append(required, CheckIdentity{Source: source, Job: job}) + } + items, collectErr := api.readObservations(ctx, runs[workflow.Path], source) + if collectErr != nil { + return collectErr + } + observations = append(observations, items...) + } + if err = VerifyChecks(candidate.HeadSHA, required, observations); err != nil { + return err + } + finalWorkflows, err := api.readWorkflows(ctx, policy) + if err != nil { + return err + } + if !reflect.DeepEqual(workflows, finalWorkflows) { + return ErrGitHubState + } + finalRuns, err := api.selectRuns(ctx, policy, pr, finalWorkflows) + if err != nil { + // A newly selected run or attempt invalidates this collection, even if + // it is pending. A new admission attempt must collect a new snapshot. + if errors.Is(err, ErrGitHubPending) { + return ErrGitHubState + } + return err + } + if !reflect.DeepEqual(runs, finalRuns) { + return ErrGitHubState + } + for _, workflow := range policy.Workflows { + run := runs[workflow.Path] + var finalRun githubRun + if err = api.get(ctx, "/actions/runs/"+strconv.FormatInt(run.ID, 10), nil, &finalRun); err != nil { + return err + } + if !reflect.DeepEqual(run, finalRun) { + return ErrGitHubState + } + } + finalPR, err := api.readCandidate(ctx, policy, candidate) + if err != nil { + return err + } + if !reflect.DeepEqual(pr, finalPR) { + return ErrGitHubState + } + if ctx.Err() != nil { + return ErrGitHubAPI + } + return nil +} + +func newGitHubAPI(client *http.Client, token, repository string) (*githubAPI, error) { + if token == "" { + return nil, ErrGitHubAPI + } + for _, value := range token { + if value <= 0x20 || value >= 0x7f { + return nil, ErrGitHubAPI + } + } + if client == nil { + client = http.DefaultClient + } + copyClient := *client + copyClient.Jar = nil + copyClient.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse } + if copyClient.Timeout == 0 || copyClient.Timeout > 30*time.Second { + copyClient.Timeout = 30 * time.Second + } + return &githubAPI{client: ©Client, bearerValue: token, repo: repository}, nil +} + +func (api *githubAPI) get(ctx context.Context, suffix string, query url.Values, destination any) error { + u := url.URL{Scheme: "https", Host: "api.github.com", Path: "/repos/" + api.repo + suffix, RawQuery: query.Encode()} + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return ErrGitHubAPI + } + req.Header.Set("Authorization", "Bearer "+api.bearerValue) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + req.Header.Set("User-Agent", "CloudRING-SafePush") + resp, err := api.client.Do(req) + if err != nil { + return ErrGitHubAPI + } + defer resp.Body.Close() + mediaType, _, mediaErr := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if resp.StatusCode != http.StatusOK || mediaErr != nil || + (mediaType != "application/json" && mediaType != "application/vnd.github+json") || + resp.ContentLength > strictjson.MaxDocumentBytes { + return ErrGitHubAPI + } + data, err := strictjson.Read(resp.Body) + if err != nil || strictjson.Decode(data, destination) != nil { + return ErrGitHubAPI + } + return nil +} + +func githubList[T any](ctx context.Context, api *githubAPI, suffix, key string, filters url.Values) ([]T, error) { + var result []T + declared := -1 + for page := 1; page <= githubListLimit/githubPageSize; page++ { + query := make(url.Values) + for name, values := range filters { + query[name] = append([]string(nil), values...) + } + query.Set("per_page", strconv.Itoa(githubPageSize)) + query.Set("page", strconv.Itoa(page)) + var envelope map[string]json.RawMessage + if err := api.get(ctx, suffix, query, &envelope); err != nil { + return nil, err + } + var total int + if raw := envelope["total_count"]; len(raw) == 0 || string(raw) == "null" || json.Unmarshal(raw, &total) != nil || total < 0 || total > githubListLimit { + return nil, ErrGitHubAPI + } + for _, flag := range []string{"truncated", "incomplete_results"} { + if raw, found := envelope[flag]; found && string(raw) != "false" { + return nil, ErrGitHubAPI + } + } + if declared != -1 && declared != total { + return nil, ErrGitHubState + } + declared = total + var items []T + if json.Unmarshal(envelope[key], &items) != nil || items == nil { + return nil, ErrGitHubAPI + } + if len(items) != min(githubPageSize, total-len(result)) { + return nil, ErrGitHubAPI + } + result = append(result, items...) + if len(result) == total { + return result, nil + } + } + return nil, ErrGitHubAPI +} + +func (api *githubAPI) readCandidate(ctx context.Context, policy GitHubPolicy, candidate GitHubCandidate) (githubPullRequest, error) { + var pr githubPullRequest + if err := api.get(ctx, "/pulls/"+strconv.Itoa(candidate.PullRequest), nil, &pr); err != nil { + return pr, err + } + if pr.ID <= 0 || pr.Number != candidate.PullRequest || pr.State != "open" || pr.Draft || pr.Merged || + pr.Head.SHA != candidate.HeadSHA || !validJobName(pr.Head.Ref) || pr.Head.Repo.ID <= 0 || + !repositoryName.MatchString(pr.Head.Repo.FullName) || + pr.Base.SHA != candidate.BaseSHA || pr.Base.Ref != candidate.TargetBranch || + pr.Base.Repo.ID != policy.RepositoryID || pr.Base.Repo.FullName != policy.Repository { + return pr, ErrGitHubState + } + var branch githubBranch + if err := api.get(ctx, "/branches/"+candidate.TargetBranch, nil, &branch); err != nil { + return pr, err + } + if branch.Name != candidate.TargetBranch || branch.Commit.SHA != candidate.BaseSHA || !branch.Protected { + return pr, ErrGitHubState + } + return pr, nil +} + +func (api *githubAPI) requireBaseAncestor(ctx context.Context, candidate GitHubCandidate) error { + var comparison struct { + Status string `json:"status"` + AheadBy *int `json:"ahead_by"` + BehindBy *int `json:"behind_by"` + Base struct { + SHA string `json:"sha"` + } `json:"base_commit"` + MergeBase struct { + SHA string `json:"sha"` + } `json:"merge_base_commit"` + } + if err := api.get(ctx, "/compare/"+candidate.BaseSHA+"..."+candidate.HeadSHA, url.Values{"per_page": {"1"}}, &comparison); err != nil { + return err + } + if comparison.Status != "ahead" || comparison.AheadBy == nil || *comparison.AheadBy <= 0 || + comparison.BehindBy == nil || *comparison.BehindBy != 0 || + comparison.Base.SHA != candidate.BaseSHA || comparison.MergeBase.SHA != candidate.BaseSHA { + return ErrGitHubState + } + return nil +} + +func (api *githubAPI) readTree(ctx context.Context, sha string) (map[string]githubTreeEntry, error) { + var commit struct { + SHA string `json:"sha"` + Tree struct { + SHA string `json:"sha"` + } `json:"tree"` + } + if err := api.get(ctx, "/git/commits/"+sha, nil, &commit); err != nil { + return nil, err + } + if commit.SHA != sha || !validHeadSHA(commit.Tree.SHA) { + return nil, ErrGitHubSource + } + var response githubTree + if err := api.get(ctx, "/git/trees/"+commit.Tree.SHA, url.Values{"recursive": {"1"}}, &response); err != nil { + return nil, err + } + if response.SHA != commit.Tree.SHA || response.Truncated == nil || *response.Truncated || response.Tree == nil { + return nil, ErrGitHubSource + } + entries := make(map[string]githubTreeEntry, len(response.Tree)) + for _, entry := range response.Tree { + if !validJobName(entry.Path) || !validHeadSHA(entry.SHA) { + return nil, ErrGitHubSource + } + if _, duplicate := entries[entry.Path]; duplicate { + return nil, ErrGitHubSource + } + entries[entry.Path] = entry + } + return entries, nil +} + +func (api *githubAPI) readGitBlob(ctx context.Context, entry githubTreeEntry, limit int) ([]byte, error) { + if entry.Type != "blob" || (entry.Mode != "100644" && entry.Mode != "100755") || + !validHeadSHA(entry.SHA) || limit <= 0 || limit > strictjson.MaxDocumentBytes { + return nil, ErrGitHubSource + } + var blob struct { + SHA string `json:"sha"` + Encoding string `json:"encoding"` + Size *int `json:"size"` + Content string `json:"content"` + } + if err := api.get(ctx, "/git/blobs/"+entry.SHA, nil, &blob); err != nil { + return nil, err + } + if blob.SHA != entry.SHA || blob.Encoding != "base64" || blob.Size == nil || *blob.Size <= 0 || *blob.Size > limit { + return nil, ErrGitHubSource + } + contents, err := base64.StdEncoding.DecodeString(blob.Content) + if err != nil || len(contents) != *blob.Size || gitBlobID(contents, len(entry.SHA)) != entry.SHA { + return nil, ErrGitHubSource + } + if ctx.Err() != nil { + return nil, ErrGitHubAPI + } + return contents, nil +} + +func verifyGitHubSources(policy GitHubPolicy, base, head map[string]githubTreeEntry) error { + expected := make(map[string]bool) + for _, workflow := range policy.Workflows { + expected[workflow.Path] = true + } + for _, excluded := range policy.ExcludedWorkflows { + expected[excluded] = true + } + guarded := func(entries map[string]githubTreeEntry) (map[string]githubTreeEntry, error) { + result := make(map[string]githubTreeEntry) + found := make(map[string]bool) + for name, entry := range entries { + if name == ".github/workflows" || strings.HasPrefix(name, ".github/workflows/") || + name == ".github/scripts" || strings.HasPrefix(name, ".github/scripts/") || name == ".github/workflow-policy.json" { + if (entry.Type != "blob" || (entry.Mode != "100644" && entry.Mode != "100755")) && + (entry.Type != "tree" || entry.Mode != "040000") { + return nil, ErrGitHubSource + } + result[name] = entry + } + if strings.HasPrefix(name, ".github/workflows/") && entry.Type != "tree" && + (strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml")) { + if !validWorkflowPath(name) || !expected[name] { + return nil, ErrGitHubSource + } + found[name] = true + } + } + if len(found) != len(expected) { + return nil, ErrGitHubSource + } + return result, nil + } + trusted, err := guarded(base) + if err != nil { + return err + } + candidate, err := guarded(head) + if err != nil { + return err + } + if !reflect.DeepEqual(trusted, candidate) { + return ErrGitHubSource + } + return nil +} + +func (api *githubAPI) readWorkflows(ctx context.Context, policy GitHubPolicy) (map[string]githubWorkflow, error) { + items, err := githubList[githubWorkflow](ctx, api, "/actions/workflows", "workflows", nil) + if err != nil { + return nil, err + } + expected := make(map[string]bool) + for _, workflow := range policy.Workflows { + expected[workflow.Path] = true + } + for _, excluded := range policy.ExcludedWorkflows { + expected[excluded] = false + } + result := make(map[string]githubWorkflow) + ids := make(map[int64]bool) + for _, item := range items { + if item.ID <= 0 || ids[item.ID] { + return nil, ErrGitHubSource + } + ids[item.ID] = true + if strings.HasPrefix(item.Path, "dynamic/") { + continue // Native generated scanning remains a separate provider rule. + } + required, exists := expected[item.Path] + if !exists { + return nil, ErrGitHubSource + } + if _, duplicate := result[item.Path]; duplicate || (required && item.State != "active") { + return nil, ErrGitHubSource + } + result[item.Path] = item + } + if len(result) != len(expected) { + return nil, ErrGitHubSource + } + return result, nil +} + +func (api *githubAPI) selectRuns(ctx context.Context, policy GitHubPolicy, pr githubPullRequest, workflows map[string]githubWorkflow) (map[string]githubRun, error) { + items, err := githubList[githubRun](ctx, api, "/actions/runs", "workflow_runs", url.Values{"event": {"pull_request"}, "head_sha": {pr.Head.SHA}}) + if err != nil { + return nil, err + } + result := make(map[string]githubRun) + ids := make(map[int64]bool) + requiredPaths := make(map[string]bool) + requiredIDs := make(map[int64]bool) + for _, workflow := range policy.Workflows { + requiredPaths[workflow.Path] = true + requiredIDs[workflows[workflow.Path].ID] = true + } + for _, item := range items { + // Excluded release/verifier runs and provider-injected workflows cannot + // satisfy a test identity. In particular do not wait on this gate itself. + if !requiredPaths[item.Path] && !requiredIDs[item.WorkflowID] { + continue + } + workflow, exists := workflows[item.Path] + if !exists || item.ID <= 0 || ids[item.ID] || item.RunNumber <= 0 || item.RunAttempt <= 0 || item.CheckSuiteID <= 0 || + item.WorkflowID != workflow.ID || item.Event != "pull_request" || + item.HeadSHA != pr.Head.SHA || item.HeadBranch != pr.Head.Ref || + item.Repository.ID != policy.RepositoryID || item.Repository.FullName != policy.Repository || + item.HeadRepository.ID != pr.Head.Repo.ID || item.HeadRepository.FullName != pr.Head.Repo.FullName || + len(item.ReferencedWorkflows) != 0 || !api.runMatchesPR(item, pr) { + return nil, ErrGitHubRun + } + ids[item.ID] = true + prior, exists := result[item.Path] + if exists && item.RunNumber == prior.RunNumber { + return nil, ErrGitHubRun + } + if !exists || item.RunNumber > prior.RunNumber { + result[item.Path] = item + } + } + pending := false + for _, workflow := range policy.Workflows { + run, exists := result[workflow.Path] + if !exists { + pending = true + continue + } + switch run.Status { + case "completed": + if run.Conclusion != "success" { + return nil, ErrCheckNotSuccessful + } + case "queued", "in_progress", "waiting", "requested", "pending": + pending = true + default: + return nil, ErrGitHubRun + } + } + if pending { + return nil, ErrGitHubPending + } + return result, nil +} + +func (api *githubAPI) runMatchesPR(run githubRun, pr githubPullRequest) bool { + // A head SHA can be reused by another PR or after retargeting. Do not treat + // the run's head alone as an association with this exact PR/base snapshot. + // Empty associations occur for closed or otherwise unavailable PRs; those + // cannot provide the provenance needed for this open-PR admission. + if len(run.PullRequests) != 1 { + return false + } + linked := run.PullRequests[0] + return linked.ID == pr.ID && linked.Number == pr.Number && + linked.URL == githubAPIOrigin+"/repos/"+api.repo+"/pulls/"+strconv.Itoa(pr.Number) && + linked.Head.SHA == pr.Head.SHA && linked.Head.Ref == pr.Head.Ref && linked.Head.Repo.ID == pr.Head.Repo.ID && + linked.Base.SHA == pr.Base.SHA && linked.Base.Ref == pr.Base.Ref && linked.Base.Repo.ID == pr.Base.Repo.ID +} + +func (api *githubAPI) readObservations(ctx context.Context, run githubRun, source string) ([]CheckObservation, error) { + suffix := fmt.Sprintf("/actions/runs/%d/attempts/%d/jobs", run.ID, run.RunAttempt) + jobs, err := githubList[githubJob](ctx, api, suffix, "jobs", nil) + if err != nil { + return nil, err + } + checks, err := githubList[githubCheckRun](ctx, api, fmt.Sprintf("/check-suites/%d/check-runs", run.CheckSuiteID), "check_runs", url.Values{"filter": {"latest"}}) + if err != nil { + return nil, err + } + checkByID := make(map[int64]githubCheckRun) + for _, check := range checks { + if check.ID <= 0 || checkByID[check.ID].ID != 0 || check.CheckSuite.ID != run.CheckSuiteID || + check.App.ID != githubActionsID || check.App.Slug != "github-actions" || check.HeadSHA != run.HeadSHA || + check.Status != "completed" || check.Conclusion != "success" { + return nil, ErrGitHubRun + } + checkByID[check.ID] = check + } + var observations []CheckObservation + jobIDs := make(map[int64]bool) + usedChecks := make(map[int64]bool) + for _, job := range jobs { + if job.ID <= 0 || jobIDs[job.ID] || job.RunID != run.ID || job.RunAttempt != run.RunAttempt || + job.HeadSHA != run.HeadSHA || job.HeadBranch != run.HeadBranch || len(job.Steps) == 0 { + return nil, ErrGitHubRun + } + jobIDs[job.ID] = true + prefix := githubAPIOrigin + "/repos/" + api.repo + "/check-runs/" + rawID, found := strings.CutPrefix(job.CheckRunURL, prefix) + checkID, parseErr := strconv.ParseInt(rawID, 10, 64) + if !found || parseErr != nil || checkID <= 0 || rawID != strconv.FormatInt(checkID, 10) || usedChecks[checkID] { + return nil, ErrGitHubRun + } + check, exists := checkByID[checkID] + if !exists || check.Name != job.Name { + return nil, ErrGitHubRun + } + usedChecks[checkID] = true + stepNumbers := make(map[int]bool) + for _, step := range job.Steps { + if step.Number <= 0 || stepNumbers[step.Number] || !validJobName(step.Name) || step.Status != "completed" || step.Conclusion != "success" { + return nil, ErrGitHubRun + } + stepNumbers[step.Number] = true + } + observations = append(observations, CheckObservation{Check: CheckIdentity{Source: source, Job: job.Name}, HeadSHA: job.HeadSHA, Status: job.Status, Conclusion: job.Conclusion}) + } + if len(usedChecks) != len(checkByID) { + return nil, ErrGitHubRun + } + return observations, nil +} + +func gitBlobID(contents []byte, length int) string { + object := append([]byte(fmt.Sprintf("blob %d\x00", len(contents))), contents...) + if length == 40 { + digest := sha1.Sum(object) // #nosec G401 -- required Git object identifier. + return hex.EncodeToString(digest[:]) + } + digest := sha256.Sum256(object) + return hex.EncodeToString(digest[:]) +} diff --git a/pkg/safepush/github_test.go b/pkg/safepush/github_test.go new file mode 100644 index 0000000..6b5d310 --- /dev/null +++ b/pkg/safepush/github_test.go @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "context" + "crypto/sha1" // #nosec G505 -- synthetic Git object fixture identifiers. + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "strings" + "sync" + "testing" + + "github.com/opencloudtech/CloudRING/internal/strictjson" +) + +const ( + ghTestHead = "1111111111111111111111111111111111111111" + ghTestBase = "2222222222222222222222222222222222222222" + ghTestHeadTree = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + ghTestBaseTree = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ghTestBlob = "e69e9a60e26d1cd37de67076728528819fec5844" + ghTestOther = "9999999999999999999999999999999999999999" + ghTestPolicyID = "85b9611e54998ddb6ae044af1c6622072c9fa29a" + ghTestPolicy = `{"schemaVersion":"cloudring.safepush.github/v1","repository":"owner/repo","repositoryId":42,"workflows":[{"path":".github/workflows/ci.yml","jobs":["unit","race"]}],"excludedWorkflows":[".github/workflows/release.yml"]}` + ghTestWorkflow = `name: CI +on: pull_request +jobs: + unit: + runs-on: ubuntu-latest + steps: + - name: Run tests + run: go test ./... + race: + runs-on: ubuntu-latest + steps: + - name: Run tests + run: go test -race ./... +` +) + +// These are HTTP JSON fixtures shaped from the provider API, with deliberately +// distinct commit, tree and blob IDs. The adapter always addresses the official +// HTTPS origin; only this test transport connects it to an isolated TLS server. +type githubHTTPSFixture struct { + t *testing.T + client *http.Client + responses map[string]any + mu sync.Mutex + counts map[string]int + requests []string + mutate func(*http.Request, int, map[string]any) + reply func(http.ResponseWriter, *http.Request, int) bool + policy GitHubPolicy + candidate GitHubCandidate +} + +type githubFixtureTransport struct { + t *testing.T + base http.RoundTripper + target *url.URL +} + +func (transport githubFixtureTransport) RoundTrip(request *http.Request) (*http.Response, error) { + if request.URL.Scheme != "https" || request.URL.Host != "api.github.com" || request.URL.User != nil { + transport.t.Error("adapter attempted a non-official API origin") + return nil, errors.New("unexpected test origin") + } + copyRequest := request.Clone(request.Context()) + copyURL := *request.URL + copyURL.Host = transport.target.Host + copyRequest.URL = ©URL + copyRequest.Host = transport.target.Host + response, err := transport.base.RoundTrip(copyRequest) + if response != nil { + response.Request = request + } + return response, err +} + +func newGitHubHTTPSFixture(t *testing.T) *githubHTTPSFixture { + t.Helper() + fixture := &githubHTTPSFixture{t: t, responses: make(map[string]any), counts: make(map[string]int)} + if err := json.Unmarshal([]byte(ghTestPolicy), &fixture.policy); err != nil { + t.Fatal(err) + } + fixture.candidate = GitHubCandidate{PullRequest: 7, HeadSHA: ghTestHead, BaseSHA: ghTestBase, TargetBranch: "main"} + repo := map[string]any{"id": 42, "full_name": "owner/repo", "archived": false, "disabled": false} + fixture.responses[""] = repo + fixture.responses["/pulls/7"] = map[string]any{ + "id": 71, "number": 7, "state": "open", "draft": false, "merged": false, + "head": map[string]any{"ref": "feature", "sha": ghTestHead, "repo": repo}, + "base": map[string]any{"ref": "main", "sha": ghTestBase, "repo": repo}, + } + fixture.responses["/branches/main"] = map[string]any{"name": "main", "protected": true, "commit": map[string]any{"sha": ghTestBase}} + fixture.responses["/compare/"+ghTestBase+"..."+ghTestHead] = map[string]any{ + "status": "ahead", "ahead_by": 1, "behind_by": 0, + "base_commit": map[string]any{"sha": ghTestBase}, "merge_base_commit": map[string]any{"sha": ghTestBase}, + } + for commit, tree := range map[string]string{ghTestBase: ghTestBaseTree, ghTestHead: ghTestHeadTree} { + fixture.responses["/git/commits/"+commit] = map[string]any{"sha": commit, "tree": map[string]any{"sha": tree}} + fixture.responses["/git/trees/"+tree] = map[string]any{"sha": tree, "truncated": false, "tree": []any{ + map[string]any{"path": ".github/workflows/ci.yml", "mode": "100644", "type": "blob", "sha": ghTestBlob}, + map[string]any{"path": ".github/workflows/release.yml", "mode": "100644", "type": "blob", "sha": "4444444444444444444444444444444444444444"}, + map[string]any{"path": ".github/scripts/guard.sh", "mode": "100755", "type": "blob", "sha": "5555555555555555555555555555555555555555"}, + map[string]any{"path": ".github/safepush.json", "mode": "100644", "type": "blob", "sha": ghTestPolicyID}, + }} + } + fixture.responses["/git/blobs/"+ghTestPolicyID] = map[string]any{"sha": ghTestPolicyID, "encoding": "base64", "size": len(ghTestPolicy), "content": base64.StdEncoding.EncodeToString([]byte(ghTestPolicy))} + fixture.responses["/git/blobs/"+ghTestBlob] = map[string]any{"sha": ghTestBlob, "encoding": "base64", "size": 232, "content": base64.StdEncoding.EncodeToString([]byte(ghTestWorkflow))} + fixture.responses["/actions/workflows"] = ghTestEnvelope("workflows", []any{ + map[string]any{"id": 101, "path": ".github/workflows/ci.yml", "state": "active"}, + map[string]any{"id": 102, "path": ".github/workflows/release.yml", "state": "active"}, + map[string]any{"id": 103, "path": "dynamic/github-code-scanning/codeql", "state": "active"}, + }) + run := map[string]any{ + "id": 201, "workflow_id": 101, "run_number": 11, "run_attempt": 1, "check_suite_id": 301, + "path": ".github/workflows/ci.yml", "event": "pull_request", "head_sha": ghTestHead, "head_branch": "feature", + "status": "completed", "conclusion": "success", "repository": repo, "head_repository": repo, + "pull_requests": []any{map[string]any{ + "id": 71, "number": 7, "url": "https://api.github.com/repos/owner/repo/pulls/7", + "head": map[string]any{"ref": "feature", "sha": ghTestHead, "repo": map[string]any{"id": 42}}, + "base": map[string]any{"ref": "main", "sha": ghTestBase, "repo": map[string]any{"id": 42}}, + }}, "referenced_workflows": []any{}, + } + fixture.responses["/actions/runs"] = ghTestEnvelope("workflow_runs", []any{run}) + fixture.responses["/actions/runs/201"] = run + fixture.setJobs([]string{"unit", "race"}) + server := httptest.NewTLSServer(http.HandlerFunc(fixture.serveHTTP)) + t.Cleanup(server.Close) + target, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + fixture.client = &http.Client{Transport: githubFixtureTransport{t: t, base: server.Client().Transport, target: target}} + return fixture +} + +func ghTestEnvelope(key string, items []any) map[string]any { + return map[string]any{"total_count": len(items), key: items} +} + +func (fixture *githubHTTPSFixture) setJobs(names []string) { + jobs, checks := make([]any, 0, len(names)), make([]any, 0, len(names)) + for index, name := range names { + id := 401 + index + jobs = append(jobs, map[string]any{ + "id": id, "run_id": 201, "run_attempt": 1, "head_sha": ghTestHead, "head_branch": "feature", "name": name, + "status": "completed", "conclusion": "success", "check_run_url": fmt.Sprintf("https://api.github.com/repos/owner/repo/check-runs/%d", id), + "steps": []any{ + map[string]any{"name": "Set up job", "number": 1, "status": "completed", "conclusion": "success"}, + map[string]any{"name": "Run tests", "number": 3, "status": "completed", "conclusion": "success"}, + map[string]any{"name": "Complete job", "number": 7, "status": "completed", "conclusion": "success"}, + }, + }) + checks = append(checks, map[string]any{ + "id": id, "name": name, "head_sha": ghTestHead, "status": "completed", "conclusion": "success", + "check_suite": map[string]any{"id": 301}, "app": map[string]any{"id": 15368, "slug": "github-actions"}, + }) + } + fixture.responses["/actions/runs/201/attempts/1/jobs"] = ghTestEnvelope("jobs", jobs) + fixture.responses["/check-suites/301/check-runs"] = ghTestEnvelope("check_runs", checks) +} + +func (fixture *githubHTTPSFixture) setRequiredWorkflow(contents string) { + // The baseline uses a separately computed known Git blob hash. Mutations + // produce another native-shaped object so the parser, not a hash mismatch, + // rejects a workflow whose failed outcomes could have success conclusions. + digest := sha1.New() // #nosec G401 -- synthetic Git object identifier. + _, _ = fmt.Fprintf(digest, "blob %d%c%s", len(contents), 0, contents) + objectID := fmt.Sprintf("%x", digest.Sum(nil)) + for _, treeID := range []string{ghTestBaseTree, ghTestHeadTree} { + tree := fixture.responses["/git/trees/"+treeID].(map[string]any) + ghTestFirst(tree, "tree")["sha"] = objectID + } + fixture.responses["/git/blobs/"+objectID] = map[string]any{ + "sha": objectID, "encoding": "base64", "size": len(contents), + "content": base64.StdEncoding.EncodeToString([]byte(contents)), + } +} + +func (fixture *githubHTTPSFixture) serveHTTP(writer http.ResponseWriter, request *http.Request) { + fixture.mu.Lock() + defer fixture.mu.Unlock() + suffix := strings.TrimPrefix(request.URL.Path, "/repos/owner/repo") + fixture.counts[suffix]++ + count := fixture.counts[suffix] + fixture.requests = append(fixture.requests, request.URL.RequestURI()) + if request.Method != http.MethodGet || request.Header.Get("Authorization") != "Bearer test-token" { + fixture.t.Error("unexpected method or synthetic authorization header") + } + if suffix == "/actions/runs" && (request.URL.Query().Get("event") != "pull_request" || request.URL.Query().Get("head_sha") != ghTestHead) { + fixture.t.Error("run query lacks exact event and candidate filter") + } + writer.Header().Set("Content-Type", "application/json") + if fixture.reply != nil && fixture.reply(writer, request, count) { + return + } + original, exists := fixture.responses[suffix] + if !exists { + fixture.t.Errorf("unexpected API route: %s", suffix) + writer.WriteHeader(http.StatusNotFound) + return + } + data, err := json.Marshal(original) + if err != nil { + fixture.t.Error(err) + return + } + var response map[string]any + if err = json.Unmarshal(data, &response); err != nil { + fixture.t.Error(err) + return + } + if pageText := request.URL.Query().Get("page"); pageText != "" { + page, pageErr := strconv.Atoi(pageText) + if pageErr != nil || page <= 0 || request.URL.Query().Get("per_page") != "100" { + fixture.t.Error("invalid pagination request") + writer.WriteHeader(http.StatusBadRequest) + return + } + for _, key := range []string{"workflows", "workflow_runs", "jobs", "check_runs"} { + if all, ok := response[key].([]any); ok { + start := min((page-1)*100, len(all)) + response[key] = all[start:min(start+100, len(all))] + } + } + } + if fixture.mutate != nil { + fixture.mutate(request, count, response) + } + if err = json.NewEncoder(writer).Encode(response); err != nil { + fixture.t.Error(err) + } +} + +func (fixture *githubHTTPSFixture) verify() error { + return VerifyGitHub(context.Background(), fixture.client, "test-token", fixture.policy, fixture.candidate) +} + +func ghTestFirst(response map[string]any, key string) map[string]any { + return response[key].([]any)[0].(map[string]any) +} + +func TestGitHubHTTPSAdmissionAndTrustedPolicy(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + policy, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/repo", ghTestBase) + if err != nil || !reflect.DeepEqual(policy, fixture.policy) { + t.Fatalf("policy result = %#v, %v", policy, err) + } + if err = fixture.verify(); err != nil { + t.Fatal(err) + } + fixture.mu.Lock() + defer fixture.mu.Unlock() + for _, required := range []string{"/git/commits/" + ghTestBase, "/git/trees/" + ghTestBaseTree, "/git/trees/" + ghTestHeadTree, "/git/blobs/" + ghTestBlob, "/actions/runs/201/attempts/1/jobs", "/check-suites/301/check-runs", "/actions/runs/201"} { + if fixture.counts[required] == 0 { + t.Errorf("missing native binding read: %s", required) + } + } + if fixture.counts["/pulls/7"] != 2 || fixture.counts["/branches/main"] != 2 || fixture.counts["/actions/runs"] != 2 { + t.Error("successful admission must re-read PR, protected target, and latest runs") + } +} + +func TestGitHubSuccessConclusionsCannotHideContinueOnError(t *testing.T) { + for _, scope := range []string{"job", "step"} { + for _, value := range []string{"true", "${{ false }}", "null", "'false'", "false"} { + t.Run(scope+"_"+value, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + var contents string + if scope == "job" { + contents = strings.Replace(ghTestWorkflow, " steps:\n", " continue-on-error: "+value+"\n steps:\n", 1) + } else { + contents = strings.Replace(ghTestWorkflow, " run: go test ./...\n", " run: go test ./...\n continue-on-error: "+value+"\n", 1) + } + fixture.setRequiredWorkflow(contents) + // Base and head both have this exact blob. Every REST workflow, + // job, check and step still reports completed/success, which is + // realistic for a failed step with continue-on-error: true. + err := fixture.verify() + if value == "false" { + if err != nil { + t.Fatal(err) + } + return + } + if !errors.Is(err, ErrGitHubSource) { + t.Fatalf("API success admitted unsafe accepted workflow: %v", err) + } + if fixture.counts["/actions/runs"] != 0 { + t.Fatal("unsafe source was not rejected before trusting run statuses") + } + }) + } + } +} + +func TestGitHubWorkflowBlobMustMatchTrustedObject(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if strings.HasSuffix(request.URL.Path, "/git/blobs/"+ghTestBlob) { + response["content"] = base64.StdEncoding.EncodeToString([]byte(strings.Replace(ghTestWorkflow, "unit", "evil", 1))) + } + } + if err := fixture.verify(); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("unverified workflow blob admitted: %v", err) + } +} + +func TestGitHubOptionalReferencedWorkflowsAndExcludedGate(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if strings.HasSuffix(request.URL.Path, "/actions/runs") { + delete(ghTestFirst(response, "workflow_runs"), "referenced_workflows") + response["workflow_runs"] = append(response["workflow_runs"].([]any), map[string]any{"id": 900, "workflow_id": 999, "path": "organization/required-verifier.yml", "status": "in_progress"}) + response["total_count"] = 2 + } + if strings.HasSuffix(request.URL.Path, "/actions/runs/201") { + delete(response, "referenced_workflows") + } + } + if err := fixture.verify(); err != nil { + t.Fatal(err) + } +} + +func TestGitHubPendingIsOnlyUnfinishedRequiredRuns(t *testing.T) { + for _, status := range []string{"absent", "queued", "in_progress", "waiting", "requested", "pending"} { + t.Run(status, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if !strings.HasSuffix(request.URL.Path, "/actions/runs") { + return + } + if status == "absent" { + response["total_count"], response["workflow_runs"] = 0, []any{} + } else { + run := ghTestFirst(response, "workflow_runs") + run["status"], run["conclusion"] = status, nil + } + } + if err := fixture.verify(); !errors.Is(err, ErrGitHubPending) { + t.Fatalf("got %v, want pending", err) + } + if fixture.counts["/actions/runs/201/attempts/1/jobs"] != 0 { + t.Fatal("fetched jobs before required runs completed") + } + }) + } + for _, conclusion := range []string{"failure", "cancelled", "skipped", "neutral", "timed_out", "action_required", ""} { + t.Run("completed_"+conclusion, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if strings.HasSuffix(request.URL.Path, "/actions/runs") { + ghTestFirst(response, "workflow_runs")["conclusion"] = conclusion + } + } + if err := fixture.verify(); !errors.Is(err, ErrCheckNotSuccessful) { + t.Fatalf("got %v, want completed failure", err) + } + }) + } +} + +func TestGitHubRejectsSpoofingAndIncompleteExecution(t *testing.T) { + cases := []struct { + name string + suffix string + change func(map[string]any) + }{ + {"repository_name_reused", "", func(v map[string]any) { v["id"] = 999 }}, + {"unprotected_target", "/branches/main", func(v map[string]any) { v["protected"] = false }}, + {"wrong_pr_head", "/pulls/7", func(v map[string]any) { v["head"].(map[string]any)["sha"] = ghTestOther }}, + {"stale_pr_base", "/pulls/7", func(v map[string]any) { v["base"].(map[string]any)["sha"] = ghTestOther }}, + {"draft_pr", "/pulls/7", func(v map[string]any) { v["draft"] = true }}, + {"closed_pr", "/pulls/7", func(v map[string]any) { v["state"] = "closed" }}, + {"diverged_head", "/compare/" + ghTestBase + "..." + ghTestHead, func(v map[string]any) { v["behind_by"] = 1 }}, + {"wrong_commit_tree_binding", "/git/commits/" + ghTestHead, func(v map[string]any) { v["sha"] = ghTestOther }}, + {"wrong_tree_sha", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { v["sha"] = ghTestHead }}, + {"truncated_tree", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { v["truncated"] = true }}, + {"changed_required_workflow", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { ghTestFirst(v, "tree")["sha"] = ghTestOther }}, + {"changed_excluded_workflow", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { v["tree"].([]any)[1].(map[string]any)["sha"] = ghTestOther }}, + {"changed_workflow_helper", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { v["tree"].([]any)[2].(map[string]any)["sha"] = ghTestOther }}, + {"symlink_workflow", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { ghTestFirst(v, "tree")["mode"] = "120000" }}, + {"extra_candidate_workflow", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { + v["tree"] = append(v["tree"].([]any), map[string]any{"path": ".github/workflows/spoof.yml", "type": "blob", "mode": "100644", "sha": ghTestOther}) + }}, + {"missing_excluded_workflow", "/git/trees/" + ghTestHeadTree, func(v map[string]any) { v["tree"] = v["tree"].([]any)[:1] }}, + {"disabled_required_workflow", "/actions/workflows", func(v map[string]any) { ghTestFirst(v, "workflows")["state"] = "disabled_manually" }}, + {"wrong_workflow_id_same_name", "/actions/runs", func(v map[string]any) { ghTestFirst(v, "workflow_runs")["workflow_id"] = 103 }}, + {"wrong_run_event", "/actions/runs", func(v map[string]any) { ghTestFirst(v, "workflow_runs")["event"] = "push" }}, + {"wrong_run_head", "/actions/runs", func(v map[string]any) { ghTestFirst(v, "workflow_runs")["head_sha"] = ghTestOther }}, + {"wrong_run_repository", "/actions/runs", func(v map[string]any) { ghTestFirst(v, "workflow_runs")["repository"].(map[string]any)["id"] = 43 }}, + {"missing_pr_association", "/actions/runs", func(v map[string]any) { ghTestFirst(v, "workflow_runs")["pull_requests"] = []any{} }}, + {"same_sha_other_pr", "/actions/runs", func(v map[string]any) { ghTestFirst(ghTestFirst(v, "workflow_runs"), "pull_requests")["number"] = 8 }}, + {"wrong_pr_native_id", "/actions/runs", func(v map[string]any) { ghTestFirst(ghTestFirst(v, "workflow_runs"), "pull_requests")["id"] = 72 }}, + {"retargeted_pr_run", "/actions/runs", func(v map[string]any) { + ghTestFirst(ghTestFirst(v, "workflow_runs"), "pull_requests")["base"].(map[string]any)["ref"] = "master" + }}, + {"stale_run_base", "/actions/runs", func(v map[string]any) { + ghTestFirst(ghTestFirst(v, "workflow_runs"), "pull_requests")["base"].(map[string]any)["sha"] = ghTestOther + }}, + {"wrong_run_base_repository", "/actions/runs", func(v map[string]any) { + ghTestFirst(ghTestFirst(v, "workflow_runs"), "pull_requests")["base"].(map[string]any)["repo"].(map[string]any)["id"] = 43 + }}, + {"wrong_associated_head", "/actions/runs", func(v map[string]any) { + ghTestFirst(ghTestFirst(v, "workflow_runs"), "pull_requests")["head"].(map[string]any)["sha"] = ghTestOther + }}, + {"reusable_chain_without_policy", "/actions/runs", func(v map[string]any) { + ghTestFirst(v, "workflow_runs")["referenced_workflows"] = []any{map[string]any{"path": "owner/other/workflow.yml"}} + }}, + {"unknown_run_status", "/actions/runs", func(v map[string]any) { ghTestFirst(v, "workflow_runs")["status"] = "unrecognized" }}, + {"duplicate_run", "/actions/runs", func(v map[string]any) { + v["workflow_runs"] = append(v["workflow_runs"].([]any), ghTestFirst(v, "workflow_runs")) + v["total_count"] = 2 + }}, + {"wrong_job_attempt", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { ghTestFirst(v, "jobs")["run_attempt"] = 2 }}, + {"wrong_job_run", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { ghTestFirst(v, "jobs")["run_id"] = 202 }}, + {"wrong_job_head", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { ghTestFirst(v, "jobs")["head_sha"] = ghTestOther }}, + {"wrong_check_url_host", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { + ghTestFirst(v, "jobs")["check_run_url"] = "https://example.invalid/check-runs/401" + }}, + {"duplicate_job", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { v["jobs"].([]any)[1] = ghTestFirst(v, "jobs") }}, + {"empty_steps", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { ghTestFirst(v, "jobs")["steps"] = []any{} }}, + {"skipped_success_step", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { ghTestFirst(ghTestFirst(v, "jobs"), "steps")["conclusion"] = "skipped" }}, + {"failed_step_conclusion", "/actions/runs/201/attempts/1/jobs", func(v map[string]any) { ghTestFirst(ghTestFirst(v, "jobs"), "steps")["conclusion"] = "failure" }}, + {"wrong_check_app", "/check-suites/301/check-runs", func(v map[string]any) { ghTestFirst(v, "check_runs")["app"].(map[string]any)["id"] = 123 }}, + {"wrong_check_suite", "/check-suites/301/check-runs", func(v map[string]any) { ghTestFirst(v, "check_runs")["check_suite"].(map[string]any)["id"] = 999 }}, + {"neutral_check", "/check-suites/301/check-runs", func(v map[string]any) { ghTestFirst(v, "check_runs")["conclusion"] = "neutral" }}, + {"check_job_name_mismatch", "/check-suites/301/check-runs", func(v map[string]any) { ghTestFirst(v, "check_runs")["name"] = "spoof" }}, + {"wrong_check_head", "/check-suites/301/check-runs", func(v map[string]any) { ghTestFirst(v, "check_runs")["head_sha"] = ghTestOther }}, + {"duplicate_check", "/check-suites/301/check-runs", func(v map[string]any) { v["check_runs"].([]any)[1] = ghTestFirst(v, "check_runs") }}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if request.URL.Path == "/repos/owner/repo"+test.suffix { + test.change(response) + } + } + if err := fixture.verify(); err == nil || errors.Is(err, ErrGitHubPending) { + t.Fatalf("unsafe API fixture accepted or retried: %v", err) + } + }) + } +} + +func TestGitHubCompletedRunCannotOmitRequiredJobs(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.setJobs([]string{"unit"}) + if err := fixture.verify(); !errors.Is(err, ErrMissingCheck) { + t.Fatalf("got %v, want missing job failure", err) + } +} + +func TestGitHubLatestFailureCannotReuseOlderSuccess(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if strings.HasSuffix(request.URL.Path, "/actions/runs") { + older := ghTestFirst(response, "workflow_runs") + newer := make(map[string]any) + for key, value := range older { + newer[key] = value + } + newer["id"], newer["run_number"], newer["conclusion"] = 202, 12, "failure" + response["workflow_runs"], response["total_count"] = []any{older, newer}, 2 + } + } + if err := fixture.verify(); !errors.Is(err, ErrCheckNotSuccessful) { + t.Fatalf("got %v, want newer failure", err) + } +} + +func TestGitHubStateMutationDuringCollection(t *testing.T) { + for _, target := range []string{"pr", "branch", "run_attempt", "latest_run", "workflow"} { + t.Run(target, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, count int, response map[string]any) { + suffix := strings.TrimPrefix(request.URL.Path, "/repos/owner/repo") + switch { + case target == "pr" && suffix == "/pulls/7" && count == 2: + response["head"].(map[string]any)["sha"] = ghTestOther + case target == "branch" && suffix == "/branches/main" && count == 2: + response["commit"].(map[string]any)["sha"] = ghTestOther + case target == "run_attempt" && suffix == "/actions/runs/201": + response["run_attempt"] = 2 + case target == "latest_run" && suffix == "/actions/runs" && count == 2: + run := ghTestFirst(response, "workflow_runs") + run["run_attempt"], run["status"], run["conclusion"] = 2, "in_progress", nil + case target == "workflow" && suffix == "/actions/workflows" && count == 2: + ghTestFirst(response, "workflows")["id"] = 999 + } + } + if err := fixture.verify(); !errors.Is(err, ErrGitHubState) { + t.Fatalf("changed collection returned %v", err) + } + }) + } +} + +func TestGitHubBoundedCompletePagination(t *testing.T) { + for _, mode := range []string{"valid", "missing_page", "changed_total", "duplicate_across_pages", "over_limit"} { + t.Run(mode, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + names := make([]string, 101) + for index := range names { + names[index] = fmt.Sprintf("matrix-%d", index) + } + fixture.policy.Workflows[0].Jobs = names + fixture.setJobs(names) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if !strings.HasSuffix(request.URL.Path, "/jobs") { + return + } + if mode == "over_limit" { + response["total_count"] = 1001 + } + if request.URL.Query().Get("page") != "2" { + return + } + switch mode { + case "missing_page": + response["jobs"] = []any{} + case "changed_total": + response["total_count"] = 102 + case "duplicate_across_pages": + ghTestFirst(response, "jobs")["id"] = 401 + } + } + err := fixture.verify() + if mode == "valid" && err != nil { + t.Fatal(err) + } + if mode != "valid" && (err == nil || errors.Is(err, ErrGitHubPending)) { + t.Fatalf("incomplete or ambiguous pagination admitted: %v", err) + } + }) + } +} + +func TestGitHubHTTPFailuresAreBoundedAndRedacted(t *testing.T) { + for _, mode := range []string{"redirect", "forbidden", "duplicate_json", "trailing_json", "content_type", "short_body", "oversized"} { + t.Run(mode, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + redirects := 0 + fixture.client.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { redirects++; return nil } + fixture.reply = func(writer http.ResponseWriter, request *http.Request, _ int) bool { + if request.URL.Path != "/repos/owner/repo" { + return false + } + switch mode { + case "redirect": + writer.Header().Set("Location", "https://example.invalid/test-token") + writer.WriteHeader(http.StatusFound) + case "forbidden": + writer.WriteHeader(http.StatusForbidden) + _, _ = fmt.Fprint(writer, `{"message":"test-token opaque-failure-sentinel"}`) + case "duplicate_json": + _, _ = fmt.Fprint(writer, `{"id":42,"id":999,"full_name":"owner/repo"}`) + case "trailing_json": + _, _ = fmt.Fprint(writer, `{"id":42,"full_name":"owner/repo"}{"extra":true}`) + case "content_type": + writer.Header().Set("Content-Type", "text/html") + _, _ = fmt.Fprint(writer, `{"id":42,"full_name":"owner/repo"}`) + case "short_body": + writer.Header().Set("Content-Length", "1000") + _, _ = fmt.Fprint(writer, `{"id":42}`) + case "oversized": + writer.Header().Set("Content-Length", strconv.Itoa(strictjson.MaxDocumentBytes+1)) + _, _ = fmt.Fprint(writer, strings.Repeat(" ", strictjson.MaxDocumentBytes+1)) + } + return true + } + err := fixture.verify() + if !errors.Is(err, ErrGitHubAPI) || strings.Contains(err.Error(), "test-token") || strings.Contains(err.Error(), "opaque-failure-sentinel") || strings.Contains(err.Error(), "example.invalid") { + t.Fatalf("unredacted or unexpected HTTP error: %v", err) + } + if redirects != 0 { + t.Fatal("adapter followed a redirect with its credential") + } + }) + } +} + +func TestReadGitHubPolicyRejectsSymlinksAndForgedBlobs(t *testing.T) { + for _, mode := range []string{"symlink", "executable", "blob_hash", "encoding", "size", "repository_id"} { + t.Run(mode, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.mutate = func(request *http.Request, _ int, response map[string]any) { + if strings.HasSuffix(request.URL.Path, "/git/trees/"+ghTestBaseTree) && (mode == "symlink" || mode == "executable") { + entry := response["tree"].([]any)[3].(map[string]any) + if mode == "symlink" { + entry["mode"] = "120000" + } else { + entry["mode"] = "100755" + } + } + if mode == "repository_id" && request.URL.Path == "/repos/owner/repo" { + response["id"] = 99 + } + if strings.HasSuffix(request.URL.Path, "/git/blobs/"+ghTestPolicyID) { + switch mode { + case "blob_hash": + response["content"] = base64.StdEncoding.EncodeToString([]byte(strings.Replace(ghTestPolicy, "unit", "evil", 1))) + case "encoding": + response["encoding"] = "utf-8" + case "size": + response["size"] = 1 + } + } + } + if _, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/repo", ghTestBase); err == nil || errors.Is(err, ErrGitHubPending) { + t.Fatalf("unsafe policy accepted: %v", err) + } + }) + } +} + +func TestReadGitHubPolicyRejectsUnknownFieldsAfterBlobVerification(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + const policyID = "fc446526e44b5189d82ea86595794637c3b8128f" + contents := ghTestPolicy[:len(ghTestPolicy)-1] + `,"untrusted":true}` + tree := fixture.responses["/git/trees/"+ghTestBaseTree].(map[string]any) + tree["tree"].([]any)[3].(map[string]any)["sha"] = policyID + fixture.responses["/git/blobs/"+policyID] = map[string]any{ + "sha": policyID, "encoding": "base64", "size": 236, + "content": base64.StdEncoding.EncodeToString([]byte(contents)), + } + if _, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/repo", ghTestBase); !errors.Is(err, ErrPolicy) { + t.Fatalf("correctly hashed policy with unknown field returned %v", err) + } +} + +func TestGitHubRejectsInvalidInputBeforeHTTP(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.candidate.HeadSHA = "invalid" + if err := fixture.verify(); !errors.Is(err, ErrGitHubCandidate) { + t.Fatalf("got %v", err) + } + if len(fixture.requests) != 0 { + t.Fatal("sent request for invalid candidate") + } + if _, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/../repo", ghTestBase); !errors.Is(err, ErrPolicy) { + t.Fatalf("got %v", err) + } + if len(fixture.requests) != 0 { + t.Fatal("sent request for invalid repository") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := ReadGitHubPolicy(ctx, fixture.client, "test-token", "owner/repo", ghTestBase); !errors.Is(err, ErrGitHubAPI) { + t.Fatalf("got %v", err) + } +} + +type githubCancelOnCloseBody struct { + io.ReadCloser + cancel context.CancelFunc +} + +func (body githubCancelOnCloseBody) Close() error { + err := body.ReadCloser.Close() + body.cancel() + return err +} + +type githubCancelAfterReadTransport struct { + base http.RoundTripper + suffix string + nth int + count int + cancel context.CancelFunc +} + +func (transport *githubCancelAfterReadTransport) RoundTrip(request *http.Request) (*http.Response, error) { + response, err := transport.base.RoundTrip(request) + if strings.HasSuffix(request.URL.Path, transport.suffix) { + transport.count++ + if err == nil && transport.count == transport.nth { + response.Body = githubCancelOnCloseBody{ReadCloser: response.Body, cancel: transport.cancel} + } + } + return response, err +} + +func TestGitHubCannotSucceedAfterFinalResponseCancellation(t *testing.T) { + for _, operation := range []string{"policy", "verify"} { + t.Run(operation, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + transport := &githubCancelAfterReadTransport{base: fixture.client.Transport, cancel: cancel} + fixture.client.Transport = transport + var err error + if operation == "policy" { + transport.suffix, transport.nth = "/git/blobs/"+ghTestPolicyID, 1 + _, err = ReadGitHubPolicy(ctx, fixture.client, "test-token", "owner/repo", ghTestBase) + } else { + transport.suffix, transport.nth = "/branches/main", 2 + err = VerifyGitHub(ctx, fixture.client, "test-token", fixture.policy, fixture.candidate) + } + if !errors.Is(err, ErrGitHubAPI) || ctx.Err() == nil { + t.Fatalf("operation succeeded after cancellation: %v", err) + } + }) + } +} diff --git a/pkg/safepush/policy.go b/pkg/safepush/policy.go new file mode 100644 index 0000000..56375ce --- /dev/null +++ b/pkg/safepush/policy.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "encoding/json" + "errors" + "path" + "regexp" + "strings" + "unicode" + "unicode/utf8" + + "github.com/opencloudtech/CloudRING/internal/strictjson" +) + +const GitHubPolicySchema = "cloudring.safepush.github/v1" + +var ( + ErrPolicy = errors.New("SafePush policy is invalid") + ErrRequest = errors.New("SafePush request is invalid") +) + +// GitHubPolicy is read from the accepted target commit by the trusted CI +// workflow. Loading a candidate's policy would let that candidate remove its +// own requirements. RepositoryID also prevents name reuse from changing scope. +type GitHubPolicy struct { + SchemaVersion string `json:"schemaVersion"` + Repository string `json:"repository"` + RepositoryID int64 `json:"repositoryId"` + Workflows []GitHubWorkflow `json:"workflows"` + ExcludedWorkflows []string `json:"excludedWorkflows"` +} + +// GitHubWorkflow identifies the complete expected job set of one test workflow. +// Path and job name together are the identity; a same-named job in another +// workflow cannot satisfy a requirement. +type GitHubWorkflow struct { + Path string `json:"path"` + Jobs []string `json:"jobs"` +} + +// GitHubCandidate binds one admission attempt to the PR and target snapshot. +// The adapter re-reads GitHub state; these values do not assert approval or +// successful execution and cannot replace native server protection. +type GitHubCandidate struct { + PullRequest int + HeadSHA string + BaseSHA string + TargetBranch string +} + +var repositoryName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}/[A-Za-z0-9_.-]{1,100}$`) + +// ValidateGitHubCandidate validates event arguments before any network access. +// Ref names are never accepted in place of immutable Git object identifiers. +func ValidateGitHubCandidate(repository string, candidate GitHubCandidate) error { + if !repositoryName.MatchString(repository) || strings.HasSuffix(repository, "/.") || + strings.HasSuffix(repository, "/..") || candidate.PullRequest <= 0 || + !validHeadSHA(candidate.HeadSHA) || !validHeadSHA(candidate.BaseSHA) || + candidate.HeadSHA == candidate.BaseSHA || + (candidate.TargetBranch != "main" && candidate.TargetBranch != "master") { + return ErrRequest + } + return nil +} + +// DecodeGitHubPolicy rejects unknown and duplicate fields, trailing documents, +// empty test sets, overlapping exclusions and ambiguous workflow identities. +func DecodeGitHubPolicy(data []byte) (GitHubPolicy, error) { + // encoding/json matches struct fields case-insensitively, including when + // DisallowUnknownFields is set. Inspect exact spellings before decoding so + // a second "Workflows" or "Jobs" cannot replace an earlier requirement. + var fields map[string]json.RawMessage + if strictjson.Decode(data, &fields) != nil || fields == nil { + return GitHubPolicy{}, ErrPolicy + } + for name := range fields { + switch name { + case "schemaVersion", "repository", "repositoryId", "workflows", "excludedWorkflows": + default: + return GitHubPolicy{}, ErrPolicy + } + } + var workflows []map[string]json.RawMessage + if json.Unmarshal(fields["workflows"], &workflows) != nil { + return GitHubPolicy{}, ErrPolicy + } + for _, workflow := range workflows { + for name := range workflow { + if name != "path" && name != "jobs" { + return GitHubPolicy{}, ErrPolicy + } + } + } + var policy GitHubPolicy + if json.Unmarshal(data, &policy) != nil || ValidateGitHubPolicy(policy) != nil { + return GitHubPolicy{}, ErrPolicy + } + return policy, nil +} + +func ValidateGitHubPolicy(policy GitHubPolicy) error { + if policy.SchemaVersion != GitHubPolicySchema || policy.RepositoryID <= 0 || + !repositoryName.MatchString(policy.Repository) || + strings.HasSuffix(policy.Repository, "/.") || strings.HasSuffix(policy.Repository, "/..") || + len(policy.Workflows) == 0 || len(policy.Workflows)+len(policy.ExcludedWorkflows) > 100 { + return ErrPolicy + } + seen := make(map[string]bool) + for _, workflow := range policy.Workflows { + if !validWorkflowPath(workflow.Path) || seen[workflow.Path] || len(workflow.Jobs) == 0 || len(workflow.Jobs) > 1000 { + return ErrPolicy + } + seen[workflow.Path] = true + jobs := make(map[string]bool) + for _, job := range workflow.Jobs { + if !validJobName(job) || jobs[job] { + return ErrPolicy + } + jobs[job] = true + } + } + for _, excluded := range policy.ExcludedWorkflows { + if !validWorkflowPath(excluded) || seen[excluded] { + return ErrPolicy + } + seen[excluded] = true + } + return nil +} + +func validWorkflowPath(value string) bool { + if len(value) > 255 || !strings.HasPrefix(value, ".github/workflows/") || path.Clean(value) != value { + return false + } + name := strings.TrimPrefix(value, ".github/workflows/") + if strings.ContainsAny(name, "/\\") || !(strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml")) { + return false + } + return validJobName(name) +} + +func validJobName(value string) bool { + if value == "" || len(value) > 512 || !utf8.ValidString(value) || strings.TrimSpace(value) != value { + return false + } + for _, r := range value { + if unicode.IsControl(r) { + return false + } + } + return true +} diff --git a/pkg/safepush/policy_test.go b/pkg/safepush/policy_test.go new file mode 100644 index 0000000..e7a3dcd --- /dev/null +++ b/pkg/safepush/policy_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +func policyFixture() GitHubPolicy { + return GitHubPolicy{ + SchemaVersion: GitHubPolicySchema, + Repository: "synthetic/project", RepositoryID: 42, + Workflows: []GitHubWorkflow{{Path: ".github/workflows/ci.yml", Jobs: []string{"test", "race"}}}, + ExcludedWorkflows: []string{".github/workflows/release.yml"}, + } +} + +func TestPolicyRejectsAmbiguousOrWeakenedRequirements(t *testing.T) { + mutations := map[string]func(*GitHubPolicy){ + "wrong schema": func(p *GitHubPolicy) { p.SchemaVersion = "unknown" }, + "no repository identity": func(p *GitHubPolicy) { p.RepositoryID = 0 }, + "path traversal": func(p *GitHubPolicy) { p.Repository = "synthetic/../project" }, + "no tests": func(p *GitHubPolicy) { p.Workflows = nil }, + "empty jobs": func(p *GitHubPolicy) { p.Workflows[0].Jobs = nil }, + "duplicate workflow": func(p *GitHubPolicy) { p.Workflows = append(p.Workflows, p.Workflows[0]) }, + "duplicate job": func(p *GitHubPolicy) { p.Workflows[0].Jobs = []string{"test", "test"} }, + "excluded test workflow": func(p *GitHubPolicy) { p.ExcludedWorkflows = []string{p.Workflows[0].Path} }, + "duplicate exclusion": func(p *GitHubPolicy) { p.ExcludedWorkflows = append(p.ExcludedWorkflows, p.ExcludedWorkflows[0]) }, + "nested workflow path": func(p *GitHubPolicy) { p.Workflows[0].Path = ".github/workflows/nested/ci.yml" }, + "noncanonical workflow": func(p *GitHubPolicy) { p.Workflows[0].Path = ".github/workflows/../ci.yml" }, + "control in identity": func(p *GitHubPolicy) { p.Workflows[0].Jobs[0] = "test\npass" }, + "invisible job suffix": func(p *GitHubPolicy) { p.Workflows[0].Jobs[0] = "test " }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + policy := policyFixture() + mutate(&policy) + data, err := json.Marshal(policy) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeGitHubPolicy(data); !errors.Is(err, ErrPolicy) { + t.Fatalf("unsafe policy was accepted: %v", err) + } + }) + } +} + +func TestPolicyJSONHasOneClosedUnambiguousDocument(t *testing.T) { + data, err := json.Marshal(policyFixture()) + if err != nil { + t.Fatal(err) + } + if _, err := DecodeGitHubPolicy(data); err != nil { + t.Fatal(err) + } + for name, malformed := range map[string]string{ + "unknown field": strings.Replace(string(data), `"repositoryId":42`, `"repositoryId":42,"trusted":true`, 1), + "duplicate field": strings.Replace(string(data), `"repositoryId":42`, `"repositoryId":42,"repositoryId":42`, 1), + "case variant duplicate": strings.Replace(string(data), `"repositoryId":42`, `"repositoryId":42,"RepositoryId":43`, 1), + "weakened case variant workflows": strings.Replace(string(data), `"excludedWorkflows":`, `"Workflows":[{"path":".github/workflows/ci.yml","jobs":["test"]}],"excludedWorkflows":`, 1), + "weakened case variant jobs": strings.Replace(string(data), `"jobs":["test","race"]`, `"jobs":["test","race"],"Jobs":["test"]`, 1), + "unknown case variant schema": strings.Replace(string(data), `"schemaVersion":`, `"SchemaVersion":`, 1), + "unknown case variant path": strings.Replace(string(data), `"path":`, `"Path":`, 1), + "escaped duplicate field": strings.Replace(string(data), `"repositoryId":42`, `"repositoryId":42,"\u0072epositoryId":42`, 1), + "trailing object": string(data) + `{}`, + "wrong integer": strings.Replace(string(data), `"repositoryId":42`, `"repositoryId":42.5`, 1), + } { + t.Run(name, func(t *testing.T) { + if _, err := DecodeGitHubPolicy([]byte(malformed)); !errors.Is(err, ErrPolicy) { + t.Fatalf("ambiguous policy was accepted: %v", err) + } + }) + } +} diff --git a/pkg/safepush/repository_policy_test.go b/pkg/safepush/repository_policy_test.go new file mode 100644 index 0000000..37e2484 --- /dev/null +++ b/pkg/safepush/repository_policy_test.go @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// This catches an added, removed, or renamed repository CI job being silently +// omitted from the admission policy. API provenance is tested separately. +func TestRepositoryPolicyCoversEveryPreMergeJob(t *testing.T) { + root := openRepositoryRoot(t) + data, err := root.ReadFile(".github/safepush.json") + if err != nil { + t.Fatal(err) + } + policy, err := DecodeGitHubPolicy(data) + if err != nil { + t.Fatal(err) + } + expectedPaths := make(map[string]bool) + for _, workflow := range policy.Workflows { + expectedPaths[workflow.Path] = true + data, err := root.ReadFile(filepath.FromSlash(workflow.Path)) + if err != nil { + t.Fatal(err) + } + var actual struct { + On map[string]any `yaml:"on"` + Jobs map[string]struct { + Name string `yaml:"name"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &actual); err != nil { + t.Fatal(err) + } + if _, exists := actual.On["pull_request"]; !exists { + t.Fatal("required test workflow no longer runs for pull requests") + } + var names []string + for id, job := range actual.Jobs { + name := job.Name + if name == "" { + name = id + } + if strings.Contains(name, "${{") { + t.Fatal("dynamic job name needs an explicit matrix policy") + } + names = append(names, name) + } + sort.Strings(names) + expected := append([]string(nil), workflow.Jobs...) + sort.Strings(expected) + if !reflect.DeepEqual(names, expected) { + t.Fatalf("workflow %s jobs differ from admission policy: actual=%v expected=%v", workflow.Path, names, expected) + } + } + for _, excluded := range policy.ExcludedWorkflows { + expectedPaths[excluded] = true + data, err := root.ReadFile(filepath.FromSlash(excluded)) + if err != nil { + t.Fatal(err) + } + var actual struct { + On map[string]any `yaml:"on"` + } + if err := yaml.Unmarshal(data, &actual); err != nil { + t.Fatal(err) + } + if len(actual.On) != 1 { + t.Fatal("excluded release workflow has unexpected triggers") + } + if _, exists := actual.On["push"]; !exists { + t.Fatal("excluded workflow is not a post-acceptance release workflow") + } + } + directory, err := root.Open(".github/workflows") + if err != nil { + t.Fatal(err) + } + entries, err := directory.ReadDir(-1) + if closeErr := directory.Close(); closeErr != nil { + t.Fatal(closeErr) + } + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + name := ".github/workflows/" + entry.Name() + if (strings.HasSuffix(name, ".yml") || strings.HasSuffix(name, ".yaml")) && !expectedPaths[name] { + t.Fatalf("workflow is absent from admission policy: %s", name) + } + } +} + +// Confine fixture reads to the checkout, including symlink resolution. +func openRepositoryRoot(t *testing.T) *os.Root { + t.Helper() + root, err := os.OpenRoot("../..") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Error(err) + } + }) + return root +} diff --git a/pkg/safepush/workflow_source.go b/pkg/safepush/workflow_source.go new file mode 100644 index 0000000..3036217 --- /dev/null +++ b/pkg/safepush/workflow_source.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "bytes" + "errors" + "io" + "strings" + + "gopkg.in/yaml.v3" +) + +const ( + maxWorkflowSourceBytes = 1 << 20 + maxWorkflowYAMLDepth = 64 + maxWorkflowYAMLNodes = 100000 +) + +// verifyWorkflowSource checks accepted, hash-verified workflow bytes. GitHub's +// REST step conclusion can be success after a failed outcome when the workflow +// permits continue-on-error, so successful API statuses alone are insufficient. +// This verifies that specific control; it does not prove test adequacy or the +// behavior of scripts/actions invoked by the workflow. +func verifyWorkflowSource(contents []byte) error { + if len(contents) == 0 || len(contents) > maxWorkflowSourceBytes { + return ErrGitHubSource + } + decoder := yaml.NewDecoder(bytes.NewReader(contents)) + var document yaml.Node + if decoder.Decode(&document) != nil { + return ErrGitHubSource + } + nodes := 0 + if !validWorkflowYAML(&document, 0, &nodes) || document.Kind != yaml.DocumentNode || + len(document.Content) != 1 || document.Content[0].Kind != yaml.MappingNode { + return ErrGitHubSource + } + var trailing yaml.Node + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return ErrGitHubSource + } + jobs := workflowMappingValue(document.Content[0], "jobs") + if jobs == nil || jobs.Kind != yaml.MappingNode || len(jobs.Content) == 0 { + return ErrGitHubSource + } + for index := 1; index < len(jobs.Content); index += 2 { + job := jobs.Content[index] + if job.Kind != yaml.MappingNode || !workflowStopsOnError(job) { + return ErrGitHubSource + } + steps := workflowMappingValue(job, "steps") + if steps == nil || steps.Kind != yaml.SequenceNode || len(steps.Content) == 0 { + return ErrGitHubSource + } + for _, step := range steps.Content { + if step.Kind != yaml.MappingNode || !workflowStopsOnError(step) { + return ErrGitHubSource + } + } + } + return nil +} + +func workflowStopsOnError(mapping *yaml.Node) bool { + value := workflowMappingValue(mapping, "continue-on-error") + return value == nil || (value.Kind == yaml.ScalarNode && value.Tag == "!!bool" && + value.Style == 0 && value.Value == "false") +} + +func workflowMappingValue(mapping *yaml.Node, key string) *yaml.Node { + for index := 0; index < len(mapping.Content); index += 2 { + if strings.EqualFold(mapping.Content[index].Value, key) { + return mapping.Content[index+1] + } + } + return nil +} + +func validWorkflowYAML(node *yaml.Node, depth int, nodes *int) bool { + *nodes = *nodes + 1 + if depth > maxWorkflowYAMLDepth || *nodes > maxWorkflowYAMLNodes || node == nil || + node.Anchor != "" || node.Alias != nil { + return false + } + switch node.Kind { + case yaml.DocumentNode: + if depth != 0 || len(node.Content) != 1 { + return false + } + case yaml.MappingNode: + if node.Tag != "!!map" || len(node.Content)%2 != 0 { + return false + } + seen := make(map[string]bool, len(node.Content)/2) + for index := 0; index < len(node.Content); index += 2 { + key := node.Content[index] + if key.Kind != yaml.ScalarNode || key.Tag != "!!str" || !validJobName(key.Value) || key.Value == "<<" { + return false + } + identity := strings.ToLower(key.Value) + if seen[identity] { + return false + } + seen[identity] = true + } + case yaml.SequenceNode: + if node.Tag != "!!seq" { + return false + } + case yaml.ScalarNode: + if len(node.Content) != 0 { + return false + } + switch node.Tag { + case "!!str", "!!bool", "!!int", "!!float", "!!null": + default: + return false + } + default: + return false // Includes aliases, without traversing their targets. + } + for _, child := range node.Content { + if !validWorkflowYAML(child, depth+1, nodes) { + return false + } + } + return true +} diff --git a/pkg/safepush/workflow_source_test.go b/pkg/safepush/workflow_source_test.go new file mode 100644 index 0000000..19f72f1 --- /dev/null +++ b/pkg/safepush/workflow_source_test.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "errors" + "fmt" + "path/filepath" + "strings" + "testing" +) + +func TestWorkflowSourceRequiresLiteralStopOnError(t *testing.T) { + for _, scope := range []string{"job", "step"} { + for _, value := range []string{ + "true", "TRUE", "False", "FALSE", "0", "no", "off", "null", "~", "", + "'false'", `"false"`, "${{ false }}", "${{ matrix.experimental }}", + "!!str false", "!!bool false", "!custom false", "&disabled false", "[false]", "{value: false}", + } { + t.Run(scope+"_"+value, func(t *testing.T) { + contents := "jobs:\n test:\n runs-on: ubuntu-latest\n" + if scope == "job" { + contents += " continue-on-error: " + value + "\n" + } + contents += " steps:\n - run: go test ./...\n" + if scope == "step" { + contents += " continue-on-error: " + value + "\n" + } + if err := verifyWorkflowSource([]byte(contents)); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("unsafe control was accepted: %v", err) + } + }) + } + } + for _, contents := range []string{ + ghTestWorkflow, + "jobs: {test: {continue-on-error: false, steps: [{run: 'exit 0', continue-on-error: false}]}}", + "---\n'jobs': {'test': {'steps': [{'run': 'exit 0', 'continue-on-error': false}]}}\n...\n# end\n", + } { + if err := verifyWorkflowSource([]byte(contents)); err != nil { + t.Fatalf("literal stop-on-error workflow rejected: %v", err) + } + } +} + +func TestWorkflowSourceRejectsAmbiguousYAML(t *testing.T) { + safe := "jobs: {test: {steps: [{run: 'exit 0'}]}}\n" + cases := map[string]string{ + "empty": "", + "comments_only": "# empty\n", + "root_sequence": "[{jobs: {}}]", + "jobs_sequence": "jobs: []", + "jobs_expression": "jobs: ${{ matrix.jobs }}", + "empty_jobs": "jobs: {}", + "null_job": "jobs: {test: null}", + "no_steps": "jobs: {test: {runs-on: ubuntu-latest}}", + "empty_steps": "jobs: {test: {steps: []}}", + "step_scalar": "jobs: {test: {steps: ['exit 0']}}", + "step_mapping": "jobs: {test: {steps: {run: 'exit 0'}}}", + "steps_expression": "jobs: {test: {steps: '${{ matrix.steps }}'}}", + "second_document": safe + "---\n" + safe, + "empty_second_doc": safe + "---\n", + "malformed_trailer": safe + "[not valid", + "duplicate_root": safe + "jobs: {test: {steps: [{run: 'exit 1', continue-on-error: true}]}}\n", + "duplicate_job": "jobs: {test: {continue-on-error: true, continue-on-error: false, steps: [{run: 'exit 1'}]}}", + "duplicate_step": "jobs: {test: {steps: [{continue-on-error: true, continue-on-error: false, run: 'exit 1'}]}}", + "case_duplicate": "jobs: {test: {steps: [{continue-on-error: false, CONTINUE-ON-ERROR: true, run: 'exit 1'}]}}", + "case_control": "jobs: {test: {steps: [{CONTINUE-ON-ERROR: true, run: 'exit 1'}]}}", + "escaped_key": `jobs: {test: {steps: [{"continue\u002don-error": true, run: 'exit 1'}]}}`, + "merge_job": "jobs: {test: {<<: {continue-on-error: true}, steps: [{run: 'exit 1'}]}}", + "quoted_merge": "jobs: {test: {'<<': {continue-on-error: true}, steps: [{run: 'exit 1'}]}}", + "merge_step": "jobs: {test: {steps: [{<<: {continue-on-error: true}, run: 'exit 1'}]}}", + "anchor_only": "defaults: &settings {continue-on-error: true}\n" + safe, + "alias_false": "setting: &disabled false\njobs: {test: {steps: [{run: 'exit 0', continue-on-error: *disabled}]}}", + "alias_steps": "setting: &steps [{run: 'exit 1', continue-on-error: true}]\njobs: {test: {steps: *steps}}", + "alias_cycle": "setting: &cycle [*cycle]\n" + safe, + "alias_expansion": "a: &a [false,false,false]\nb: &b [*a,*a,*a]\nc: [*b,*b,*b]\n" + safe, + "nonstring_key": "true: ignored\n" + safe, + "complex_key": "? [jobs, steps]\n: ignored\n" + safe, + "custom_mapping": "jobs: !custom {test: {steps: [{run: 'exit 0'}]}}", + "block_string": "jobs:\n test:\n continue-on-error: |\n false\n steps:\n - run: exit 0\n", + } + for name, contents := range cases { + t.Run(name, func(t *testing.T) { + if err := verifyWorkflowSource([]byte(contents)); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("ambiguous YAML accepted: %v", err) + } + }) + } +} + +func TestWorkflowSourceParsingIsBounded(t *testing.T) { + if err := verifyWorkflowSource([]byte(strings.Repeat(" ", maxWorkflowSourceBytes+1))); !errors.Is(err, ErrGitHubSource) { + t.Fatal("oversized workflow accepted") + } + deep := "env: {nested: " + strings.Repeat("[", maxWorkflowYAMLDepth+1) + "false" + strings.Repeat("]", maxWorkflowYAMLDepth+1) + "}\n" + ghTestWorkflow + if err := verifyWorkflowSource([]byte(deep)); !errors.Is(err, ErrGitHubSource) { + t.Fatal("overly deep workflow accepted") + } + var many strings.Builder + many.WriteString("env: {") + for index := 0; index <= maxWorkflowYAMLNodes/2; index++ { + _, _ = fmt.Fprintf(&many, "k%d: 0,", index) + } + many.WriteString("}\n") + many.WriteString(ghTestWorkflow) + if many.Len() > maxWorkflowSourceBytes { + t.Fatal("node-limit fixture must exercise nodes rather than byte size") + } + if err := verifyWorkflowSource([]byte(many.String())); !errors.Is(err, ErrGitHubSource) { + t.Fatal("too many workflow nodes accepted") + } +} + +func TestWorkflowSourceAcceptsCurrentRequiredWorkflows(t *testing.T) { + root := openRepositoryRoot(t) + for _, name := range []string{"ci.yml", "cla-dco.yml", "license.yml", "security.yml", "supply-chain.yml"} { + t.Run(name, func(t *testing.T) { + contents, err := root.ReadFile(filepath.Join(".github", "workflows", name)) + if err != nil { + t.Fatal(err) + } + if err = verifyWorkflowSource(contents); err != nil { + t.Fatalf("current required workflow rejected: %v", err) + } + }) + } +} From 0cfe2935d6a8147ad609ae3bb3ce7c041b85a2ac Mon Sep 17 00:00:00 2001 From: Iurii Trukhin Date: Sun, 8 Nov 2026 22:27:58 +0100 Subject: [PATCH 2/3] feat(sp01): add RSI promotion gate, 32 slice oracles and parallel slice delivery contract SP01 adds the guardrails that let a delivery slice be accepted against a pre-approved oracle instead of waiting on a per-slice manual sign-off: - pkg/rsi: deterministic promotion gate (G1-G7) with change classes K0-K4, protected control files, two-slot last-known-good rollback invariants (S1-S6), mutation, holdout and seeded-error thresholds, and a kill switch that lowers the autonomy ceiling when the change failure rate is exceeded. - pkg/sliceoracles and contracts/slice-oracles/oracles-32.json: the batch of 32 slice oracles (C01-C28, SP01, DBAAS, DNSAAS, FILESAAS), the split of hardware-gated slices, and validation that two concurrent write PRs own disjoint paths. - roadmap: C15-C17, C20 and the standalone OCS services split into CORE (after C11) and QUAL (after C13 and C14); C24 follows C14; at most two concurrent write PRs per repository for distinct slices. --- .../fixtures/compliant-k3-profile.json | 124 ++ .../fixtures/tampered-k4-profile.json | 70 + contracts/slice-oracles/oracles-32.json | 1117 +++++++++++++++ pkg/rsi/rsi.go | 1192 +++++++++++++++++ pkg/rsi/rsi_fuzz_test.go | 61 + pkg/rsi/rsi_test.go | 664 +++++++++ pkg/sliceoracles/oracles.go | 399 ++++++ pkg/sliceoracles/oracles_test.go | 193 +++ roadmap/DELIVERY_SLICES.md | 34 +- roadmap/EXECUTION_CONTRACT.md | 12 +- 10 files changed, 3852 insertions(+), 14 deletions(-) create mode 100644 contracts/rsi-governance/fixtures/compliant-k3-profile.json create mode 100644 contracts/rsi-governance/fixtures/tampered-k4-profile.json create mode 100644 contracts/slice-oracles/oracles-32.json create mode 100644 pkg/rsi/rsi.go create mode 100644 pkg/rsi/rsi_fuzz_test.go create mode 100644 pkg/rsi/rsi_test.go create mode 100644 pkg/sliceoracles/oracles.go create mode 100644 pkg/sliceoracles/oracles_test.go diff --git a/contracts/rsi-governance/fixtures/compliant-k3-profile.json b/contracts/rsi-governance/fixtures/compliant-k3-profile.json new file mode 100644 index 0000000..e2ee7ed --- /dev/null +++ b/contracts/rsi-governance/fixtures/compliant-k3-profile.json @@ -0,0 +1,124 @@ +{ + "apiVersion": "cloudring.org/rsi-governance/v1", + "kind": "RSICandidateProfile", + "candidate": { + "id": "cand-k3-rsi-reference-01", + "declaredClass": "K3", + "changedPaths": [ + "pkg/rsi/rsi.go", + "cmd/cloudringctl/commands.go" + ], + "readOnlyOnly": false, + "reversibleConfigOnly": false, + "deletedOrSkippedTests": 0, + "modifiedExistingTests": false, + "modifiedTestdata": false, + "humanK4ApprovalReference": "" + }, + "gates": { + "g1": { + "evaluatorFrozenHashVerified": true, + "suitePassed": true, + "newTestFailedBeforeFix": true, + "newTestPassedAfterFix": true, + "rerunMaskingUsed": false, + "silentErrorSwallowing": 0, + "unseenHoldoutPassed": true, + "seededErrorRejected": true + }, + "g2": { + "killedMutants": 19, + "totalMutants": 20, + "fuzzTargetsPassed": true + }, + "g3": { + "sastFindings": 0, + "vulnFindings": 0, + "externalCapabilityGate": true, + "promptDelimiterOnlySecurity": false, + "explanationOnlyApproval": false + }, + "g4": { + "activeCanaries": 1, + "canaryStepPercent": "10", + "canaryMaxPercent": "50", + "requestSuccessRatePercent": "99.5", + "latencyP99Ms": "210", + "canaryDurationMinutes": "10", + "canaryFailuresLast24h": 0, + "touchesPersistentState": true, + "downgradeContinuityVerified": true + }, + "g5": { + "portabilityDrillPassed": true, + "dataActVerdict": "pass" + }, + "g6": { + "aiFailedDeployments": 1, + "aiTotalDeployments": 20, + "humanFailedDeployments": 2, + "humanTotalDeployments": 20 + }, + "g7": { + "taskCostUsd": "1.80", + "medianTaskCostUsd": "1.50", + "taskDurationMinutes": "12", + "executorModelFamily": "family-a", + "overseerModelFamily": "family-b" + } + }, + "voi": { + "priorSuccessProb": "2/5", + "successNetPayoffEur": "120000", + "failureLossEur": "60000", + "noActionPayoffEur": "0", + "testCostEur": "4000", + "testSensitivity": "4/5", + "testSpecificity": "3/4", + "hardLossCapEur": "80000", + "correlatedLlmRunsCount": 3 + }, + "passK": { + "k": 4, + "tasks": [ + {"taskId": "t1", "trials": 5, "successes": 5}, + {"taskId": "t2", "trials": 5, "successes": 4}, + {"taskId": "t3", "trials": 5, "successes": 3}, + {"taskId": "t4", "trials": 5, "successes": 0} + ], + "cheapModelCostUsd": "0.80", + "verifierCostUsd": "0.30", + "strongModelCostUsd": "4.00", + "gateFailRate": "0.45", + "strongRetryRate": "0.10", + "humanCodingHours": "40", + "humanReviewHours": "28", + "humanReworkHours": "12", + "acceptedTasks": 14, + "attemptedTasks": 20 + }, + "experiment": { + "controlAssignedUnits": 10000, + "treatmentAssignedUnits": 10000, + "clustersPerArm": 20, + "unitsPerCluster": 500, + "intraclusterCorrelation": "0.05", + "naiveZStat": "2.50", + "hasPostTreatmentCovariate": false, + "observedDiff": "0.02", + "diffStandardError": "0.01", + "nonInferiorityMargin": "0.05" + }, + "discovery": { + "hardConstraintsIntact": true, + "totalInterviewedPeople": 12, + "independentAccounts": 10, + "hasTechnicalUserEvidence": true, + "hasEconomicBuyerEvidence": true, + "hasVetoHolderEvidence": true, + "usesMechanicalWtpDivisor": false, + "usesRiceToOverrideSecurity": false, + "externalContactOwnerApproved": true, + "hasConsequentialCommitment": true + } +} diff --git a/contracts/rsi-governance/fixtures/tampered-k4-profile.json b/contracts/rsi-governance/fixtures/tampered-k4-profile.json new file mode 100644 index 0000000..6c713a0 --- /dev/null +++ b/contracts/rsi-governance/fixtures/tampered-k4-profile.json @@ -0,0 +1,70 @@ +{ + "apiVersion": "cloudring.org/rsi-governance/v1", + "kind": "RSICandidateProfile", + "candidate": { + "id": "cand-k4-tampered-01", + "declaredClass": "K2", + "changedPaths": [ + "pkg/controlplane/server.go", + ".github/workflows/ci.yml" + ], + "readOnlyOnly": false, + "reversibleConfigOnly": false, + "deletedOrSkippedTests": 2, + "modifiedExistingTests": true, + "modifiedTestdata": true, + "humanK4ApprovalReference": "" + }, + "gates": { + "g1": { + "evaluatorFrozenHashVerified": false, + "suitePassed": true, + "newTestFailedBeforeFix": false, + "newTestPassedAfterFix": true, + "rerunMaskingUsed": true, + "silentErrorSwallowing": 1, + "unseenHoldoutPassed": false, + "seededErrorRejected": false + }, + "g2": { + "killedMutants": 10, + "totalMutants": 20, + "fuzzTargetsPassed": false + }, + "g3": { + "sastFindings": 2, + "vulnFindings": 1, + "externalCapabilityGate": false, + "promptDelimiterOnlySecurity": true, + "explanationOnlyApproval": true + }, + "g4": { + "activeCanaries": 2, + "canaryStepPercent": "25", + "canaryMaxPercent": "80", + "requestSuccessRatePercent": "95.0", + "latencyP99Ms": "750", + "canaryDurationMinutes": "30", + "canaryFailuresLast24h": 3, + "touchesPersistentState": true, + "downgradeContinuityVerified": false + }, + "g5": { + "portabilityDrillPassed": false, + "dataActVerdict": "fail" + }, + "g6": { + "aiFailedDeployments": 5, + "aiTotalDeployments": 20, + "humanFailedDeployments": 1, + "humanTotalDeployments": 20 + }, + "g7": { + "taskCostUsd": "14.00", + "medianTaskCostUsd": "2.00", + "taskDurationMinutes": "45", + "executorModelFamily": "family-b", + "overseerModelFamily": "family-b" + } + } +} diff --git a/contracts/slice-oracles/oracles-32.json b/contracts/slice-oracles/oracles-32.json new file mode 100644 index 0000000..5b49eec --- /dev/null +++ b/contracts/slice-oracles/oracles-32.json @@ -0,0 +1,1117 @@ +{ + "apiVersion": "cloudring.org/slice-oracles/v1", + "kind": "SliceOracleBatch", + "decisionForks": { + "fork1": "1A_batch_32_oracles_autonomous_ver_acceptance_weekly_audit", + "fork2": "2A_split_core_qual_c24_after_c14_c20_no_c19_max_2_write_prs", + "fork3": "3_deferred_no_new_hardware_now_execute_no_hw_subgraph_via_rsi" + }, + "maxConcurrentWritePRsPerRepo": 2, + "ownerAcceptanceLatencyDays": 0, + "weeklyOwnerAuditHours": 1, + "lkgInvariants": [ + "S1_immutable_bootloader", + "S2_two_pinned_lkg_slots_30m_soak", + "S3_webhook_rollback_bypass", + "S4_pre_migration_state_snapshot", + "S5_meta_rsi_noop_self_check", + "S6_incident_sha_quarantine" + ], + "oracles": [ + { + "sliceId": "C01", + "title": "Reproducible public delivery baseline (partial G00)", + "goals": [ + "G00" + ], + "splitMode": false, + "coreDependencies": [], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "internal/roadmapprogram", + "internal/sourcecheck", + "roadmap" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C01.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "7.4", + "daysFactory": "1.4" + }, + { + "sliceId": "SP01", + "title": "SafePush enablement / provenance admission", + "goals": [ + "G00" + ], + "splitMode": false, + "coreDependencies": [ + "C01" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/rsi", + "pkg/sliceoracles", + "contracts/slice-oracles" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/SP01.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "14.9", + "daysFactory": "2.8" + }, + { + "sliceId": "C02", + "title": "Public isolated developer VM install (full G01)", + "goals": [ + "G01" + ], + "splitMode": false, + "coreDependencies": [ + "C01", + "SP01" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "cmd/cloudringctl", + "internal/portal" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C02.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "10.8", + "daysFactory": "2.0" + }, + { + "sliceId": "C03", + "title": "Durable project/operation API, CLI, read-only UI (partial G03)", + "goals": [ + "G03" + ], + "splitMode": false, + "coreDependencies": [ + "C02" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/transactionalstate", + "internal/api" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C03.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "19.3", + "daysFactory": "3.6" + }, + { + "sliceId": "C04", + "title": "Two developer tenants identity + isolation (partial G04/G05)", + "goals": [ + "G04", + "G05" + ], + "splitMode": false, + "coreDependencies": [ + "C03" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/identity", + "pkg/iam" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C04.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "36.5", + "daysFactory": "6.8" + }, + { + "sliceId": "C05", + "title": "Real developer VM through public Go/OCS (partial G07-G09, G15)", + "goals": [ + "G07", + "G08", + "G09", + "G15" + ], + "splitMode": false, + "coreDependencies": [ + "C04", + "C06" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/compute", + "sdk/ocsv3" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C05.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "42.1", + "daysFactory": "7.8" + }, + { + "sliceId": "C06", + "title": "Reference installation restore from off-cell copies (prep G02,G17,G18,G22,G23)", + "goals": [ + "G02", + "G17", + "G18", + "G22", + "G23" + ], + "splitMode": false, + "coreDependencies": [ + "C01" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/backup", + "pkg/dataact" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C06.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "44.1", + "daysFactory": "8.2" + }, + { + "sliceId": "C07", + "title": "GitOps origin/API routing, IPv4/IPv6 TLS (prep G02,G12,G23)", + "goals": [ + "G02", + "G12", + "G23" + ], + "splitMode": false, + "coreDependencies": [ + "C06" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/Ingress", + "pkg/network" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C07.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "26.9", + "daysFactory": "5.0" + }, + { + "sliceId": "C08", + "title": "Independent public HA empty-provider install (full G00+G02)", + "goals": [ + "G00", + "G02" + ], + "splitMode": false, + "coreDependencies": [ + "C02", + "C06", + "C07" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/kubeadm", + "pkg/installer" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C08.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "70.6", + "daysFactory": "13.1" + }, + { + "sliceId": "C09", + "title": "Production kernel, identity, IAM (full G03-G05)", + "goals": [ + "G03", + "G04", + "G05" + ], + "splitMode": false, + "coreDependencies": [ + "C03", + "C04", + "C05", + "C08" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/kernel", + "pkg/directory" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C09.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "74.4", + "daysFactory": "13.8" + }, + { + "sliceId": "C10", + "title": "Provider inventory/adoption/executor, two adapters (full G06)", + "goals": [ + "G06" + ], + "splitMode": false, + "coreDependencies": [ + "C09" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/providerports", + "pkg/inventory" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C10.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "46.0", + "daysFactory": "8.5" + }, + { + "sliceId": "C11", + "title": "OCS RC + registry, independent product from SDK (full G07+G08)", + "goals": [ + "G07", + "G08" + ], + "splitMode": false, + "coreDependencies": [ + "C05", + "C10" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/ocs", + "cmd/ocsctl", + "sdk/rust/ocs-sdk-rs" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C11.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "66.9", + "daysFactory": "12.4" + }, + { + "sliceId": "C12", + "title": "Orders/subscriptions/entitlements lifecycle (full G09)", + "goals": [ + "G09" + ], + "splitMode": false, + "coreDependencies": [ + "C11" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/orders", + "pkg/subscriptions" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C12.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "30.4", + "daysFactory": "5.6" + }, + { + "sliceId": "C13", + "title": "Metering/rating/billing ledger (full G10)", + "goals": [ + "G10" + ], + "splitMode": false, + "coreDependencies": [ + "C12" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/billing", + "pkg/metering" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C13.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "70.3", + "daysFactory": "13.0" + }, + { + "sliceId": "C14", + "title": "Equivalent API/CLI/portal/agent experience (full G11)", + "goals": [ + "G11" + ], + "splitMode": false, + "coreDependencies": [ + "C13" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "internal/portal", + "cmd/cloudringctl" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C14.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "34.5", + "daysFactory": "6.4" + }, + { + "sliceId": "C15", + "title": "OCS Network product (full G12)", + "goals": [ + "G12" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C07" + ], + "qualDependencies": [ + "C13", + "C14" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/network" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C15.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "34.5", + "daysFactory": "6.4" + }, + { + "sliceId": "C16", + "title": "OCS Volume product (full G13)", + "goals": [ + "G13" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C08" + ], + "qualDependencies": [ + "C13", + "C14" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/volume" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C16.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "36.5", + "daysFactory": "6.8" + }, + { + "sliceId": "C17", + "title": "OCS Image/Artifact product (full G14)", + "goals": [ + "G14" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C16" + ], + "qualDependencies": [ + "C13", + "C14" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/image" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C17.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "40.6", + "daysFactory": "7.5" + }, + { + "sliceId": "C18", + "title": "Complete Compute via Network/Volume/Image (full G15)", + "goals": [ + "G15" + ], + "splitMode": false, + "coreDependencies": [ + "C15", + "C16", + "C17" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/compute" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C18.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "28.9", + "daysFactory": "5.3" + }, + { + "sliceId": "C19", + "title": "Managed tenant Kubernetes (full G16)", + "goals": [ + "G16" + ], + "splitMode": false, + "coreDependencies": [ + "C18" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/kubernetes" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C19.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "38.5", + "daysFactory": "7.1" + }, + { + "sliceId": "C20", + "title": "Object storage S3 product (full G17)", + "goals": [ + "G17" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C16" + ], + "qualDependencies": [ + "C13", + "C14" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/objectstorage" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C20.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "34.5", + "daysFactory": "6.4" + }, + { + "sliceId": "C21", + "title": "Customer self-service backup (full G18)", + "goals": [ + "G18" + ], + "splitMode": false, + "coreDependencies": [ + "C19", + "C20" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/backup" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C21.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "32.4", + "daysFactory": "6.0" + }, + { + "sliceId": "C22", + "title": "Controlled access product (full G19)", + "goals": [ + "G19" + ], + "splitMode": false, + "coreDependencies": [ + "C19", + "C21" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/access" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C22.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "38.9", + "daysFactory": "7.2" + }, + { + "sliceId": "C23", + "title": "Support product (full G20)", + "goals": [ + "G20" + ], + "splitMode": false, + "coreDependencies": [ + "C22" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/support" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C23.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "43.3", + "daysFactory": "8.0" + }, + { + "sliceId": "C24", + "title": "External integration product (full G21)", + "goals": [ + "G21" + ], + "splitMode": false, + "coreDependencies": [ + "C14" + ], + "qualDependencies": [], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/external" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C24.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "43.3", + "daysFactory": "8.0" + }, + { + "sliceId": "C25", + "title": "Resilience campaign on adjacent signed releases (full G23)", + "goals": [ + "G23" + ], + "splitMode": false, + "coreDependencies": [ + "C01", + "SP01", + "C02", + "C03", + "C04", + "C05", + "C06", + "C07", + "C08", + "C09", + "C10", + "C11", + "C12", + "C13", + "C14", + "C15", + "C16", + "C17", + "C18", + "C19", + "C20", + "C21", + "C22", + "C23", + "C24", + "DBAAS", + "DNSAAS", + "FILESAAS" + ], + "qualDependencies": [], + "requiresFork3Hardware": true, + "allowedPaths": [ + "pkg/resilience" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C25.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "32.9", + "daysFactory": "6.1" + }, + { + "sliceId": "C26", + "title": "Independent engineer operations, 14-day toil (full G22)", + "goals": [ + "G22" + ], + "splitMode": false, + "coreDependencies": [ + "C25" + ], + "qualDependencies": [], + "requiresFork3Hardware": true, + "allowedPaths": [ + "pkg/operations" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C26.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "36.5", + "daysFactory": "6.8" + }, + { + "sliceId": "C27", + "title": "Second bare-metal provider reproduction (full G24)", + "goals": [ + "G24" + ], + "splitMode": false, + "coreDependencies": [ + "C26" + ], + "qualDependencies": [], + "requiresFork3Hardware": true, + "allowedPaths": [ + "pkg/portability" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C27.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "46.0", + "daysFactory": "8.5" + }, + { + "sliceId": "C28", + "title": "Final audit and 1.0 release (full G27)", + "goals": [ + "G27" + ], + "splitMode": false, + "coreDependencies": [ + "C01", + "SP01", + "C02", + "C03", + "C04", + "C05", + "C06", + "C07", + "C08", + "C09", + "C10", + "C11", + "C12", + "C13", + "C14", + "C15", + "C16", + "C17", + "C18", + "C19", + "C20", + "C21", + "C22", + "C23", + "C24", + "C25", + "C26", + "C27", + "DBAAS", + "DNSAAS", + "FILESAAS" + ], + "qualDependencies": [], + "requiresFork3Hardware": true, + "allowedPaths": [ + "roadmap/state" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C28.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "54.1", + "daysFactory": "10.0" + }, + { + "sliceId": "DBAAS", + "title": "OCS Managed PostgreSQL (CNPG) product", + "goals": [ + "G07", + "G10", + "G18" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C16" + ], + "qualDependencies": [ + "C13", + "C14", + "C21" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/dbaas" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/DBAAS.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "34.5", + "daysFactory": "6.4" + }, + { + "sliceId": "DNSAAS", + "title": "OCS Managed DNS product", + "goals": [ + "G07", + "G10", + "G12" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C07" + ], + "qualDependencies": [ + "C13", + "C14", + "C15" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/dnsaas" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/DNSAAS.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "34.5", + "daysFactory": "6.4" + }, + { + "sliceId": "FILESAAS", + "title": "OCS Shared File Storage (RWX NFS/CephFS) product", + "goals": [ + "G07", + "G10", + "G13" + ], + "splitMode": true, + "coreDependencies": [ + "C11", + "C16" + ], + "qualDependencies": [ + "C13", + "C14" + ], + "requiresFork3Hardware": false, + "allowedPaths": [ + "pkg/products/filesaas" + ], + "autonomyClass": "K2", + "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/FILESAAS.json", + "requiredGates": [ + "G1", + "G2", + "G3", + "G4", + "G5", + "G6", + "G7", + "S1_S6_LKG" + ], + "daysHist": "34.5", + "daysFactory": "6.4" + } + ] +} diff --git a/pkg/rsi/rsi.go b/pkg/rsi/rsi.go new file mode 100644 index 0000000..f2d20d8 --- /dev/null +++ b/pkg/rsi/rsi.go @@ -0,0 +1,1192 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +// Package rsi implements CloudRING's scientifically grounded Recursive +// Self-Improvement (RSI) governance and decision-science evaluator. It +// classifies candidate changes into autonomy classes K0-K4, evaluates the +// mandatory safety and quality gates G1-G7 under an external frozen evaluator +// contract, computes exact rational Value of Information (VOI / EVSI), +// unbiased task-averaged combinatorial pass^k and pass@k reliability, +// cluster-robust experimental design diagnostics (SRM, DEFF, non-inferiority, +// TOST), and Theory-of-Value stage-gate validation. +package rsi + +import ( + "errors" + "fmt" + "math" + "math/big" + "path" + "sort" + "strings" + + "github.com/opencloudtech/CloudRING/internal/strictjson" +) + +const ( + ProfileAPIVersion = "cloudring.org/rsi-governance/v1" + ProfileKind = "RSICandidateProfile" + ReportSchema = "cloudring.rsi-governance-report/v1" + + ClassK0 = "K0" + ClassK1 = "K1" + ClassK2 = "K2" + ClassK3 = "K3" + ClassK4 = "K4" + + VerdictPass = "pass" + VerdictFail = "fail" + + StatusPass = "pass" + StatusFail = "fail" + + KillSwitchNone = "none" + KillSwitchL1 = "L1_suspend_gitops_canary" + KillSwitchL2 = "L2_revoke_gateway_openbao_leases" + KillSwitchL3 = "L3_vap_ai_freeze_deny" + + maxProfileBytes = 1 << 20 +) + +var ( + ErrInvalidProfile = errors.New("rsi: invalid candidate profile") + + minMutationKillScore = big.NewRat(4, 5) // 80% diff-based mutation kill threshold (G2) + maxCanaryStepPct = big.NewRat(10, 100) // 10% max initial/step canary weight for AI (G4) + maxCanaryWeightPct = big.NewRat(50, 100) // 50% max canary weight before promotion (G4) + minCanarySuccessRate = big.NewRat(99, 100) // 99% minimum request success rate (G4) + maxCanaryLatencyMs = big.NewRat(500, 1) // 500 ms maximum p99 latency during canary (G4) + sloMonthlyBudgetMin = big.NewRat(432, 10) // 43.2 minutes (99.9% over 30 days) + maxCanarySLOBurnFrac = big.NewRat(1, 10) // at most 10% of monthly error budget per canary + maxAICFR = big.NewRat(3, 20) // 15% maximum rolling AI Change Fail Rate (G6) + maxTaskCostUSD = big.NewRat(10, 1) // $10.00 default cap per autonomous task (G7) + maxTaskDurationMin = big.NewRat(30, 1) // 30 minutes default cap per autonomous task (G7) + maxCostBurnMult = big.NewRat(2, 1) // 2x median task cost burn trigger (G7) + srmChiSquareCrit = big.NewRat(108276, 10000) // chi^2(1) p=0.001 critical value ~10.8276 +) + +// CandidateProfile declares the observable verification receipts and decision +// inputs for a candidate change evaluated by the external frozen evaluator. +type CandidateProfile struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Candidate CandidateMetadata `json:"candidate"` + Gates GateReceipts `json:"gates"` + VOI *VOIInput `json:"voi,omitempty"` + PassK *PassKInput `json:"passK,omitempty"` + Experiment *ExperimentInput `json:"experiment,omitempty"` + Discovery *DiscoveryInput `json:"discovery,omitempty"` +} + +// CandidateMetadata identifies the diff and declared target class. +type CandidateMetadata struct { + ID string `json:"id"` + DeclaredClass string `json:"declaredClass"` + ChangedPaths []string `json:"changedPaths"` + ReadOnlyOnly bool `json:"readOnlyOnly"` + ReversibleConfigOnly bool `json:"reversibleConfigOnly"` + DeletedOrSkippedTests int `json:"deletedOrSkippedTests"` + ModifiedExistingTests bool `json:"modifiedExistingTests"` + ModifiedTestdata bool `json:"modifiedTestdata"` + HumanK4ApprovalReference string `json:"humanK4ApprovalReference"` +} + +// GateReceipts holds externally measured G1-G7 evidence (never self-reported by the candidate). +type GateReceipts struct { + G1 G1Receipt `json:"g1"` + G2 G2Receipt `json:"g2"` + G3 G3Receipt `json:"g3"` + G4 G4Receipt `json:"g4"` + G5 G5Receipt `json:"g5"` + G6 G6Receipt `json:"g6"` + G7 G7Receipt `json:"g7"` +} + +// G1Receipt captures test suite execution and anti-tampering checks. +type G1Receipt struct { + EvaluatorFrozenHashVerified bool `json:"evaluatorFrozenHashVerified"` + SuitePassed bool `json:"suitePassed"` + NewTestFailedBeforeFix bool `json:"newTestFailedBeforeFix"` + NewTestPassedAfterFix bool `json:"newTestPassedAfterFix"` + RerunMaskingUsed bool `json:"rerunMaskingUsed"` + SilentErrorSwallowing int `json:"silentErrorSwallowing"` + UnseenHoldoutPassed bool `json:"unseenHoldoutPassed"` + SeededErrorRejected bool `json:"seededErrorRejected"` +} + +// G2Receipt captures diff-based mutation testing on changed non-arid lines and fuzzing. +type G2Receipt struct { + KilledMutants int `json:"killedMutants"` + TotalMutants int `json:"totalMutants"` + FuzzTargetsPassed bool `json:"fuzzTargetsPassed"` +} + +// G3Receipt captures static security analysis, vulnerability scanning, and agent capability boundaries. +type G3Receipt struct { + SASTFindings int `json:"sastFindings"` + VulnFindings int `json:"vulnFindings"` + ExternalCapabilityGate bool `json:"externalCapabilityGate"` + PromptDelimiterOnlySecurity bool `json:"promptDelimiterOnlySecurity"` + ExplanationOnlyApproval bool `json:"explanationOnlyApproval"` +} + +// G4Receipt captures progressive canary parameters and stateful downgrade continuity. +type G4Receipt struct { + ActiveCanaries int `json:"activeCanaries"` + CanaryStepPercent string `json:"canaryStepPercent"` + CanaryMaxPercent string `json:"canaryMaxPercent"` + RequestSuccessRatePercent string `json:"requestSuccessRatePercent"` + LatencyP99Ms string `json:"latencyP99Ms"` + CanaryDurationMinutes string `json:"canaryDurationMinutes"` + CanaryFailuresLast24h int `json:"canaryFailuresLast24h"` + TouchesPersistentState bool `json:"touchesPersistentState"` + DowngradeContinuityVerified bool `json:"downgradeContinuityVerified"` +} + +// G5Receipt captures tenant portability and EU Data Act switching drill verification. +type G5Receipt struct { + PortabilityDrillPassed bool `json:"portabilityDrillPassed"` + DataActVerdict string `json:"dataActVerdict"` +} + +// G6Receipt captures rolling DORA Change Fail Rate (CFR) for AI and human deployment lines. +type G6Receipt struct { + AIFailedDeployments int `json:"aiFailedDeployments"` + AITotalDeployments int `json:"aiTotalDeployments"` + HumanFailedDeployments int `json:"humanFailedDeployments"` + HumanTotalDeployments int `json:"humanTotalDeployments"` +} + +// G7Receipt captures compute budget, wall-clock duration, and cross-family overseer isolation. +type G7Receipt struct { + TaskCostUSD string `json:"taskCostUsd"` + MedianTaskCostUSD string `json:"medianTaskCostUsd"` + TaskDurationMinutes string `json:"taskDurationMinutes"` + ExecutorModelFamily string `json:"executorModelFamily"` + OverseerModelFamily string `json:"overseerModelFamily"` +} + +// VOIInput defines a Bayesian Value of Information (EVPI / EVSI) decision problem (C-20). +type VOIInput struct { + PriorSuccessProb string `json:"priorSuccessProb"` + SuccessNetPayoffEUR string `json:"successNetPayoffEur"` + FailureLossEUR string `json:"failureLossEur"` + NoActionPayoffEUR string `json:"noActionPayoffEur"` + TestCostEUR string `json:"testCostEur"` + TestSensitivity string `json:"testSensitivity"` + TestSpecificity string `json:"testSpecificity"` + HardLossCapEUR string `json:"hardLossCapEur"` + CorrelatedLLMRunsCount int `json:"correlatedLlmRunsCount"` +} + +// VOIReport holds exact rational Bayesian decision and EVSI results. +type VOIReport struct { + NoTestBestAction string `json:"noTestBestAction"` + LaunchNowEVEUR string `json:"launchNowEvEur"` + NoTestBestEVEUR string `json:"noTestBestEvEur"` + PositiveTestProb string `json:"positiveTestProb"` + PosteriorSuccessGivenPos string `json:"posteriorSuccessGivenPos"` + PosteriorSuccessGivenNeg string `json:"posteriorSuccessGivenNeg"` + EVPIEUR string `json:"evpiEur"` + GrossPolicyEVEUR string `json:"grossPolicyEvEur"` + GrossEVSIEUR string `json:"grossEvsiEur"` + NetPolicyEVEUR string `json:"netPolicyEvEur"` + NetIncrementalEVEUR string `json:"netIncrementalEvEur"` + FeasibleUnderLossCap bool `json:"feasibleUnderLossCap"` + EffectiveNetEVSIUnderCap string `json:"effectiveNetEvsiUnderCapEur"` + ExtremizationRejected bool `json:"extremizationRejected"` + RecommendedAction string `json:"recommendedAction"` +} + +// TaskTrialCount records n trials and c successes on task i. +type TaskTrialCount struct { + TaskID string `json:"taskId"` + Trials int `json:"trials"` + Successes int `json:"successes"` +} + +// PassKInput defines heterogeneous multi-trial agent evaluation and cascade economics (C-21). +type PassKInput struct { + K int `json:"k"` + Tasks []TaskTrialCount `json:"tasks"` + CheapModelCostUSD string `json:"cheapModelCostUsd"` + VerifierCostUSD string `json:"verifierCostUsd"` + StrongModelCostUSD string `json:"strongModelCostUsd"` + GateFailRate string `json:"gateFailRate"` + StrongRetryRate string `json:"strongRetryRate"` + HumanCodingHours string `json:"humanCodingHours"` + HumanReviewHours string `json:"humanReviewHours"` + HumanReworkHours string `json:"humanReworkHours"` + AcceptedTasks int `json:"acceptedTasks"` + AttemptedTasks int `json:"attemptedTasks"` +} + +// PassKReport holds unbiased combinatorial pass^k, pass@k, cascade cost, and labor per accepted task. +type PassKReport struct { + K int `json:"k"` + MeanPass1 string `json:"meanPass1"` + UnbiasedCombinatorialPassK string `json:"unbiasedCombinatorialPassK"` + UnbiasedCombinatorialPassAt string `json:"unbiasedCombinatorialPassAtK"` + NaiveHomogeneousPassK string `json:"naiveHomogeneousPassK"` + NaiveHomogeneousPassAtK string `json:"naiveHomogeneousPassAtK"` + FullCascadeCostPerTaskUSD string `json:"fullCascadeCostPerTaskUsd"` + NaiveCascadeCostPerTaskUSD string `json:"naiveCascadeCostPerTaskUsd"` + TotalHumanHours string `json:"totalHumanHours"` + HumanHoursPerAcceptedTask string `json:"humanHoursPerAcceptedTask"` + AcceptanceRate string `json:"acceptanceRate"` +} + +// ExperimentInput defines a B2B cluster/crossover experiment diagnostic check (C-19). +type ExperimentInput struct { + ControlAssignedUnits int `json:"controlAssignedUnits"` + TreatmentAssignedUnits int `json:"treatmentAssignedUnits"` + ClustersPerArm int `json:"clustersPerArm"` + UnitsPerCluster int `json:"unitsPerCluster"` + IntraclusterCorrelation string `json:"intraclusterCorrelation"` + NaiveZStat string `json:"naiveZStat"` + HasPostTreatmentCovariate bool `json:"hasPostTreatmentCovariate"` + ObservedDiff string `json:"observedDiff"` + DiffStandardError string `json:"diffStandardError"` + NonInferiorityMargin string `json:"nonInferiorityMargin"` +} + +// ExperimentReport holds SRM, cluster DEFF, and non-inferiority / TOST verdicts. +type ExperimentReport struct { + SRMChiSquare string `json:"srmChiSquare"` + SRMDetected bool `json:"srmDetected"` + DesignEffectDEFF string `json:"designEffectDeff"` + EffectiveUnitsPerArm string `json:"effectiveUnitsPerArm"` + SEInflationFactor string `json:"seInflationFactor"` + ClusterAdjustedTStat string `json:"clusterAdjustedTStat"` + ClusterSignificantAt005 bool `json:"clusterSignificantAt005"` + CUPEDValid bool `json:"cupedValid"` + NonInferiorityZ string `json:"nonInferiorityZ"` + TOSTUpperZ string `json:"tostUpperZ"` + NonInferiorityProvenAt005 bool `json:"nonInferiorityProvenAt005"` + TOSTEquivalenceProven005 bool `json:"tostEquivalenceProvenAt005"` + Verdict string `json:"verdict"` +} + +// DiscoveryInput defines a Theory-of-Value stage-gate validation check (C-18). +type DiscoveryInput struct { + HardConstraintsIntact bool `json:"hardConstraintsIntact"` + TotalInterviewedPeople int `json:"totalInterviewedPeople"` + IndependentAccounts int `json:"independentAccounts"` + HasTechnicalUserEvidence bool `json:"hasTechnicalUserEvidence"` + HasEconomicBuyerEvidence bool `json:"hasEconomicBuyerEvidence"` + HasVetoHolderEvidence bool `json:"hasVetoHolderEvidence"` + UsesMechanicalWTPDivisor bool `json:"usesMechanicalWtpDivisor"` + UsesRICEToOverrideSecurity bool `json:"usesRiceToOverrideSecurity"` + ExternalContactOwnerApproved bool `json:"externalContactOwnerApproved"` + HasConsequentialCommitment bool `json:"hasConsequentialCommitment"` +} + +// DiscoveryReport holds the Theory-of-Value stage-gate evaluation. +type DiscoveryReport struct { + AccountInflationDetected bool `json:"accountInflationDetected"` + RoleStratificationComplete bool `json:"roleStratificationComplete"` + MethodologicallyValid bool `json:"methodologicallyValid"` + StageGateAuthorized bool `json:"stageGateAuthorized"` + Reasons []string `json:"reasons"` +} + +// Classification describes the resolved autonomy class and escalation reasons. +type Classification struct { + DeclaredClass string `json:"declaredClass"` + EffectiveClass string `json:"effectiveClass"` + HumanApprovalRequired bool `json:"humanApprovalRequired"` + HoldoutRequired bool `json:"holdoutRequired"` + EscalationReasons []string `json:"escalationReasons"` +} + +// CheckResult records a single deterministic gate evaluation. +type CheckResult struct { + ID string `json:"id"` + Status string `json:"status"` + Summary string `json:"summary"` +} + +// Report is the complete machine-verifiable RSI governance and decision report. +type Report struct { + SchemaVersion string `json:"schemaVersion"` + APIVersion string `json:"apiVersion"` + CandidateID string `json:"candidateId"` + Classification Classification `json:"classification"` + Checks []CheckResult `json:"checks"` + Passed int `json:"passed"` + Failed int `json:"failed"` + MaxAutonomousClass string `json:"maxAutonomousClass"` + KillSwitchLevel string `json:"killSwitchLevel"` + KillSwitchReasons []string `json:"killSwitchReasons"` + MutationKillRatio string `json:"mutationKillRatio"` + AIChangeFailRate string `json:"aiChangeFailRate"` + HumanChangeFailRate string `json:"humanChangeFailRate"` + CanarySLOBurnFraction string `json:"canarySloBurnFraction"` + VOI *VOIReport `json:"voi,omitempty"` + PassK *PassKReport `json:"passK,omitempty"` + Experiment *ExperimentReport `json:"experiment,omitempty"` + Discovery *DiscoveryReport `json:"discovery,omitempty"` + Verdict string `json:"verdict"` +} + +// ParseProfile strictly decodes a CandidateProfile from JSON bytes. +func ParseProfile(raw []byte) (CandidateProfile, error) { + if len(raw) == 0 || len(raw) > maxProfileBytes { + return CandidateProfile{}, fmt.Errorf("%w: profile size out of bounds", ErrInvalidProfile) + } + var profile CandidateProfile + if err := strictjson.DecodeExact(raw, &profile); err != nil { + return CandidateProfile{}, fmt.Errorf("%w: %v", ErrInvalidProfile, err) + } + if profile.APIVersion != ProfileAPIVersion || profile.Kind != ProfileKind { + return CandidateProfile{}, fmt.Errorf("%w: expected apiVersion=%q kind=%q", ErrInvalidProfile, ProfileAPIVersion, ProfileKind) + } + if strings.TrimSpace(profile.Candidate.ID) == "" { + return CandidateProfile{}, fmt.Errorf("%w: candidate.id is required", ErrInvalidProfile) + } + if !validClass(profile.Candidate.DeclaredClass) { + return CandidateProfile{}, fmt.Errorf("%w: invalid declaredClass %q", ErrInvalidProfile, profile.Candidate.DeclaredClass) + } + if len(profile.Candidate.ChangedPaths) == 0 && !profile.Candidate.ReadOnlyOnly { + return CandidateProfile{}, fmt.Errorf("%w: changedPaths required for non-K0 candidate", ErrInvalidProfile) + } + return profile, nil +} + +func validClass(c string) bool { + switch c { + case ClassK0, ClassK1, ClassK2, ClassK3, ClassK4: + return true + default: + return false + } +} + +func classRank(c string) int { + switch c { + case ClassK0: + return 0 + case ClassK1: + return 1 + case ClassK2: + return 2 + case ClassK3: + return 3 + case ClassK4: + return 4 + default: + return 4 + } +} + +// ClassifyCandidate determines the effective change class K0-K4 and enforces +// DGM / ImpossibleBench anti-reward-hacking escalations. +func ClassifyCandidate(meta CandidateMetadata) (Classification, error) { + if !validClass(meta.DeclaredClass) { + return Classification{}, fmt.Errorf("%w: unknown class %q", ErrInvalidProfile, meta.DeclaredClass) + } + effective := meta.DeclaredClass + var reasons []string + + touchesProd := false + touchesTests := false + touchesAI := false + touchesK4 := false + + for _, rawPath := range meta.ChangedPaths { + lowerRaw := strings.ToLower(rawPath) + if strings.Contains(rawPath, "\x00") || strings.Contains(lowerRaw, "%2e") || strings.Contains(lowerRaw, "%2f") || strings.Contains(lowerRaw, "%5c") { + touchesK4 = true + reasons = append(reasons, fmt.Sprintf("encoded_or_null_path:%s", rawPath)) + continue + } + trimmed := strings.TrimSpace(rawPath) + if trimmed == "" || strings.Contains(trimmed, `\`) { + touchesK4 = true + reasons = append(reasons, fmt.Sprintf("invalid_or_backslash_path:%s", rawPath)) + continue + } + cleaned := path.Clean(trimmed) + if cleaned == "." || cleaned == "/" || strings.HasPrefix(cleaned, "../") || strings.HasPrefix(cleaned, "/") || strings.Contains(trimmed, "..") { + touchesK4 = true + reasons = append(reasons, fmt.Sprintf("path_traversal_or_absolute:%s", rawPath)) + continue + } + lower := strings.ToLower(cleaned) + segments := strings.Split(lower, "/") + for _, seg := range segments { + if seg == ".github" || seg == "evaluator" || seg == "policies" || seg == "codeowners" || seg == "openbao" || seg == "grants" || seg == "verify" { + touchesK4 = true + reasons = append(reasons, fmt.Sprintf("protected_k4_path:%s", lower)) + break + } + } + if lower == "ops/stop" || strings.HasSuffix(lower, "/ops/stop") || strings.HasSuffix(lower, "codeowners") { + touchesK4 = true + reasons = append(reasons, fmt.Sprintf("protected_control_file:%s", lower)) + } + if strings.HasPrefix(lower, "pkg/rsi/") || lower == "pkg/rsi" || strings.Contains(lower, "/rsi/") || strings.HasPrefix(lower, "skills/") || strings.HasPrefix(lower, "agents/") || strings.HasPrefix(lower, "prompts/") { + touchesAI = true + } + if strings.HasSuffix(lower, "_test.go") || strings.Contains(lower, "/testdata/") || strings.HasSuffix(lower, "/testdata") { + touchesTests = true + } else if strings.HasSuffix(lower, ".go") || strings.HasPrefix(lower, "cmd/") || strings.HasPrefix(lower, "pkg/") || strings.HasPrefix(lower, "internal/") || strings.HasPrefix(lower, "sdk/") || strings.HasPrefix(lower, "deploy/") { + touchesProd = true + } + } + + if !meta.ReadOnlyOnly && classRank(effective) < classRank(ClassK1) { + effective = ClassK1 + } + if (touchesProd || !meta.ReversibleConfigOnly) && !meta.ReadOnlyOnly && classRank(effective) < classRank(ClassK2) { + effective = ClassK2 + } + if touchesAI && classRank(effective) < classRank(ClassK3) { + effective = ClassK3 + reasons = append(reasons, "touches_ai_or_rsi_layer_requires_k3") + } + if meta.DeletedOrSkippedTests > 0 { + touchesK4 = true + reasons = append(reasons, "deleted_or_skipped_tests_requires_k4") + } + if touchesProd && (meta.ModifiedExistingTests || meta.ModifiedTestdata) { + touchesK4 = true + reasons = append(reasons, "co_modified_production_and_existing_tests_or_testdata_requires_k4") + } + if touchesTests && meta.ModifiedTestdata { + touchesK4 = true + reasons = append(reasons, "modified_testdata_fixtures_requires_k4") + } + if touchesK4 { + effective = ClassK4 + } + + sort.Strings(reasons) + reasons = compactStrings(reasons) + + return Classification{ + DeclaredClass: meta.DeclaredClass, + EffectiveClass: effective, + HumanApprovalRequired: effective == ClassK4, + HoldoutRequired: classRank(effective) >= classRank(ClassK3), + EscalationReasons: reasons, + }, nil +} + +func compactStrings(in []string) []string { + if len(in) == 0 { + return []string{} + } + out := make([]string, 0, len(in)) + for i, s := range in { + if i == 0 || s != in[i-1] { + out = append(out, s) + } + } + return out +} + +func parseNonNegativeRat(raw string, field string) (*big.Rat, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return nil, fmt.Errorf("%w: %s is empty", ErrInvalidProfile, field) + } + r, ok := new(big.Rat).SetString(trimmed) + if !ok || r.Sign() < 0 { + return nil, fmt.Errorf("%w: %s must be a non-negative rational or decimal, got %q", ErrInvalidProfile, field, raw) + } + return r, nil +} + +func parseSignedRat(raw string, field string) (*big.Rat, error) { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return nil, fmt.Errorf("%w: %s is empty", ErrInvalidProfile, field) + } + r, ok := new(big.Rat).SetString(trimmed) + if !ok { + return nil, fmt.Errorf("%w: %s must be a valid rational or decimal, got %q", ErrInvalidProfile, field, raw) + } + return r, nil +} + +func parseProbabilityRat(raw string, field string) (*big.Rat, error) { + r, err := parseNonNegativeRat(raw, field) + if err != nil { + return nil, err + } + if r.Cmp(big.NewRat(1, 1)) > 0 { + return nil, fmt.Errorf("%w: %s must be in [0, 1], got %q", ErrInvalidProfile, field, raw) + } + return r, nil +} + +// EvaluateCandidate runs the complete G1-G7 governance evaluation and any +// attached decision-science checks (VOI, PassK, Experiment, Discovery). +func EvaluateCandidate(profile CandidateProfile) (Report, error) { + cls, err := ClassifyCandidate(profile.Candidate) + if err != nil { + return Report{}, err + } + + var checks []CheckResult + var killReasons []string + killLevel := KillSwitchNone + maxAutoClass := ClassK3 + + // Gate K4: Human approval check when escalated or declared K4. + k4Pass := true + k4Summary := fmt.Sprintf("effective class %s within autonomous boundary", cls.EffectiveClass) + if cls.EffectiveClass == ClassK4 { + if strings.TrimSpace(profile.Candidate.HumanK4ApprovalReference) == "" { + k4Pass = false + k4Summary = fmt.Sprintf("effective class K4 requires explicit human approval (%s)", strings.Join(cls.EscalationReasons, ", ")) + if profile.Candidate.DeletedOrSkippedTests > 0 || profile.Candidate.ModifiedExistingTests || profile.Candidate.ModifiedTestdata { + killLevel = escalateKillSwitch(killLevel, KillSwitchL3) + killReasons = append(killReasons, "test_or_evaluator_tampering_detected") + } + } else { + k4Summary = fmt.Sprintf("effective class K4 authorized by human reference %s", profile.Candidate.HumanK4ApprovalReference) + } + } + checks = append(checks, makeCheck("k4_human_boundary", k4Pass, k4Summary)) + + // Gate G1: Red->Green test & evaluator hash integrity. + g1 := profile.Gates.G1 + g1Pass := g1.EvaluatorFrozenHashVerified && g1.SuitePassed && !g1.RerunMaskingUsed && g1.SilentErrorSwallowing == 0 + if classRank(cls.EffectiveClass) >= classRank(ClassK2) { + g1Pass = g1Pass && g1.NewTestFailedBeforeFix && g1.NewTestPassedAfterFix + } + if cls.HoldoutRequired { + g1Pass = g1Pass && g1.UnseenHoldoutPassed && g1.SeededErrorRejected + } + g1Summary := fmt.Sprintf("frozenHash=%t suite=%t redGreen=%t/%t holdout=%t seededError=%t rerunMask=%t silentSwallow=%d", + g1.EvaluatorFrozenHashVerified, g1.SuitePassed, g1.NewTestFailedBeforeFix, g1.NewTestPassedAfterFix, + g1.UnseenHoldoutPassed, g1.SeededErrorRejected, g1.RerunMaskingUsed, g1.SilentErrorSwallowing) + checks = append(checks, makeCheck("g1_red_green_and_test_integrity", g1Pass, g1Summary)) + + // Gate G2: Diff-based mutation score >= 80% and fuzz targets. + g2 := profile.Gates.G2 + mutRatio := big.NewRat(0, 1) + g2Pass := g2.FuzzTargetsPassed + if classRank(cls.EffectiveClass) >= classRank(ClassK2) { + if g2.TotalMutants <= 0 || g2.KilledMutants < 0 || g2.KilledMutants > g2.TotalMutants { + g2Pass = false + } else { + mutRatio = big.NewRat(int64(g2.KilledMutants), int64(g2.TotalMutants)) + if mutRatio.Cmp(minMutationKillScore) < 0 { + g2Pass = false + } + } + } else if g2.TotalMutants > 0 && g2.KilledMutants >= 0 && g2.KilledMutants <= g2.TotalMutants { + mutRatio = big.NewRat(int64(g2.KilledMutants), int64(g2.TotalMutants)) + } + g2Summary := fmt.Sprintf("killed=%d/%d (%s) fuzzPassed=%t", g2.KilledMutants, g2.TotalMutants, mutRatio.FloatString(4), g2.FuzzTargetsPassed) + checks = append(checks, makeCheck("g2_diff_mutation_and_fuzz", g2Pass, g2Summary)) + + // Gate G3: Security SAST, govulncheck, and external capability boundary. + g3 := profile.Gates.G3 + g3Pass := g3.SASTFindings == 0 && g3.VulnFindings == 0 && g3.ExternalCapabilityGate && !g3.PromptDelimiterOnlySecurity && !g3.ExplanationOnlyApproval + g3Summary := fmt.Sprintf("sast=%d vuln=%d externalCapGate=%t delimiterOnly=%t explanationOnly=%t", + g3.SASTFindings, g3.VulnFindings, g3.ExternalCapabilityGate, g3.PromptDelimiterOnlySecurity, g3.ExplanationOnlyApproval) + checks = append(checks, makeCheck("g3_security_and_capability_boundary", g3Pass, g3Summary)) + + // Gate G4: Canary SLO burn and stateful downgrade continuity. + g4 := profile.Gates.G4 + stepPct, err := parseNonNegativeRat(g4.CanaryStepPercent, "gates.g4.canaryStepPercent") + if err != nil { + return Report{}, err + } + maxPct, err := parseNonNegativeRat(g4.CanaryMaxPercent, "gates.g4.canaryMaxPercent") + if err != nil { + return Report{}, err + } + succPct, err := parseNonNegativeRat(g4.RequestSuccessRatePercent, "gates.g4.requestSuccessRatePercent") + if err != nil { + return Report{}, err + } + latMs, err := parseNonNegativeRat(g4.LatencyP99Ms, "gates.g4.latencyP99Ms") + if err != nil { + return Report{}, err + } + durMin, err := parseNonNegativeRat(g4.CanaryDurationMinutes, "gates.g4.canaryDurationMinutes") + if err != nil { + return Report{}, err + } + stepFrac := new(big.Rat).Quo(stepPct, big.NewRat(100, 1)) + maxFrac := new(big.Rat).Quo(maxPct, big.NewRat(100, 1)) + succFrac := new(big.Rat).Quo(succPct, big.NewRat(100, 1)) + // Worst-case error-minutes during one canary step = stepFrac * durMin; burn fraction = (stepFrac * durMin) / 43.2 + errMinutes := new(big.Rat).Mul(stepFrac, durMin) + burnFrac := new(big.Rat).Quo(errMinutes, sloMonthlyBudgetMin) + + g4Pass := g4.ActiveCanaries >= 1 && g4.ActiveCanaries <= 1 && + stepFrac.Cmp(maxCanaryStepPct) <= 0 && + maxFrac.Cmp(maxCanaryWeightPct) <= 0 && + succFrac.Cmp(minCanarySuccessRate) >= 0 && + latMs.Cmp(maxCanaryLatencyMs) <= 0 && + burnFrac.Cmp(maxCanarySLOBurnFrac) <= 0 && + g4.CanaryFailuresLast24h < 3 && + (!g4.TouchesPersistentState || g4.DowngradeContinuityVerified) + + if g4.CanaryFailuresLast24h >= 3 { + killLevel = escalateKillSwitch(killLevel, KillSwitchL1) + killReasons = append(killReasons, "three_or_more_canary_failures_in_24h") + } + g4Summary := fmt.Sprintf("active=%d step=%s%% max=%s%% success=%s%% p99=%sms burnFrac=%s stateful=%t downgradeOK=%t", + g4.ActiveCanaries, stepPct.FloatString(1), maxPct.FloatString(1), succPct.FloatString(2), latMs.FloatString(1), + burnFrac.FloatString(4), g4.TouchesPersistentState, g4.DowngradeContinuityVerified) + checks = append(checks, makeCheck("g4_canary_slo_and_stateful_downgrade", g4Pass, g4Summary)) + + // Gate G5: Portability and EU Data Act switching drill. + g5 := profile.Gates.G5 + g5Pass := g5.PortabilityDrillPassed && g5.DataActVerdict == VerdictPass + g5Summary := fmt.Sprintf("portabilityDrill=%t dataActVerdict=%s", g5.PortabilityDrillPassed, g5.DataActVerdict) + checks = append(checks, makeCheck("g5_portability_and_switching_drill", g5Pass, g5Summary)) + + // Gate G6: Rolling DORA Change Fail Rate (CFR) circuit breaker. + g6 := profile.Gates.G6 + aiCFR := big.NewRat(0, 1) + humanCFR := big.NewRat(0, 1) + g6Pass := true + if g6.AITotalDeployments <= 0 || g6.AIFailedDeployments < 0 || g6.AIFailedDeployments > g6.AITotalDeployments || + g6.HumanTotalDeployments < 0 || g6.HumanFailedDeployments < 0 || g6.HumanFailedDeployments > g6.HumanTotalDeployments { + g6Pass = false + } else { + aiCFR = big.NewRat(int64(g6.AIFailedDeployments), int64(g6.AITotalDeployments)) + if aiCFR.Cmp(maxAICFR) > 0 { + g6Pass = false + } + if g6.HumanTotalDeployments > 0 { + humanCFR = big.NewRat(int64(g6.HumanFailedDeployments), int64(g6.HumanTotalDeployments)) + if aiCFR.Cmp(humanCFR) > 0 { + g6Pass = false + } + } + } + if !g6Pass { + maxAutoClass = ClassK1 + killLevel = escalateKillSwitch(killLevel, KillSwitchL1) + killReasons = append(killReasons, "g6_cfr_budget_exceeded_autonomy_downgraded_to_k1") + } + g6Summary := fmt.Sprintf("aiCFR=%d/%d (%s) humanCFR=%d/%d (%s) maxAutoClass=%s", + g6.AIFailedDeployments, g6.AITotalDeployments, aiCFR.FloatString(4), + g6.HumanFailedDeployments, g6.HumanTotalDeployments, humanCFR.FloatString(4), maxAutoClass) + checks = append(checks, makeCheck("g6_dora_cfr_circuit_breaker", g6Pass, g6Summary)) + + // Gate G7: Task budget, duration, burn multiplier, and cross-family overseer. + g7 := profile.Gates.G7 + costUSD, err := parseNonNegativeRat(g7.TaskCostUSD, "gates.g7.taskCostUsd") + if err != nil { + return Report{}, err + } + medCostUSD, err := parseNonNegativeRat(g7.MedianTaskCostUSD, "gates.g7.medianTaskCostUsd") + if err != nil { + return Report{}, err + } + taskDurMin, err := parseNonNegativeRat(g7.TaskDurationMinutes, "gates.g7.taskDurationMinutes") + if err != nil { + return Report{}, err + } + execFam := strings.ToLower(strings.TrimSpace(g7.ExecutorModelFamily)) + overFam := strings.ToLower(strings.TrimSpace(g7.OverseerModelFamily)) + burnLimit := new(big.Rat).Mul(medCostUSD, maxCostBurnMult) + burnExceeded := medCostUSD.Sign() <= 0 || costUSD.Cmp(burnLimit) > 0 + if burnExceeded { + killLevel = escalateKillSwitch(killLevel, KillSwitchL2) + killReasons = append(killReasons, "g7_task_cost_burn_exceeded_2x_median") + } + g7Pass := costUSD.Cmp(maxTaskCostUSD) <= 0 && + taskDurMin.Cmp(maxTaskDurationMin) <= 0 && + !burnExceeded && + execFam != "" && overFam != "" && execFam != overFam + g7Summary := fmt.Sprintf("cost=$%s (median=$%s) duration=%smin executor=%s overseer=%s", + costUSD.FloatString(2), medCostUSD.FloatString(2), taskDurMin.FloatString(1), execFam, overFam) + checks = append(checks, makeCheck("g7_budget_and_cross_family_overseer", g7Pass, g7Summary)) + + var voiReport *VOIReport + if profile.VOI != nil { + vr, err := EvaluateVOI(*profile.VOI) + if err != nil { + return Report{}, err + } + voiReport = &vr + voiPass := vr.ExtremizationRejected && (vr.FeasibleUnderLossCap || vr.RecommendedAction == "do_not_launch_or_test_until_loss_restructured") + checks = append(checks, makeCheck("c20_voi_and_loss_cap", voiPass, + fmt.Sprintf("netEVSI=%s EUR feasibleUnderCap=%t action=%s", vr.NetIncrementalEVEUR, vr.FeasibleUnderLossCap, vr.RecommendedAction))) + } + + var passKReport *PassKReport + if profile.PassK != nil { + pr, err := EvaluatePassK(*profile.PassK) + if err != nil { + return Report{}, err + } + passKReport = &pr + checks = append(checks, makeCheck("c21_combinatorial_passk_and_cascade_cost", true, + fmt.Sprintf("pass^%d=%s pass@%d=%s fullCascade=$%s hoursPerAccepted=%s", + pr.K, pr.UnbiasedCombinatorialPassK, pr.K, pr.UnbiasedCombinatorialPassAt, pr.FullCascadeCostPerTaskUSD, pr.HumanHoursPerAcceptedTask))) + } + + var expReport *ExperimentReport + if profile.Experiment != nil { + er, err := EvaluateExperiment(*profile.Experiment) + if err != nil { + return Report{}, err + } + expReport = &er + expPass := !er.SRMDetected && er.CUPEDValid + checks = append(checks, makeCheck("c19_experiment_srm_deff_tost", expPass, + fmt.Sprintf("srmChi2=%s deff=%s tCluster=%s cupedValid=%t verdict=%s", + er.SRMChiSquare, er.DesignEffectDEFF, er.ClusterAdjustedTStat, er.CUPEDValid, er.Verdict))) + } + + var discReport *DiscoveryReport + if profile.Discovery != nil { + dr, err := EvaluateDiscovery(*profile.Discovery) + if err != nil { + return Report{}, err + } + discReport = &dr + checks = append(checks, makeCheck("c18_theory_of_value_stage_gate", dr.MethodologicallyValid, + fmt.Sprintf("valid=%t authorized=%t reasons=%v", dr.MethodologicallyValid, dr.StageGateAuthorized, dr.Reasons))) + } + + passed := 0 + failed := 0 + for _, c := range checks { + if c.Status == StatusPass { + passed++ + } else { + failed++ + } + } + if classRank(cls.EffectiveClass) > classRank(maxAutoClass) && cls.EffectiveClass != ClassK4 { + failed++ + checks = append(checks, makeCheck("autonomy_ceiling_enforcement", false, + fmt.Sprintf("candidate requires %s but current maxAutonomousClass is %s", cls.EffectiveClass, maxAutoClass))) + } + + verdict := VerdictPass + if failed > 0 { + verdict = VerdictFail + } + sort.Strings(killReasons) + killReasons = compactStrings(killReasons) + + return Report{ + SchemaVersion: ReportSchema, + APIVersion: ProfileAPIVersion, + CandidateID: profile.Candidate.ID, + Classification: cls, + Checks: checks, + Passed: passed, + Failed: failed, + MaxAutonomousClass: maxAutoClass, + KillSwitchLevel: killLevel, + KillSwitchReasons: killReasons, + MutationKillRatio: mutRatio.RatString(), + AIChangeFailRate: aiCFR.RatString(), + HumanChangeFailRate: humanCFR.RatString(), + CanarySLOBurnFraction: burnFrac.RatString(), + VOI: voiReport, + PassK: passKReport, + Experiment: expReport, + Discovery: discReport, + Verdict: verdict, + }, nil +} + +func makeCheck(id string, pass bool, summary string) CheckResult { + status := StatusFail + if pass { + status = StatusPass + } + return CheckResult{ID: id, Status: status, Summary: summary} +} + +func escalateKillSwitch(current, candidate string) string { + rank := map[string]int{ + KillSwitchNone: 0, + KillSwitchL1: 1, + KillSwitchL2: 2, + KillSwitchL3: 3, + } + if rank[candidate] > rank[current] { + return candidate + } + return current +} + +// EvaluateVOI computes exact rational Bayesian posteriors, EVPI, Gross EVSI, +// Net EVSI, and enforces non-compensatory lexicographic loss caps (C-20). +func EvaluateVOI(in VOIInput) (VOIReport, error) { + p, err := parseProbabilityRat(in.PriorSuccessProb, "voi.priorSuccessProb") + if err != nil { + return VOIReport{}, err + } + if p.Sign() == 0 || p.Cmp(big.NewRat(1, 1)) == 0 { + return VOIReport{}, fmt.Errorf("%w: voi.priorSuccessProb must be strictly in (0, 1)", ErrInvalidProfile) + } + win, err := parseNonNegativeRat(in.SuccessNetPayoffEUR, "voi.successNetPayoffEur") + if err != nil { + return VOIReport{}, err + } + lossMag, err := parseNonNegativeRat(in.FailureLossEUR, "voi.failureLossEur") + if err != nil { + return VOIReport{}, err + } + noAct, err := parseSignedRat(in.NoActionPayoffEUR, "voi.noActionPayoffEur") + if err != nil { + return VOIReport{}, err + } + testCost, err := parseNonNegativeRat(in.TestCostEUR, "voi.testCostEur") + if err != nil { + return VOIReport{}, err + } + sens, err := parseProbabilityRat(in.TestSensitivity, "voi.testSensitivity") + if err != nil { + return VOIReport{}, err + } + spec, err := parseProbabilityRat(in.TestSpecificity, "voi.testSpecificity") + if err != nil { + return VOIReport{}, err + } + capEUR, err := parseNonNegativeRat(in.HardLossCapEUR, "voi.hardLossCapEur") + if err != nil { + return VOIReport{}, err + } + + one := big.NewRat(1, 1) + q := new(big.Rat).Sub(one, p) + + // Launch EV = p * win - (1-p) * lossMag + launchEV := new(big.Rat).Sub(new(big.Rat).Mul(p, win), new(big.Rat).Mul(q, lossMag)) + noTestBestEV := new(big.Rat).Set(noAct) + noTestAction := "do_not_launch" + if launchEV.Cmp(noAct) > 0 { + noTestBestEV.Set(launchEV) + noTestAction = "launch_now" + } + + // P(T+) = p*sens + (1-p)*(1-spec) + fpr := new(big.Rat).Sub(one, spec) + pPos := new(big.Rat).Add(new(big.Rat).Mul(p, sens), new(big.Rat).Mul(q, fpr)) + pNeg := new(big.Rat).Sub(one, pPos) + if pPos.Sign() <= 0 || pNeg.Sign() <= 0 { + return VOIReport{}, fmt.Errorf("%w: degenerate test probabilities P(T+)=%s", ErrInvalidProfile, pPos.RatString()) + } + + postPos := new(big.Rat).Quo(new(big.Rat).Mul(p, sens), pPos) + fnr := new(big.Rat).Sub(one, sens) + postNeg := new(big.Rat).Quo(new(big.Rat).Mul(p, fnr), pNeg) + + evGivenPos := new(big.Rat).Sub(new(big.Rat).Mul(postPos, win), new(big.Rat).Mul(new(big.Rat).Sub(one, postPos), lossMag)) + bestGivenPos := maxRat(evGivenPos, noAct) + evGivenNeg := new(big.Rat).Sub(new(big.Rat).Mul(postNeg, win), new(big.Rat).Mul(new(big.Rat).Sub(one, postNeg), lossMag)) + bestGivenNeg := maxRat(evGivenNeg, noAct) + + grossPolicyEV := new(big.Rat).Add(new(big.Rat).Mul(pPos, bestGivenPos), new(big.Rat).Mul(pNeg, bestGivenNeg)) + grossEVSI := new(big.Rat).Sub(grossPolicyEV, noTestBestEV) + netPolicyEV := new(big.Rat).Sub(grossPolicyEV, testCost) + netIncrementalEV := new(big.Rat).Sub(grossEVSI, testCost) + + // Perfect information EV = p * max(win, noAct) + (1-p) * max(-lossMag, noAct) + negLoss := new(big.Rat).Neg(lossMag) + perfectEV := new(big.Rat).Add(new(big.Rat).Mul(p, maxRat(win, noAct)), new(big.Rat).Mul(q, maxRat(negLoss, noAct))) + evpi := new(big.Rat).Sub(perfectEV, noTestBestEV) + + feasibleUnderCap := lossMag.Cmp(capEUR) <= 0 + effectiveNetEVSI := new(big.Rat).Set(netIncrementalEV) + recAction := noTestAction + if !feasibleUnderCap { + // Non-compensatory lexicographic constraint: launch is infeasible regardless of test signal. + effectiveNetEVSI = new(big.Rat).Neg(testCost) + recAction = "do_not_launch_or_test_until_loss_restructured" + } else if netIncrementalEV.Sign() > 0 { + recAction = "run_test_and_follow_posterior_threshold" + } + + return VOIReport{ + NoTestBestAction: noTestAction, + LaunchNowEVEUR: ratToCleanString(launchEV), + NoTestBestEVEUR: ratToCleanString(noTestBestEV), + PositiveTestProb: pPos.RatString(), + PosteriorSuccessGivenPos: postPos.RatString(), + PosteriorSuccessGivenNeg: postNeg.RatString(), + EVPIEUR: ratToCleanString(evpi), + GrossPolicyEVEUR: ratToCleanString(grossPolicyEV), + GrossEVSIEUR: ratToCleanString(grossEVSI), + NetPolicyEVEUR: ratToCleanString(netPolicyEV), + NetIncrementalEVEUR: ratToCleanString(netIncrementalEV), + FeasibleUnderLossCap: feasibleUnderCap, + EffectiveNetEVSIUnderCap: ratToCleanString(effectiveNetEVSI), + ExtremizationRejected: in.CorrelatedLLMRunsCount >= 0, + RecommendedAction: recAction, + }, nil +} + +func maxRat(a, b *big.Rat) *big.Rat { + if a.Cmp(b) >= 0 { + return new(big.Rat).Set(a) + } + return new(big.Rat).Set(b) +} + +func ratToCleanString(r *big.Rat) string { + if r.IsInt() { + return r.Num().String() + } + return r.RatString() +} + +func binom(n, k int) *big.Int { + if k < 0 || k > n { + return big.NewInt(0) + } + return new(big.Int).Binomial(int64(n), int64(k)) +} + +// EvaluatePassK computes Yao et al. unbiased task-averaged combinatorial pass^k +// and pass@k, full cascade cost with retry amplification, and quality-adjusted +// human labor per accepted task (C-21). +func EvaluatePassK(in PassKInput) (PassKReport, error) { + if in.K <= 0 || len(in.Tasks) == 0 { + return PassKReport{}, fmt.Errorf("%w: passK requires k >= 1 and non-empty tasks", ErrInvalidProfile) + } + sumPass1 := big.NewRat(0, 1) + sumPassK := big.NewRat(0, 1) + sumPassAtK := big.NewRat(0, 1) + + for _, t := range in.Tasks { + if t.Trials < in.K || t.Successes < 0 || t.Successes > t.Trials { + return PassKReport{}, fmt.Errorf("%w: invalid task %q trials=%d successes=%d for k=%d", ErrInvalidProfile, t.TaskID, t.Trials, t.Successes, in.K) + } + sumPass1.Add(sumPass1, big.NewRat(int64(t.Successes), int64(t.Trials))) + denom := binom(t.Trials, in.K) + numPassK := binom(t.Successes, in.K) + taskPassK := new(big.Rat).SetFrac(numPassK, denom) + sumPassK.Add(sumPassK, taskPassK) + + numFailAll := binom(t.Trials-t.Successes, in.K) + taskFailAll := new(big.Rat).SetFrac(numFailAll, denom) + taskPassAtK := new(big.Rat).Sub(big.NewRat(1, 1), taskFailAll) + sumPassAtK.Add(sumPassAtK, taskPassAtK) + } + + mRat := big.NewRat(int64(len(in.Tasks)), 1) + meanP := new(big.Rat).Quo(sumPass1, mRat) + unbiasedPassK := new(big.Rat).Quo(sumPassK, mRat) + unbiasedPassAtK := new(big.Rat).Quo(sumPassAtK, mRat) + + naivePassK := powRat(meanP, in.K) + oneMinusP := new(big.Rat).Sub(big.NewRat(1, 1), meanP) + naivePassAtK := new(big.Rat).Sub(big.NewRat(1, 1), powRat(oneMinusP, in.K)) + + cCheap, err := parseNonNegativeRat(in.CheapModelCostUSD, "passK.cheapModelCostUsd") + if err != nil { + return PassKReport{}, err + } + cVer, err := parseNonNegativeRat(in.VerifierCostUSD, "passK.verifierCostUsd") + if err != nil { + return PassKReport{}, err + } + cStrong, err := parseNonNegativeRat(in.StrongModelCostUSD, "passK.strongModelCostUsd") + if err != nil { + return PassKReport{}, err + } + pFail, err := parseProbabilityRat(in.GateFailRate, "passK.gateFailRate") + if err != nil { + return PassKReport{}, err + } + rRetry, err := parseNonNegativeRat(in.StrongRetryRate, "passK.strongRetryRate") + if err != nil { + return PassKReport{}, err + } + + // Full post-generation cascade cost = cCheap + cVer + pFail * cStrong * (1 + rRetry) + retryFactor := new(big.Rat).Add(big.NewRat(1, 1), rRetry) + strongExpected := new(big.Rat).Mul(new(big.Rat).Mul(pFail, cStrong), retryFactor) + fullCascade := new(big.Rat).Add(new(big.Rat).Add(cCheap, cVer), strongExpected) + + // Naive weighted average often ignores verifier overhead and retries: (1 - pFail)*cCheap + pFail*cStrong + oneMinusFail := new(big.Rat).Sub(big.NewRat(1, 1), pFail) + naiveCascade := new(big.Rat).Add(new(big.Rat).Mul(oneMinusFail, cCheap), new(big.Rat).Mul(pFail, cStrong)) + + hCode, err := parseNonNegativeRat(in.HumanCodingHours, "passK.humanCodingHours") + if err != nil { + return PassKReport{}, err + } + hRev, err := parseNonNegativeRat(in.HumanReviewHours, "passK.humanReviewHours") + if err != nil { + return PassKReport{}, err + } + hRew, err := parseNonNegativeRat(in.HumanReworkHours, "passK.humanReworkHours") + if err != nil { + return PassKReport{}, err + } + if in.AcceptedTasks <= 0 || in.AttemptedTasks < in.AcceptedTasks { + return PassKReport{}, fmt.Errorf("%w: invalid acceptedTasks=%d attemptedTasks=%d", ErrInvalidProfile, in.AcceptedTasks, in.AttemptedTasks) + } + totalHours := new(big.Rat).Add(new(big.Rat).Add(hCode, hRev), hRew) + hoursPerAccepted := new(big.Rat).Quo(totalHours, big.NewRat(int64(in.AcceptedTasks), 1)) + accRate := big.NewRat(int64(in.AcceptedTasks), int64(in.AttemptedTasks)) + + return PassKReport{ + K: in.K, + MeanPass1: meanP.RatString(), + UnbiasedCombinatorialPassK: unbiasedPassK.RatString(), + UnbiasedCombinatorialPassAt: unbiasedPassAtK.RatString(), + NaiveHomogeneousPassK: naivePassK.RatString(), + NaiveHomogeneousPassAtK: naivePassAtK.RatString(), + FullCascadeCostPerTaskUSD: fullCascade.FloatString(2), + NaiveCascadeCostPerTaskUSD: naiveCascade.FloatString(2), + TotalHumanHours: ratToCleanString(totalHours), + HumanHoursPerAcceptedTask: hoursPerAccepted.RatString(), + AcceptanceRate: accRate.RatString(), + }, nil +} + +func powRat(base *big.Rat, exp int) *big.Rat { + num := new(big.Int).Exp(base.Num(), big.NewInt(int64(exp)), nil) + den := new(big.Int).Exp(base.Denom(), big.NewInt(int64(exp)), nil) + return new(big.Rat).SetFrac(num, den) +} + +// EvaluateExperiment computes SRM chi-square, cluster DEFF, effective sample +// size, and one-sided non-inferiority vs two-sided TOST equivalence (C-19). +func EvaluateExperiment(in ExperimentInput) (ExperimentReport, error) { + if in.ControlAssignedUnits <= 0 || in.TreatmentAssignedUnits <= 0 { + return ExperimentReport{}, fmt.Errorf("%w: assigned unit counts must be positive", ErrInvalidProfile) + } + if in.ClustersPerArm < 2 || in.UnitsPerCluster < 1 { + return ExperimentReport{}, fmt.Errorf("%w: clustersPerArm >= 2 and unitsPerCluster >= 1 required", ErrInvalidProfile) + } + rho, err := parseProbabilityRat(in.IntraclusterCorrelation, "experiment.intraclusterCorrelation") + if err != nil { + return ExperimentReport{}, err + } + zNaive, err := parseSignedRat(in.NaiveZStat, "experiment.naiveZStat") + if err != nil { + return ExperimentReport{}, err + } + diff, err := parseSignedRat(in.ObservedDiff, "experiment.observedDiff") + if err != nil { + return ExperimentReport{}, err + } + se, err := parseNonNegativeRat(in.DiffStandardError, "experiment.diffStandardError") + if err != nil || se.Sign() == 0 { + return ExperimentReport{}, fmt.Errorf("%w: diffStandardError must be strictly positive", ErrInvalidProfile) + } + margin, err := parseNonNegativeRat(in.NonInferiorityMargin, "experiment.nonInferiorityMargin") + if err != nil || margin.Sign() == 0 { + return ExperimentReport{}, fmt.Errorf("%w: nonInferiorityMargin must be strictly positive", ErrInvalidProfile) + } + + // SRM chi^2 for 50/50 split: (N_C - N_T)^2 / (N_C + N_T) + totalUnits := int64(in.ControlAssignedUnits + in.TreatmentAssignedUnits) + diffUnits := int64(in.ControlAssignedUnits - in.TreatmentAssignedUnits) + srmChi2 := big.NewRat(diffUnits*diffUnits, totalUnits) + srmDetected := srmChi2.Cmp(srmChiSquareCrit) >= 0 + + // DEFF = 1 + (m - 1) * rho + mMinus1 := big.NewRat(int64(in.UnitsPerCluster-1), 1) + deff := new(big.Rat).Add(big.NewRat(1, 1), new(big.Rat).Mul(mMinus1, rho)) + rawArmN := big.NewRat(int64(in.ClustersPerArm*in.UnitsPerCluster), 1) + nEff := new(big.Rat).Quo(rawArmN, deff) + + deffFloat, _ := deff.Float64() + seInflation := math.Sqrt(deffFloat) + zNaiveFloat, _ := zNaive.Float64() + tCluster := zNaiveFloat / seInflation + + // Approximate two-sided p < 0.05 critical value for df = 2*(ClustersPerArm - 1) + df := 2 * (in.ClustersPerArm - 1) + tCrit := approximateTCrit005TwoSided(df) + clusterSig := math.Abs(tCluster) >= tCrit && !srmDetected && !in.HasPostTreatmentCovariate + + // Non-inferiority against -margin: z_NI = (diff - (-margin)) / se = (diff + margin) / se + zNI := new(big.Rat).Quo(new(big.Rat).Add(diff, margin), se) + // TOST upper against +margin: z_upper = (diff - margin) / se + zUpper := new(big.Rat).Quo(new(big.Rat).Sub(diff, margin), se) + + zNIFloat, _ := zNI.Float64() + zUpperFloat, _ := zUpper.Float64() + // One-sided alpha=0.05 z-critical = 1.6448536 + niProven := zNIFloat >= 1.6448536 && !srmDetected + tostProven := zNIFloat >= 1.6448536 && zUpperFloat <= -1.6448536 && !srmDetected + + verdict := VerdictPass + if srmDetected || in.HasPostTreatmentCovariate { + verdict = VerdictFail + } + + return ExperimentReport{ + SRMChiSquare: srmChi2.FloatString(4), + SRMDetected: srmDetected, + DesignEffectDEFF: deff.FloatString(4), + EffectiveUnitsPerArm: nEff.FloatString(2), + SEInflationFactor: fmt.Sprintf("%.4f", seInflation), + ClusterAdjustedTStat: fmt.Sprintf("%.4f", tCluster), + ClusterSignificantAt005: clusterSig, + CUPEDValid: !in.HasPostTreatmentCovariate, + NonInferiorityZ: zNI.FloatString(4), + TOSTUpperZ: zUpper.FloatString(4), + NonInferiorityProvenAt005: niProven, + TOSTEquivalenceProven005: tostProven, + Verdict: verdict, + }, nil +} + +func approximateTCrit005TwoSided(df int) float64 { + if df <= 1 { + return 12.706 + } + // Peizer-Pratt / Cornish-Fisher expansion for t_{0.975, df} + z := 1.959963984540054 + d := float64(df) + g1 := (z*z*z + z) / 4.0 + g2 := (5*math.Pow(z, 5) + 16*z*z*z + 3*z) / 96.0 + return z + g1/d + g2/(d*d) +} + +// EvaluateDiscovery checks a product discovery and stage-gate packet against C-18 invariants. +func EvaluateDiscovery(in DiscoveryInput) (DiscoveryReport, error) { + if in.TotalInterviewedPeople < 0 || in.IndependentAccounts < 0 || in.IndependentAccounts > in.TotalInterviewedPeople { + return DiscoveryReport{}, fmt.Errorf("%w: invalid interview/account counts", ErrInvalidProfile) + } + accountInflation := in.TotalInterviewedPeople > in.IndependentAccounts && in.IndependentAccounts < 5 + roleComplete := in.HasTechnicalUserEvidence && in.HasEconomicBuyerEvidence && in.HasVetoHolderEvidence + + var reasons []string + if !in.HardConstraintsIntact { + reasons = append(reasons, "hard_architectural_or_security_constraints_violated") + } + if accountInflation { + reasons = append(reasons, "multiple_interviewees_in_same_firm_counted_as_independent_accounts") + } + if !roleComplete { + reasons = append(reasons, "missing_stratified_evidence_across_user_buyer_and_veto_holder") + } + if in.UsesMechanicalWTPDivisor { + reasons = append(reasons, "mechanical_hypothetical_wtp_divisor_is_invalid") + } + if in.UsesRICEToOverrideSecurity { + reasons = append(reasons, "rice_score_cannot_override_security_or_isolation") + } + if !in.ExternalContactOwnerApproved { + reasons = append(reasons, "external_contact_requires_explicit_owner_approval") + } + if !in.HasConsequentialCommitment { + reasons = append(reasons, "praise_without_consequential_migration_or_pilot_commitment") + } + + valid := len(reasons) == 0 + return DiscoveryReport{ + AccountInflationDetected: accountInflation, + RoleStratificationComplete: roleComplete, + MethodologicallyValid: valid, + StageGateAuthorized: valid && in.HasConsequentialCommitment && in.ExternalContactOwnerApproved, + Reasons: reasons, + }, nil +} diff --git a/pkg/rsi/rsi_fuzz_test.go b/pkg/rsi/rsi_fuzz_test.go new file mode 100644 index 0000000..cc63a9b --- /dev/null +++ b/pkg/rsi/rsi_fuzz_test.go @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package rsi + +import ( + "encoding/json" + "testing" +) + +func FuzzEvaluateCandidate(f *testing.F) { + prof := validCompliantK3Profile() + seedPass, err := json.Marshal(prof) + if err != nil { + f.Fatal(err) + } + f.Add(seedPass) + + profFail := validCompliantK3Profile() + profFail.Candidate.ChangedPaths = []string{".github/workflows/ci.yml"} + profFail.Candidate.DeletedOrSkippedTests = 1 + seedFail, err := json.Marshal(profFail) + if err != nil { + f.Fatal(err) + } + f.Add(seedFail) + f.Add([]byte(`{"apiVersion":"cloudring.org/rsi-governance/v1","kind":"RSICandidateProfile"}`)) + + f.Fuzz(func(t *testing.T, raw []byte) { + parsed, err := ParseProfile(raw) + if err != nil { + return + } + rep, err := EvaluateCandidate(parsed) + if err != nil { + return + } + if rep.Passed+rep.Failed != len(rep.Checks) { + t.Fatalf("check count mismatch: passed=%d failed=%d total=%d", rep.Passed, rep.Failed, len(rep.Checks)) + } + if rep.Failed > 0 && rep.Verdict != VerdictFail { + t.Fatalf("failed checks=%d but verdict=%s", rep.Failed, rep.Verdict) + } + if rep.Failed == 0 && rep.Verdict != VerdictPass { + t.Fatalf("zero failed checks but verdict=%s", rep.Verdict) + } + if rep.Classification.EffectiveClass == ClassK4 && strings_trim(parsed.Candidate.HumanK4ApprovalReference) == "" && rep.Verdict == VerdictPass { + t.Fatalf("unapproved K4 candidate must never pass: %+v", rep) + } + }) +} + +func strings_trim(s string) string { + for len(s) > 0 && (s[0] == ' ' || s[0] == '\t' || s[0] == '\n' || s[0] == '\r') { + s = s[1:] + } + for len(s) > 0 && (s[len(s)-1] == ' ' || s[len(s)-1] == '\t' || s[len(s)-1] == '\n' || s[len(s)-1] == '\r') { + s = s[:len(s)-1] + } + return s +} diff --git a/pkg/rsi/rsi_test.go b/pkg/rsi/rsi_test.go new file mode 100644 index 0000000..98e25c0 --- /dev/null +++ b/pkg/rsi/rsi_test.go @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package rsi + +import ( + "encoding/json" + "strings" + "testing" +) + +func validCompliantK3Profile() CandidateProfile { + return CandidateProfile{ + APIVersion: ProfileAPIVersion, + Kind: ProfileKind, + Candidate: CandidateMetadata{ + ID: "cand-rsi-001", + DeclaredClass: ClassK3, + ChangedPaths: []string{"pkg/rsi/rsi.go", "cmd/cloudringctl/commands.go"}, + ReadOnlyOnly: false, + ReversibleConfigOnly: false, + DeletedOrSkippedTests: 0, + ModifiedExistingTests: false, + ModifiedTestdata: false, + }, + Gates: GateReceipts{ + G1: G1Receipt{ + EvaluatorFrozenHashVerified: true, + SuitePassed: true, + NewTestFailedBeforeFix: true, + NewTestPassedAfterFix: true, + RerunMaskingUsed: false, + SilentErrorSwallowing: 0, + UnseenHoldoutPassed: true, + SeededErrorRejected: true, + }, + G2: G2Receipt{ + KilledMutants: 18, + TotalMutants: 20, + FuzzTargetsPassed: true, + }, + G3: G3Receipt{ + SASTFindings: 0, + VulnFindings: 0, + ExternalCapabilityGate: true, + PromptDelimiterOnlySecurity: false, + ExplanationOnlyApproval: false, + }, + G4: G4Receipt{ + ActiveCanaries: 1, + CanaryStepPercent: "10", + CanaryMaxPercent: "50", + RequestSuccessRatePercent: "99.5", + LatencyP99Ms: "220", + CanaryDurationMinutes: "10", + CanaryFailuresLast24h: 0, + TouchesPersistentState: true, + DowngradeContinuityVerified: true, + }, + G5: G5Receipt{ + PortabilityDrillPassed: true, + DataActVerdict: VerdictPass, + }, + G6: G6Receipt{ + AIFailedDeployments: 1, + AITotalDeployments: 20, + HumanFailedDeployments: 2, + HumanTotalDeployments: 20, + }, + G7: G7Receipt{ + TaskCostUSD: "1.80", + MedianTaskCostUSD: "1.50", + TaskDurationMinutes: "12", + ExecutorModelFamily: "family-a", + OverseerModelFamily: "family-b", + }, + }, + VOI: &VOIInput{ + PriorSuccessProb: "2/5", + SuccessNetPayoffEUR: "120000", + FailureLossEUR: "60000", + NoActionPayoffEUR: "0", + TestCostEUR: "4000", + TestSensitivity: "4/5", + TestSpecificity: "3/4", + HardLossCapEUR: "80000", + CorrelatedLLMRunsCount: 3, + }, + PassK: &PassKInput{ + K: 4, + Tasks: []TaskTrialCount{ + {TaskID: "t1", Trials: 5, Successes: 5}, + {TaskID: "t2", Trials: 5, Successes: 4}, + {TaskID: "t3", Trials: 5, Successes: 3}, + {TaskID: "t4", Trials: 5, Successes: 0}, + }, + CheapModelCostUSD: "0.80", + VerifierCostUSD: "0.30", + StrongModelCostUSD: "4.00", + GateFailRate: "0.45", + StrongRetryRate: "0.10", + HumanCodingHours: "40", + HumanReviewHours: "28", + HumanReworkHours: "12", + AcceptedTasks: 14, + AttemptedTasks: 20, + }, + Experiment: &ExperimentInput{ + ControlAssignedUnits: 10000, + TreatmentAssignedUnits: 10000, + ClustersPerArm: 20, + UnitsPerCluster: 500, + IntraclusterCorrelation: "0.05", + NaiveZStat: "2.50", + HasPostTreatmentCovariate: false, + ObservedDiff: "0.02", + DiffStandardError: "0.01", + NonInferiorityMargin: "0.05", + }, + Discovery: &DiscoveryInput{ + HardConstraintsIntact: true, + TotalInterviewedPeople: 12, + IndependentAccounts: 10, + HasTechnicalUserEvidence: true, + HasEconomicBuyerEvidence: true, + HasVetoHolderEvidence: true, + UsesMechanicalWTPDivisor: false, + UsesRICEToOverrideSecurity: false, + ExternalContactOwnerApproved: true, + HasConsequentialCommitment: true, + }, + } +} + +func TestEvaluateCompliantK3ProfilePassesAllGates(t *testing.T) { + prof := validCompliantK3Profile() + raw, err := json.Marshal(prof) + if err != nil { + t.Fatal(err) + } + parsed, err := ParseProfile(raw) + if err != nil { + t.Fatalf("ParseProfile failed: %v", err) + } + rep, err := EvaluateCandidate(parsed) + if err != nil { + t.Fatalf("EvaluateCandidate failed: %v", err) + } + if rep.Verdict != VerdictPass || rep.Failed != 0 || rep.KillSwitchLevel != KillSwitchNone { + t.Fatalf("expected pass verdict with 0 failed checks, got verdict=%s failed=%d kill=%s checks=%+v", + rep.Verdict, rep.Failed, rep.KillSwitchLevel, rep.Checks) + } + if rep.Classification.EffectiveClass != ClassK3 || !rep.Classification.HoldoutRequired { + t.Fatalf("expected effective K3 with holdout, got %+v", rep.Classification) + } + if rep.VOI == nil || rep.VOI.LaunchNowEVEUR != "12000" || rep.VOI.PositiveTestProb != "47/100" || + rep.VOI.PosteriorSuccessGivenPos != "32/47" || rep.VOI.PosteriorSuccessGivenNeg != "8/53" || + rep.VOI.GrossPolicyEVEUR != "29400" || rep.VOI.GrossEVSIEUR != "17400" || + rep.VOI.NetPolicyEVEUR != "25400" || rep.VOI.NetIncrementalEVEUR != "13400" || rep.VOI.EVPIEUR != "36000" { + t.Fatalf("unexpected VOI report: %+v", rep.VOI) + } + if rep.PassK == nil || rep.PassK.UnbiasedCombinatorialPassK != "3/10" || + rep.PassK.UnbiasedCombinatorialPassAt != "3/4" || rep.PassK.FullCascadeCostPerTaskUSD != "3.08" || + rep.PassK.NaiveCascadeCostPerTaskUSD != "2.24" || rep.PassK.HumanHoursPerAcceptedTask != "40/7" { + t.Fatalf("unexpected PassK report: %+v", rep.PassK) + } +} + +func TestClassifyCandidateEscalatesTamperingAndProtectedPathsToK4(t *testing.T) { + cases := []struct { + name string + meta CandidateMetadata + reason string + }{ + { + name: "github-workflow-path", + meta: CandidateMetadata{ID: "c1", DeclaredClass: ClassK2, ChangedPaths: []string{".GitHub/workflows/ci.yml"}}, + reason: "protected_k4_path:.github/workflows/ci.yml", + }, + { + name: "path-traversal-evaluator", + meta: CandidateMetadata{ID: "c2", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg/../evaluator/score.go"}}, + reason: "path_traversal_or_absolute:pkg/../evaluator/score.go", + }, + { + name: "ops-stop-modification", + meta: CandidateMetadata{ID: "c3", DeclaredClass: ClassK1, ChangedPaths: []string{"ops/STOP"}}, + reason: "protected_control_file:ops/stop", + }, + { + name: "deleted-tests", + meta: CandidateMetadata{ID: "c4", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg/ocs/spec.go"}, DeletedOrSkippedTests: 1}, + reason: "deleted_or_skipped_tests_requires_k4", + }, + { + name: "co-modified-prod-and-existing-tests", + meta: CandidateMetadata{ID: "c5", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg/ocs/spec.go"}, ModifiedExistingTests: true}, + reason: "co_modified_production_and_existing_tests_or_testdata_requires_k4", + }, + { + name: "url-encoded-traversal", + meta: CandidateMetadata{ID: "c6", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg/%2e%2e/evaluator/score.go"}}, + reason: "encoded_or_null_path:pkg/%2e%2e/evaluator/score.go", + }, + { + name: "url-encoded-slash-upper", + meta: CandidateMetadata{ID: "c7", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg%2Fevaluator/score.go"}}, + reason: "encoded_or_null_path:pkg%2Fevaluator/score.go", + }, + { + name: "url-encoded-backslash", + meta: CandidateMetadata{ID: "c8", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg%5Cevaluator/score.go"}}, + reason: "encoded_or_null_path:pkg%5Cevaluator/score.go", + }, + { + name: "null-byte-path", + meta: CandidateMetadata{ID: "c9", DeclaredClass: ClassK2, ChangedPaths: []string{"pkg/ocs/spec.go\x00.txt"}}, + reason: "encoded_or_null_path:pkg/ocs/spec.go\x00.txt", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cls, err := ClassifyCandidate(tc.meta) + if err != nil { + t.Fatal(err) + } + if cls.EffectiveClass != ClassK4 || !cls.HumanApprovalRequired { + t.Fatalf("expected K4 escalation, got %+v", cls) + } + found := false + for _, r := range cls.EscalationReasons { + if strings.Contains(r, tc.reason) { + found = true + } + } + if !found { + t.Fatalf("expected reason %q in %v", tc.reason, cls.EscalationReasons) + } + }) + } +} + +func TestExactGateThresholdsAndKillSwitchEscalations(t *testing.T) { + // Exact 80% (16/20) mutation kill ratio must pass G2; 15/20 (75%) must fail. + prof := validCompliantK3Profile() + prof.Gates.G2.KilledMutants = 16 + prof.Gates.G2.TotalMutants = 20 + rep, err := EvaluateCandidate(prof) + if err != nil || rep.Verdict != VerdictPass { + t.Fatalf("expected 16/20 (80%%) mutation kill score to pass, got err=%v verdict=%s", err, rep.Verdict) + } + + prof.Gates.G2.KilledMutants = 15 + rep, err = EvaluateCandidate(prof) + if err != nil || rep.Verdict != VerdictFail { + t.Fatalf("expected 15/20 (75%%) mutation kill score to fail, got err=%v verdict=%s", err, rep.Verdict) + } + + // G6: AI CFR = 3/20 (15%) with human CFR = 3/20 (15%) must pass; 4/20 (20%) must fail and downgrade to K1 + KillSwitchL1. + prof = validCompliantK3Profile() + prof.Gates.G6.AIFailedDeployments = 3 + prof.Gates.G6.AITotalDeployments = 20 + prof.Gates.G6.HumanFailedDeployments = 3 + prof.Gates.G6.HumanTotalDeployments = 20 + rep, err = EvaluateCandidate(prof) + if err != nil || rep.Verdict != VerdictPass || rep.MaxAutonomousClass != ClassK3 { + t.Fatalf("expected 3/20 CFR boundary to pass, got err=%v verdict=%s maxAuto=%s", err, rep.Verdict, rep.MaxAutonomousClass) + } + + prof.Gates.G6.AIFailedDeployments = 4 + rep, err = EvaluateCandidate(prof) + if err != nil || rep.Verdict != VerdictFail || rep.MaxAutonomousClass != ClassK1 || rep.KillSwitchLevel != KillSwitchL1 { + t.Fatalf("expected 4/20 CFR to trip G6 circuit breaker to K1/L1, got verdict=%s maxAuto=%s kill=%s", + rep.Verdict, rep.MaxAutonomousClass, rep.KillSwitchLevel) + } + + // G7: Cost burn > 2x median trips KillSwitchL2. + prof = validCompliantK3Profile() + prof.Gates.G7.TaskCostUSD = "3.01" + prof.Gates.G7.MedianTaskCostUSD = "1.50" + rep, err = EvaluateCandidate(prof) + if err != nil || rep.Verdict != VerdictFail || rep.KillSwitchLevel != KillSwitchL2 { + t.Fatalf("expected >2x median cost burn to trigger L2 kill switch, got verdict=%s kill=%s", rep.Verdict, rep.KillSwitchLevel) + } + + // Unapproved test tampering trips KillSwitchL3. + prof = validCompliantK3Profile() + prof.Candidate.DeletedOrSkippedTests = 2 + rep, err = EvaluateCandidate(prof) + if err != nil || rep.Verdict != VerdictFail || rep.KillSwitchLevel != KillSwitchL3 { + t.Fatalf("expected unapproved test deletion to trigger L3 kill switch, got verdict=%s kill=%s", rep.Verdict, rep.KillSwitchLevel) + } +} + +func TestHardLossCapAndExperimentAndDiscoveryTraps(t *testing.T) { + // C-20: Hard loss cap EUR 20,000 < FailureLossEUR 60,000 blocks launch and sets effective net EVSI to -4000. + voiIn := validCompliantK3Profile().VOI + voiIn.HardLossCapEUR = "20000" + vr, err := EvaluateVOI(*voiIn) + if err != nil { + t.Fatal(err) + } + if vr.FeasibleUnderLossCap || vr.EffectiveNetEVSIUnderCap != "-4000" || vr.RecommendedAction != "do_not_launch_or_test_until_loss_restructured" { + t.Fatalf("expected hard loss cap to block launch and test, got %+v", vr) + } + + // C-19: SRM 10250 vs 9750 (chi^2 = 12.5000), DEFF = 25.9500, N_eff = 385.36, non-inferiority z = 0.0000 (not proven). + er, err := EvaluateExperiment(ExperimentInput{ + ControlAssignedUnits: 10250, + TreatmentAssignedUnits: 9750, + ClustersPerArm: 20, + UnitsPerCluster: 500, + IntraclusterCorrelation: "0.05", + NaiveZStat: "2.50", + HasPostTreatmentCovariate: true, + ObservedDiff: "-0.05", + DiffStandardError: "0.058949", + NonInferiorityMargin: "0.05", + }) + if err != nil { + t.Fatal(err) + } + if !er.SRMDetected || er.SRMChiSquare != "12.5000" || er.DesignEffectDEFF != "25.9500" || + er.EffectiveUnitsPerArm != "385.36" || er.ClusterSignificantAt005 || er.CUPEDValid || + er.NonInferiorityProvenAt005 || er.TOSTEquivalenceProven005 || er.Verdict != VerdictFail { + t.Fatalf("unexpected C-19 experiment diagnostics: %+v", er) + } + + // C-18: 14 engineers across 3 firms + WTP/1.35 + RICE override rejected. + dr, err := EvaluateDiscovery(DiscoveryInput{ + HardConstraintsIntact: false, + TotalInterviewedPeople: 14, + IndependentAccounts: 3, + HasTechnicalUserEvidence: true, + HasEconomicBuyerEvidence: false, + HasVetoHolderEvidence: false, + UsesMechanicalWTPDivisor: true, + UsesRICEToOverrideSecurity: true, + ExternalContactOwnerApproved: false, + HasConsequentialCommitment: false, + }) + if err != nil { + t.Fatal(err) + } + if !dr.AccountInflationDetected || dr.RoleStratificationComplete || dr.MethodologicallyValid || dr.StageGateAuthorized || len(dr.Reasons) != 7 { + t.Fatalf("expected all 7 C-18 discovery errors flagged, got %+v", dr) + } +} + +func TestOrthogonalSingleGateInvariantsKillMutants(t *testing.T) { + cases := []struct { + name string + mutate func(*CandidateProfile) + wantVerdict string + wantKillSwitch string + }{ + { + name: "g1_frozen_hash_false", + mutate: func(p *CandidateProfile) { + p.Gates.G1.EvaluatorFrozenHashVerified = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g1_seeded_error_false", + mutate: func(p *CandidateProfile) { + p.Gates.G1.SeededErrorRejected = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g1_unseen_holdout_false", + mutate: func(p *CandidateProfile) { + p.Gates.G1.UnseenHoldoutPassed = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g1_suite_passed_false", + mutate: func(p *CandidateProfile) { + p.Gates.G1.SuitePassed = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g1_rerun_masking_true", + mutate: func(p *CandidateProfile) { + p.Gates.G1.RerunMaskingUsed = true + }, + wantVerdict: VerdictFail, + }, + { + name: "g1_silent_error_swallowing_nonzero", + mutate: func(p *CandidateProfile) { + p.Gates.G1.SilentErrorSwallowing = 1 + }, + wantVerdict: VerdictFail, + }, + { + name: "g2_fuzz_targets_false", + mutate: func(p *CandidateProfile) { + p.Gates.G2.FuzzTargetsPassed = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g3_external_cap_false", + mutate: func(p *CandidateProfile) { + p.Gates.G3.ExternalCapabilityGate = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g3_prompt_delimiter_only_true", + mutate: func(p *CandidateProfile) { + p.Gates.G3.PromptDelimiterOnlySecurity = true + }, + wantVerdict: VerdictFail, + }, + { + name: "g3_explanation_only_approval_true", + mutate: func(p *CandidateProfile) { + p.Gates.G3.ExplanationOnlyApproval = true + }, + wantVerdict: VerdictFail, + }, + { + name: "g4_stateful_downgrade_false", + mutate: func(p *CandidateProfile) { + p.Gates.G4.TouchesPersistentState = true + p.Gates.G4.DowngradeContinuityVerified = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g5_data_act_fail", + mutate: func(p *CandidateProfile) { + p.Gates.G5.DataActVerdict = "fail" + }, + wantVerdict: VerdictFail, + }, + { + name: "g5_portability_drill_false", + mutate: func(p *CandidateProfile) { + p.Gates.G5.PortabilityDrillPassed = false + }, + wantVerdict: VerdictFail, + }, + { + name: "g6_ai_cfr_4_of_20_with_zero_human", + mutate: func(p *CandidateProfile) { + p.Gates.G6.AIFailedDeployments = 4 + p.Gates.G6.AITotalDeployments = 20 + p.Gates.G6.HumanFailedDeployments = 0 + p.Gates.G6.HumanTotalDeployments = 0 + }, + wantVerdict: VerdictFail, + }, + { + name: "g7_exact_2x_burn_boundary_passes", + mutate: func(p *CandidateProfile) { + p.Gates.G7.TaskCostUSD = "3.00" + p.Gates.G7.MedianTaskCostUSD = "1.50" + }, + wantVerdict: VerdictPass, + wantKillSwitch: KillSwitchNone, + }, + { + name: "g7_same_family_overseer_fails", + mutate: func(p *CandidateProfile) { + p.Gates.G7.ExecutorModelFamily = "family-a" + p.Gates.G7.OverseerModelFamily = "family-a" + }, + wantVerdict: VerdictFail, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + prof := validCompliantK3Profile() + tc.mutate(&prof) + rep, err := EvaluateCandidate(prof) + if err != nil { + t.Fatalf("EvaluateCandidate returned unexpected error: %v", err) + } + if rep.Verdict != tc.wantVerdict { + t.Fatalf("expected verdict %q, got %q (checks=%+v)", tc.wantVerdict, rep.Verdict, rep.Checks) + } + if tc.wantKillSwitch != "" && rep.KillSwitchLevel != tc.wantKillSwitch { + t.Fatalf("expected killSwitchLevel %q, got %q", tc.wantKillSwitch, rep.KillSwitchLevel) + } + }) + } +} + +func TestOrthogonalDiscoveryAndDecisionScienceInvariants(t *testing.T) { + baseDiscovery := func() DiscoveryInput { + return DiscoveryInput{ + HardConstraintsIntact: true, + TotalInterviewedPeople: 12, + IndependentAccounts: 10, + HasTechnicalUserEvidence: true, + HasEconomicBuyerEvidence: true, + HasVetoHolderEvidence: true, + UsesMechanicalWTPDivisor: false, + UsesRICEToOverrideSecurity: false, + ExternalContactOwnerApproved: true, + HasConsequentialCommitment: true, + } + } + + validRep, err := EvaluateDiscovery(baseDiscovery()) + if err != nil { + t.Fatalf("expected valid baseline discovery input, got err=%v", err) + } + if !validRep.MethodologicallyValid || !validRep.StageGateAuthorized || !validRep.RoleStratificationComplete || validRep.AccountInflationDetected || len(validRep.Reasons) != 0 { + t.Fatalf("expected valid baseline discovery report, got %+v", validRep) + } + + flips := []struct { + name string + mutate func(*DiscoveryInput) + expectedReason string + }{ + { + name: "hard-constraints-violated", + mutate: func(in *DiscoveryInput) { in.HardConstraintsIntact = false }, + expectedReason: "hard_architectural_or_security_constraints_violated", + }, + { + name: "account-inflation-14-in-3", + mutate: func(in *DiscoveryInput) { + in.TotalInterviewedPeople = 14 + in.IndependentAccounts = 3 + }, + expectedReason: "multiple_interviewees_in_same_firm_counted_as_independent_accounts", + }, + { + name: "missing-technical-user-evidence", + mutate: func(in *DiscoveryInput) { in.HasTechnicalUserEvidence = false }, + expectedReason: "missing_stratified_evidence_across_user_buyer_and_veto_holder", + }, + { + name: "missing-economic-buyer-evidence", + mutate: func(in *DiscoveryInput) { in.HasEconomicBuyerEvidence = false }, + expectedReason: "missing_stratified_evidence_across_user_buyer_and_veto_holder", + }, + { + name: "missing-veto-holder-evidence", + mutate: func(in *DiscoveryInput) { in.HasVetoHolderEvidence = false }, + expectedReason: "missing_stratified_evidence_across_user_buyer_and_veto_holder", + }, + { + name: "uses-mechanical-wtp-divisor", + mutate: func(in *DiscoveryInput) { in.UsesMechanicalWTPDivisor = true }, + expectedReason: "mechanical_hypothetical_wtp_divisor_is_invalid", + }, + { + name: "uses-rice-to-override-security", + mutate: func(in *DiscoveryInput) { in.UsesRICEToOverrideSecurity = true }, + expectedReason: "rice_score_cannot_override_security_or_isolation", + }, + { + name: "external-contact-unapproved", + mutate: func(in *DiscoveryInput) { in.ExternalContactOwnerApproved = false }, + expectedReason: "external_contact_requires_explicit_owner_approval", + }, + { + name: "missing-consequential-commitment", + mutate: func(in *DiscoveryInput) { in.HasConsequentialCommitment = false }, + expectedReason: "praise_without_consequential_migration_or_pilot_commitment", + }, + } + + for _, tc := range flips { + t.Run(tc.name, func(t *testing.T) { + in := baseDiscovery() + tc.mutate(&in) + dr, err := EvaluateDiscovery(in) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if dr.MethodologicallyValid || dr.StageGateAuthorized { + t.Fatalf("expected single flip %s to invalidate discovery and block stage gate, got %+v", tc.name, dr) + } + if len(dr.Reasons) != 1 || dr.Reasons[0] != tc.expectedReason { + t.Fatalf("expected exact single reason %q, got %v", tc.expectedReason, dr.Reasons) + } + }) + } + + voiIn := *validCompliantK3Profile().VOI + voiIn.TestCostEUR = "40000" + vr, err := EvaluateVOI(voiIn) + if err != nil { + t.Fatalf("EvaluateVOI failed: %v", err) + } + if vr.EVPIEUR != "36000" || vr.GrossEVSIEUR != "17400" || vr.NetIncrementalEVEUR != "-22600" || + vr.NetPolicyEVEUR != "-10600" || vr.RecommendedAction != "launch_now" { + t.Fatalf("expected overpriced test to preserve launch_now with negative net EVSI, got %+v", vr) + } + + baseExp := *validCompliantK3Profile().Experiment + baseExp.ObservedDiff = "0.00" + baseExp.DiffStandardError = "0.01" + baseExp.NonInferiorityMargin = "0.05" + baseExp.NaiveZStat = "15.0" + cleanExp, err := EvaluateExperiment(baseExp) + if err != nil { + t.Fatalf("EvaluateExperiment clean failed: %v", err) + } + if cleanExp.SRMDetected || !cleanExp.CUPEDValid || !cleanExp.ClusterSignificantAt005 || + !cleanExp.NonInferiorityProvenAt005 || !cleanExp.TOSTEquivalenceProven005 || cleanExp.Verdict != VerdictPass { + t.Fatalf("expected clean experiment to prove cluster sig, NI, and TOST equivalence, got %+v", cleanExp) + } + + srmOnly := baseExp + srmOnly.ControlAssignedUnits = 10250 + srmOnly.TreatmentAssignedUnits = 9750 + srmRep, err := EvaluateExperiment(srmOnly) + if err != nil { + t.Fatalf("EvaluateExperiment srmOnly failed: %v", err) + } + if !srmRep.SRMDetected || !srmRep.CUPEDValid || srmRep.ClusterSignificantAt005 || + srmRep.NonInferiorityProvenAt005 || srmRep.TOSTEquivalenceProven005 || srmRep.Verdict != VerdictFail { + t.Fatalf("expected isolated SRM to invalidate experiment while keeping CUPEDValid=true, got %+v", srmRep) + } + + cupedOnly := baseExp + cupedOnly.HasPostTreatmentCovariate = true + cupedRep, err := EvaluateExperiment(cupedOnly) + if err != nil { + t.Fatalf("EvaluateExperiment cupedOnly failed: %v", err) + } + if cupedRep.SRMDetected || cupedRep.CUPEDValid || cupedRep.ClusterSignificantAt005 || + !cupedRep.NonInferiorityProvenAt005 || !cupedRep.TOSTEquivalenceProven005 || cupedRep.Verdict != VerdictFail { + t.Fatalf("expected isolated post-treatment covariate to set CUPEDValid=false and Verdict=fail while SRMDetected=false, got %+v", cupedRep) + } + + pkRep, err := EvaluatePassK(PassKInput{ + K: 3, + Tasks: []TaskTrialCount{ + {TaskID: "t1", Trials: 6, Successes: 5}, + {TaskID: "t2", Trials: 5, Successes: 3}, + {TaskID: "t3", Trials: 4, Successes: 2}, + }, + CheapModelCostUSD: "0.50", + VerifierCostUSD: "0.20", + StrongModelCostUSD: "3.00", + GateFailRate: "1/2", + StrongRetryRate: "1/5", + HumanCodingHours: "10", + HumanReviewHours: "5", + HumanReworkHours: "3", + AcceptedTasks: 9, + AttemptedTasks: 12, + }) + if err != nil { + t.Fatalf("EvaluatePassK heterogeneous failed: %v", err) + } + if pkRep.MeanPass1 != "29/45" || pkRep.UnbiasedCombinatorialPassK != "1/5" || pkRep.UnbiasedCombinatorialPassAt != "1" || + pkRep.HumanHoursPerAcceptedTask != "2" || pkRep.AcceptanceRate != "3/4" { + t.Fatalf("unexpected heterogeneous PassK report: %+v", pkRep) + } +} diff --git a/pkg/sliceoracles/oracles.go b/pkg/sliceoracles/oracles.go new file mode 100644 index 0000000..42503cc --- /dev/null +++ b/pkg/sliceoracles/oracles.go @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +// Package sliceoracles implements CloudRING's Fork 1A (32-oracle batch +// autonomous acceptance), Fork 2A (CORE vs QUAL dependency split and 2-PR +// non-overlapping path concurrency), and Deferred Fork 3 (no-new-hardware +// subgraph execution under two-slot LKG safety S1-S6 and RSI gates G1-G7). +package sliceoracles + +import ( + "encoding/json" + "errors" + "fmt" + "path" + "slices" + "strings" + + "github.com/opencloudtech/CloudRING/pkg/rsi" +) + +const ( + ExpectedAPIVersion = "cloudring.org/slice-oracles/v1" + ExpectedKind = "SliceOracleBatch" + ExpectedSliceCount = 32 + MaxWritePRsPerRepo = 2 + MinPinnedLKGSlots = 2 + MinSoakMinutes = 30 +) + +// Batch represents the frozen 32-oracle contract (Fork 1A + Fork 2A + Deferred Fork 3). +type Batch struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + DecisionForks DecisionForks `json:"decisionForks"` + MaxConcurrentWritePRsPerRepo int `json:"maxConcurrentWritePRsPerRepo"` + OwnerAcceptanceLatencyDays int `json:"ownerAcceptanceLatencyDays"` + WeeklyOwnerAuditHours int `json:"weeklyOwnerAuditHours"` + LKGInvariants []string `json:"lkgInvariants"` + Oracles []SliceOracle `json:"oracles"` +} + +// DecisionForks records the locked ROUTE-v2 §7 fork choices. +type DecisionForks struct { + Fork1 string `json:"fork1"` + Fork2 string `json:"fork2"` + Fork3 string `json:"fork3"` +} + +// SliceOracle defines the machine-verifiable acceptance oracle for one delivery slice. +type SliceOracle struct { + SliceID string `json:"sliceId"` + Title string `json:"title"` + Goals []string `json:"goals"` + SplitMode bool `json:"splitMode"` + CoreDependencies []string `json:"coreDependencies"` + QualDependencies []string `json:"qualDependencies"` + RequiresFork3Hardware bool `json:"requiresFork3Hardware"` + AllowedPaths []string `json:"allowedPaths"` + AutonomyClass string `json:"autonomyClass"` + VerificationCommand string `json:"verificationCommand"` + RequiredGates []string `json:"requiredGates"` + DaysHist string `json:"daysHist"` + DaysFactory string `json:"daysFactory"` +} + +// LKGSafetyState captures the 6 necessary and sufficient self-evolution liveness invariants (S1-S6). +type LKGSafetyState struct { + ImmutableBootloaderVerified bool `json:"immutableBootloaderVerified"` + PinnedLKGSlots int `json:"pinnedLkgSlots"` + SoakMinutes int `json:"soakMinutes"` + WebhookBypassVerified bool `json:"webhookBypassVerified"` + PreMigrationSnapshotVerified bool `json:"preMigrationSnapshotVerified"` + MetaRSISelfCheckPassed bool `json:"metaRsiSelfCheckPassed"` + CandidateSHA string `json:"candidateSha"` + QuarantinedSHAs []string `json:"quarantinedShas"` +} + +// ActivePR describes an open write PR for Fork 2A concurrency validation. +type ActivePR struct { + SliceID string `json:"sliceId"` + TouchedPaths []string `json:"touchedPaths"` +} + +// SliceAcceptanceReceipt returns the deterministic verdict for a slice candidate. +type SliceAcceptanceReceipt struct { + SliceID string `json:"sliceId"` + Phase string `json:"phase"` + Verdict string `json:"verdict"` + PromotedToLKG bool `json:"promotedToLkg"` + Blockers []string `json:"blockers,omitempty"` + RSIReport *rsi.Report `json:"rsiReport,omitempty"` +} + +// ParseBatch strictly decodes and validates the 32-oracle batch contract. +func ParseBatch(data []byte) (*Batch, error) { + if len(data) == 0 { + return nil, errors.New("sliceoracles: empty oracle batch") + } + var b Batch + dec := json.NewDecoder(strings.NewReader(string(data))) + dec.DisallowUnknownFields() + if err := dec.Decode(&b); err != nil { + return nil, fmt.Errorf("sliceoracles: decode batch: %w", err) + } + if err := b.Validate(); err != nil { + return nil, err + } + return &b, nil +} + +// Validate checks structural, Fork 1A/2A/3, and DAG acyclicity invariants. +func (b *Batch) Validate() error { + if b == nil { + return errors.New("sliceoracles: nil batch") + } + if b.APIVersion != ExpectedAPIVersion { + return fmt.Errorf("sliceoracles: invalid apiVersion %q", b.APIVersion) + } + if b.Kind != ExpectedKind { + return fmt.Errorf("sliceoracles: invalid kind %q", b.Kind) + } + if b.MaxConcurrentWritePRsPerRepo != MaxWritePRsPerRepo { + return fmt.Errorf("sliceoracles: maxConcurrentWritePRsPerRepo must be %d, got %d", MaxWritePRsPerRepo, b.MaxConcurrentWritePRsPerRepo) + } + if b.OwnerAcceptanceLatencyDays != 0 { + return fmt.Errorf("sliceoracles: Fork 1A requires ownerAcceptanceLatencyDays=0, got %d", b.OwnerAcceptanceLatencyDays) + } + if len(b.LKGInvariants) != 6 { + return fmt.Errorf("sliceoracles: expected 6 S1-S6 LKG invariants, got %d", len(b.LKGInvariants)) + } + if len(b.Oracles) != ExpectedSliceCount { + return fmt.Errorf("sliceoracles: expected %d slice oracles, got %d", ExpectedSliceCount, len(b.Oracles)) + } + + byID := make(map[string]SliceOracle, len(b.Oracles)) + for _, o := range b.Oracles { + if strings.TrimSpace(o.SliceID) == "" { + return errors.New("sliceoracles: empty sliceId") + } + if _, exists := byID[o.SliceID]; exists { + return fmt.Errorf("sliceoracles: duplicate sliceId %q", o.SliceID) + } + if len(o.Goals) == 0 || len(o.AllowedPaths) == 0 || len(o.RequiredGates) == 0 { + return fmt.Errorf("sliceoracles: slice %s missing goals, allowedPaths, or requiredGates", o.SliceID) + } + if o.SplitMode && len(o.QualDependencies) == 0 { + return fmt.Errorf("sliceoracles: split slice %s must declare qualDependencies", o.SliceID) + } + byID[o.SliceID] = o + } + + // Verify Fork 2A structural rules: C24 depends on C14 (not C23); C20 does not depend on C19. + c24, ok := byID["C24"] + if !ok || !slices.Equal(c24.CoreDependencies, []string{"C14"}) { + return errors.New("sliceoracles: Fork 2A requires C24 coreDependencies to be [C14]") + } + c20, ok := byID["C20"] + if !ok || slices.Contains(c20.CoreDependencies, "C19") { + return errors.New("sliceoracles: Fork 2A requires C20 coreDependencies not to wait for C19") + } + + // Verify all dependency references exist and form a DAG. + visiting := make(map[string]bool, len(byID)) + visited := make(map[string]bool, len(byID)) + var dfs func(id string) error + dfs = func(id string) error { + if visiting[id] { + return fmt.Errorf("sliceoracles: dependency cycle detected at %s", id) + } + if visited[id] { + return nil + } + visiting[id] = true + o := byID[id] + for _, dep := range append(slices.Clone(o.CoreDependencies), o.QualDependencies...) { + if _, exists := byID[dep]; !exists { + return fmt.Errorf("sliceoracles: slice %s references unknown dependency %s", id, dep) + } + if err := dfs(dep); err != nil { + return err + } + } + visiting[id] = false + visited[id] = true + return nil + } + for id := range byID { + if err := dfs(id); err != nil { + return err + } + } + return nil +} + +// PartitionNoNewHardware splits the 32 oracles into immediately executable no-new-hardware slices +// vs slices gated on deferred Fork 3 hardware. +func (b *Batch) PartitionNoNewHardware() (noNewHW []SliceOracle, deferredHW []SliceOracle) { + for _, o := range b.Oracles { + if o.RequiresFork3Hardware { + deferredHW = append(deferredHW, o) + } else { + noNewHW = append(noNewHW, o) + } + } + return noNewHW, deferredHW +} + +// ReadyNoNewHardwareQueue returns the ordered list of no-new-hardware slices (and phase "CORE" or "FULL") +// whose dependencies are satisfied given completedCORE and completedFULL sets. +func (b *Batch) ReadyNoNewHardwareQueue(completedCORE, completedFULL map[string]bool) []string { + var ready []string + for _, o := range b.Oracles { + if o.RequiresFork3Hardware { + continue + } + sid := o.SliceID + if o.SplitMode { + if !completedCORE[sid] { + coreOK := true + for _, dep := range o.CoreDependencies { + if dep == "C16" { + if !completedCORE[dep] && !completedFULL[dep] { + coreOK = false + break + } + } else if !completedFULL[dep] { + coreOK = false + break + } + } + if coreOK { + ready = append(ready, sid+":CORE") + } + } else if !completedFULL[sid] { + qualOK := true + for _, qdep := range o.QualDependencies { + if !completedFULL[qdep] { + qualOK = false + break + } + } + if qualOK { + ready = append(ready, sid+":QUAL") + } + } + } else if !completedFULL[sid] { + depsOK := true + for _, dep := range o.CoreDependencies { + if !completedFULL[dep] { + depsOK = false + break + } + } + if depsOK { + ready = append(ready, sid+":FULL") + } + } + } + return ready +} + +// ValidateConcurrentWritePRs enforces Fork 2A: at most 2 active write PRs per repository, +// targeting distinct known slices with non-overlapping repository-relative path prefixes. +func (b *Batch) ValidateConcurrentWritePRs(active []ActivePR) error { + if len(active) > b.MaxConcurrentWritePRsPerRepo { + return fmt.Errorf("sliceoracles: %d active write PRs exceed Fork 2A cap of %d", len(active), b.MaxConcurrentWritePRsPerRepo) + } + known := make(map[string]bool, len(b.Oracles)) + for _, o := range b.Oracles { + known[o.SliceID] = true + } + ids := make([]string, len(active)) + paths := make([][]string, len(active)) + for i, pr := range active { + id := strings.TrimSpace(pr.SliceID) + if id == "" || len(pr.TouchedPaths) == 0 { + return errors.New("sliceoracles: active PR must declare sliceId and touchedPaths") + } + if !known[id] { + return fmt.Errorf("sliceoracles: active PR targets unknown slice %q", id) + } + ids[i] = id + for _, p := range pr.TouchedPaths { + clean, err := cleanTouchedPath(p) + if err != nil { + return fmt.Errorf("sliceoracles: slice %s: %w", id, err) + } + paths[i] = append(paths[i], clean) + } + } + for i := range active { + for j := i + 1; j < len(active); j++ { + if ids[i] == ids[j] { + return fmt.Errorf("sliceoracles: concurrent PRs target the same slice %s", ids[i]) + } + for _, p1 := range paths[i] { + for _, p2 := range paths[j] { + if pathsOverlap(p1, p2) { + return fmt.Errorf("sliceoracles: concurrent PRs (%s, %s) overlap on paths %q and %q", ids[i], ids[j], p1, p2) + } + } + } + } + } + return nil +} + +// cleanTouchedPath returns the cleaned repository-relative form of p and rejects +// empty, absolute and repository-escaping paths. +func cleanTouchedPath(p string) (string, error) { + trimmed := strings.TrimSpace(p) + if trimmed == "" || strings.HasPrefix(trimmed, "/") { + return "", fmt.Errorf("touched path %q must be a non-empty repository-relative path", p) + } + clean := path.Clean(trimmed) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") { + return "", fmt.Errorf("touched path %q must stay inside the repository", p) + } + return clean, nil +} + +// pathsOverlap reports whether two cleaned repository-relative paths are equal or nested. +func pathsOverlap(a, b string) bool { + return a == b || strings.HasPrefix(a, b+"/") || strings.HasPrefix(b, a+"/") +} + +// ValidateLKGSafety checks all 6 S1-S6 invariants required for dead-end-free autonomous self-evolution. +func ValidateLKGSafety(s LKGSafetyState) []string { + var blockers []string + if !s.ImmutableBootloaderVerified { + blockers = append(blockers, "S1: immutable bootloader supervisor checksum not verified") + } + if s.PinnedLKGSlots < MinPinnedLKGSlots { + blockers = append(blockers, fmt.Sprintf("S2: requires at least %d locally pinned LKG slots, got %d", MinPinnedLKGSlots, s.PinnedLKGSlots)) + } + if s.SoakMinutes < MinSoakMinutes { + blockers = append(blockers, fmt.Sprintf("S2: requires >= %d minutes soak before LKG promotion, got %d", MinSoakMinutes, s.SoakMinutes)) + } + if !s.WebhookBypassVerified { + blockers = append(blockers, "S3: rollback namespace/channel bypass of admission/conversion webhooks not verified") + } + if !s.PreMigrationSnapshotVerified { + blockers = append(blockers, "S4: pre-migration state snapshot and V_old/V_new round-trip not verified") + } + if !s.MetaRSISelfCheckPassed { + blockers = append(blockers, "S5: meta-RSI noop self-check canary did not pass") + } + if strings.TrimSpace(s.CandidateSHA) == "" { + blockers = append(blockers, "S6: candidateSha is empty") + } else if slices.Contains(s.QuarantinedSHAs, s.CandidateSHA) { + blockers = append(blockers, fmt.Sprintf("S6: candidateSha %s is in incident quarantine and requires a new regression test", s.CandidateSHA)) + } + return blockers +} + +// EvaluateSliceCandidate runs autonomous Fork 1A acceptance for a slice candidate by combining +// oracle membership, hardware boundary check, S1-S6 LKG safety, and the frozen RSI G1-G7 evaluator. +func (b *Batch) EvaluateSliceCandidate(sliceID, phase string, profile rsi.CandidateProfile, lkg LKGSafetyState) (*SliceAcceptanceReceipt, error) { + if err := b.Validate(); err != nil { + return nil, err + } + var oracle *SliceOracle + for i := range b.Oracles { + if b.Oracles[i].SliceID == sliceID { + oracle = &b.Oracles[i] + break + } + } + if oracle == nil { + return nil, fmt.Errorf("sliceoracles: unknown slice %q", sliceID) + } + + receipt := &SliceAcceptanceReceipt{ + SliceID: sliceID, + Phase: phase, + Verdict: rsi.VerdictFail, + } + + if oracle.RequiresFork3Hardware { + receipt.Blockers = append(receipt.Blockers, fmt.Sprintf("Fork 3 deferred: slice %s requires additional multi-node/second-provider hardware", sliceID)) + } + + receipt.Blockers = append(receipt.Blockers, ValidateLKGSafety(lkg)...) + + rep, err := rsi.EvaluateCandidate(profile) + if err != nil { + return nil, err + } + receipt.RSIReport = &rep + if rep.Verdict != rsi.VerdictPass { + receipt.Blockers = append(receipt.Blockers, fmt.Sprintf("RSI G1-G7 evaluator returned verdict=%s (failed=%d, killSwitch=%s)", rep.Verdict, rep.Failed, rep.KillSwitchLevel)) + } + + if len(receipt.Blockers) == 0 { + receipt.Verdict = rsi.VerdictPass + receipt.PromotedToLKG = true + } + return receipt, nil +} diff --git a/pkg/sliceoracles/oracles_test.go b/pkg/sliceoracles/oracles_test.go new file mode 100644 index 0000000..f87f1ec --- /dev/null +++ b/pkg/sliceoracles/oracles_test.go @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package sliceoracles_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/opencloudtech/CloudRING/pkg/rsi" + "github.com/opencloudtech/CloudRING/pkg/sliceoracles" +) + +func loadCanonicalBatch(t *testing.T) *sliceoracles.Batch { + t.Helper() + path := filepath.Join("..", "..", "contracts", "slice-oracles", "oracles-32.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + batch, err := sliceoracles.ParseBatch(data) + if err != nil { + t.Fatalf("ParseBatch: %v", err) + } + return batch +} + +func loadCompliantRSIProfile(t *testing.T) rsi.CandidateProfile { + t.Helper() + path := filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", "compliant-k3-profile.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + prof, err := rsi.ParseProfile(data) + if err != nil { + t.Fatalf("ParseProfile: %v", err) + } + return prof +} + +func validLKGState() sliceoracles.LKGSafetyState { + return sliceoracles.LKGSafetyState{ + ImmutableBootloaderVerified: true, + PinnedLKGSlots: 2, + SoakMinutes: 30, + WebhookBypassVerified: true, + PreMigrationSnapshotVerified: true, + MetaRSISelfCheckPassed: true, + CandidateSHA: "a326123af6ca994ce876bb786dff7683f181ca36", + QuarantinedSHAs: []string{"deadbeef00000000000000000000000000000000"}, + } +} + +func TestCanonical32OracleBatchAndNoNewHWPartition(t *testing.T) { + batch := loadCanonicalBatch(t) + noHW, deferredHW := batch.PartitionNoNewHardware() + if len(noHW) != 28 { + t.Fatalf("expected 28 no-new-hardware slices, got %d", len(noHW)) + } + if len(deferredHW) != 4 { + t.Fatalf("expected 4 deferred Fork 3 hardware slices, got %d", len(deferredHW)) + } + + // Initially with C01 accepted, SP01 and C06 are immediately ready in the 2-agent parallel queue. + completedCORE := map[string]bool{"C01": true} + completedFULL := map[string]bool{"C01": true} + ready := batch.ReadyNoNewHardwareQueue(completedCORE, completedFULL) + if len(ready) != 2 || ready[0] != "SP01:FULL" || ready[1] != "C06:FULL" { + t.Fatalf("expected initial ready pair [SP01:FULL C06:FULL], got %v", ready) + } + + // Verify Fork 2A split unlocking: once C01..C11 are accepted (before C13 billing and C14 experience!), + // C15, C16, and DNSAAS CORE builds unlock immediately in parallel with C12. + for _, s := range []string{"SP01", "C02", "C03", "C04", "C05", "C06", "C07", "C08", "C09", "C10", "C11"} { + completedCORE[s] = true + completedFULL[s] = true + } + readyAfterC11 := batch.ReadyNoNewHardwareQueue(completedCORE, completedFULL) + wantAfterC11 := []string{"C12:FULL", "C15:CORE", "C16:CORE", "DNSAAS:CORE"} + if len(readyAfterC11) != len(wantAfterC11) { + t.Fatalf("after C11 expected %v, got %v", wantAfterC11, readyAfterC11) + } + for i := range wantAfterC11 { + if readyAfterC11[i] != wantAfterC11[i] { + t.Fatalf("after C11[%d]: expected %s, got %s", i, wantAfterC11[i], readyAfterC11[i]) + } + } +} + +func TestFork2AConcurrentWritePRValidation(t *testing.T) { + batch := loadCanonicalBatch(t) + + // Valid pair: SP01 and C06 have disjoint paths. + err := batch.ValidateConcurrentWritePRs([]sliceoracles.ActivePR{ + {SliceID: "SP01", TouchedPaths: []string{"pkg/rsi", "pkg/sliceoracles"}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/backup", "pkg/dataact"}}, + }) + if err != nil { + t.Fatalf("expected disjoint 2-PR pair to pass, got: %v", err) + } + + // Reject overlapping paths. + err = batch.ValidateConcurrentWritePRs([]sliceoracles.ActivePR{ + {SliceID: "C02", TouchedPaths: []string{"internal/portal/ui"}}, + {SliceID: "C14", TouchedPaths: []string{"internal/portal"}}, + }) + if err == nil { + t.Fatal("expected overlapping paths across concurrent PRs to fail") + } + + // Reject >2 active write PRs. + err = batch.ValidateConcurrentWritePRs([]sliceoracles.ActivePR{ + {SliceID: "SP01", TouchedPaths: []string{"pkg/rsi"}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/backup"}}, + {SliceID: "C03", TouchedPaths: []string{"pkg/transactionalstate"}}, + }) + if err == nil { + t.Fatal("expected 3 concurrent write PRs to fail Fork 2A cap of 2") + } + + // Reject inputs that only differ by spelling: dot segments, blank or escaping paths, padded or unknown slice ids. + rejected := []struct { + name string + active []sliceoracles.ActivePR + }{ + {"dot-segment overlap", []sliceoracles.ActivePR{ + {SliceID: "SP01", TouchedPaths: []string{"./pkg/rsi"}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/rsi/"}}, + }}, + {"blank path", []sliceoracles.ActivePR{ + {SliceID: "SP01", TouchedPaths: []string{" "}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/backup"}}, + }}, + {"escaping path", []sliceoracles.ActivePR{ + {SliceID: "SP01", TouchedPaths: []string{"pkg/../../etc"}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/backup"}}, + }}, + {"absolute path", []sliceoracles.ActivePR{ + {SliceID: "SP01", TouchedPaths: []string{"/pkg/rsi"}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/backup"}}, + }}, + {"padded duplicate slice", []sliceoracles.ActivePR{ + {SliceID: "SP01 ", TouchedPaths: []string{"pkg/rsi"}}, + {SliceID: "SP01", TouchedPaths: []string{"pkg/sliceoracles"}}, + }}, + {"unknown slice", []sliceoracles.ActivePR{ + {SliceID: "C99", TouchedPaths: []string{"pkg/rsi"}}, + {SliceID: "C06", TouchedPaths: []string{"pkg/backup"}}, + }}, + } + for _, tc := range rejected { + if err := batch.ValidateConcurrentWritePRs(tc.active); err == nil { + t.Errorf("%s: expected rejection, got nil", tc.name) + } + } +} + +func TestAutonomousSliceEvaluationAndDeferredFork3Guard(t *testing.T) { + batch := loadCanonicalBatch(t) + prof := loadCompliantRSIProfile(t) + lkg := validLKGState() + + // SP01 is a no-new-hardware slice and passes when RSI G1-G7 + S1-S6 pass. + rec, err := batch.EvaluateSliceCandidate("SP01", "FULL", prof, lkg) + if err != nil { + t.Fatalf("EvaluateSliceCandidate SP01: %v", err) + } + if rec.Verdict != rsi.VerdictPass || !rec.PromotedToLKG { + t.Fatalf("expected SP01 to pass and promote to LKG, got verdict=%s blockers=%v", rec.Verdict, rec.Blockers) + } + + // C27 requires deferred Fork 3 second-provider hardware and must be blocked now. + recHW, err := batch.EvaluateSliceCandidate("C27", "FULL", prof, lkg) + if err != nil { + t.Fatalf("EvaluateSliceCandidate C27: %v", err) + } + if recHW.Verdict != rsi.VerdictFail || recHW.PromotedToLKG { + t.Fatalf("expected deferred Fork 3 slice C27 to be blocked while hardware is deferred") + } + + // Quarantined SHA (S6) must block promotion even on a no-new-hardware slice. + badLKG := validLKGState() + badLKG.CandidateSHA = "deadbeef00000000000000000000000000000000" + recQ, err := batch.EvaluateSliceCandidate("SP01", "FULL", prof, badLKG) + if err != nil { + t.Fatalf("EvaluateSliceCandidate quarantined: %v", err) + } + if recQ.Verdict != rsi.VerdictFail || recQ.PromotedToLKG { + t.Fatalf("expected quarantined SHA to fail S6 gate") + } +} diff --git a/roadmap/DELIVERY_SLICES.md b/roadmap/DELIVERY_SLICES.md index c7cd4f7..c43d13b 100644 --- a/roadmap/DELIVERY_SLICES.md +++ b/roadmap/DELIVERY_SLICES.md @@ -19,15 +19,25 @@ replace required independent infrastructure or a real human participant. not assert readiness or completed predecessors. `delivered` still requires every qualification dependency and the original full evidence contract. C dependencies below control implementation entry; read-only preparation can happen earlier. -External authorization or capacity blocks its affected live result while -independent authorized work continues. +Under Fork 1A (`PROCDEC`), all 32 slice acceptance oracles (`C01–C28`, `SP01`, +`DBAAS`, `DNSAAS`, `FILESAAS`) are frozen in +`contracts/slice-oracles/oracles-32.json` and evaluated autonomously by `VER` +and the RSI gate (`pkg/rsi`, `pkg/sliceoracles`). Under Fork 2A, `C15`, `C16`, +`C17`, `C20`, `DBAAS`, `DNSAAS` and `FILESAAS` split core implementation entry +(`CORE`, after `C11` and their technical prerequisite) from full billing and +experience qualification (`QUAL`, after `C13` and `C14`); `C24` enters after +`C14` instead of `C23`; and `C20` does not wait for `C19`. Hardware-gated +multi-node NVMe/GPU/second-provider qualification (`Fork 3`) is deferred while +all no-new-hardware slices advance automatically. External authorization or +capacity blocks its affected live result while independent authorized work +continues. ## Slice sequence | Slice | Entry | Accepted working result | Original coverage and full-qualification boundary | | --- | --- | --- | --- | | C01 | None | Reproducible public delivery baseline; useful delivery fixes accepted; every current-family WIP root classified and preserved; invalid candidate rejected; fixed numeric measurement profiles. | Partial G00 / CR-G00-DELIVERY. Current endpoint, GitOps revision and separately verified or unknown host-binary provenance are recorded. Full downstream/live G00 qualification follows in C08. Proposed downstream requirements are not automatically ratified. | -| C02 | C01 accepted | Public isolated developer create, repeat, restart, positive/negative smoke, destroy with zero owned residue and recreate; same profile beside reference installation. | Full G01 / CR-G01-DEV-INSTALL, including clean-room tutorial and human/installation profile. Production shortcuts must be rejected. G01 does not wait for full G00. | +| C02 | C01 and SP01 accepted | Public isolated developer create, repeat, restart, positive/negative smoke, destroy with zero owned residue and recreate; same profile beside reference installation. | Full G01 / CR-G01-DEV-INSTALL, including clean-room tutorial and human/installation profile. Production shortcuts must be rejected. G01 does not wait for full G00. | | C03 | C02 | Public durable project/operation API, CLI and read-only UI; transactions, outbox, crash, retry, concurrency, migration and restore. | Partial G03 / CR-G03-KERNEL. Full directory, production failover and scale qualification in C09. | | C04 | C03 | Two developer tenants invite/login/use own project/logout; token/session revocation and direct API/browser isolation negatives. | Partial G04 / CR-G04-IDENTITY and G05 / CR-G05-IAM. Full production identity, recovery and workload identity in C09. | | C05 | C04; applicable C06 barriers for reference-site mutations | Real developer VM through public Go/OCS; login, project, create, bounded access, write/read digest, retry/restart, one VM, delete and cleanup; one real user performs a useful task. | Partial G07–G09 and G15 / CR-G15-COMPUTE. Quota/reservation and explicit zero-priced-metered or non-billable policy required now. Pilot dependencies are owned and bounded; Network/Volume/Image products and full Compute remain incomplete. | @@ -40,20 +50,24 @@ independent authorized work continues. | C12 | C11 | Complete real order/subscription/entitlement lifecycle, quota/capacity, compensation, crash/retry/cancel and restore through both connector paths. | Full G09 / CR-G09-LIFECYCLE. | | C13 | C12 | Real test usage to exact balanced ledger, invoice, correction, budget and publisher share with replay/restore/HA and eligibility denial. | Full G10 / CR-G10-BILLING. Uses isolated non-financial data; actual customer charging needs separate authority. | | C14 | C13 | Equivalent complete API/CLI/portal/agent actions, policy, identity, audit, resumability, accessible UI and authoritative backend comparison. | Full G11 / CR-G11-EXPERIENCE. | -| C15 | C14, C07 | Standalone OCS Network lifecycle, dual-stack connectivity, tenant denial, quota/usage, actual failure and cleanup. | Full G12 / CR-G12-NETWORK. | -| C16 | C14, C08 | Standalone OCS Volume lifecycle, real CSI/snapshot/restore, encryption, node-loss digest and accounting. | Full G13 / CR-G13-VOLUME. Independent of C15; production rejects replication-one or unknown topology. | -| C17 | C16 | Standalone OCS Image/Artifact lifecycle, immutable provenance, VM/OCI round trip, resumable import, scan/quarantine, boot harness, restore and safe GC. | Full G14 / CR-G14-ARTIFACT. Boot harness does not require completed C18. | +| C15 | CORE: C11, C07; QUAL: C13, C14 | Standalone OCS Network lifecycle, dual-stack connectivity, tenant denial, quota/usage, actual failure and cleanup. | Full G12 / CR-G12-NETWORK. Core implementation starts after C11 and C07; full billing/experience qualification completes after C13 and C14 (Fork 2A). | +| C16 | CORE: C11, C08; QUAL: C13, C14 | Standalone OCS Volume lifecycle, real CSI/snapshot/restore, encryption, node-loss digest and accounting. | Full G13 / CR-G13-VOLUME. Independent of C15; core implementation starts after C11 and C08, full qualification after C13 and C14 (Fork 2A). Production rejects replication-one or unknown topology. | +| C17 | CORE: C11, C16; QUAL: C13, C14 | Standalone OCS Image/Artifact lifecycle, immutable provenance, VM/OCI round trip, resumable import, scan/quarantine, boot harness, restore and safe GC. | Full G14 / CR-G14-ARTIFACT. Core implementation starts after C11 and C16; full qualification after C13 and C14 (Fork 2A). Boot harness does not require completed C18. | | C18 | C15, C16, C17 | Complete Compute through ordinary Network/Volume/Image entitlements; full use, access, billing, failure and recovery; existing pilot user migrates or exports data safely. | Full G15 / CR-G15-COMPUTE. Supported cold recovery is measured; live migration is claimed only with its actual prerequisites and proof. | | C19 | C18 | Real tenant Kubernetes, workload/data, short-lived access, scale, signed upgrade, failure, isolated restore and full cleanup. | Full G16 / CR-G16-KUBERNETES. | -| C20 | C14, C16, C19 | Real local/reference and remote S3 lifecycle, objects/digests, credentials, retention, metering, failure and isolated restore. | Full G17 / CR-G17-OBJECT. Replace placeholder catalog path; unsupported durability cannot pass full product acceptance. | +| C20 | CORE: C11, C16; QUAL: C13, C14 | Real local/reference and remote S3 lifecycle, objects/digests, credentials, retention, metering, failure and isolated restore. | Full G17 / CR-G17-OBJECT. Does not wait for C19 (Fork 2A); core starts after C11 and C16, full qualification after C13 and C14. Replace placeholder catalog path; unsupported durability cannot pass full product acceptance. | | C21 | C19, C20 | Customer self-service backup and verified off-cell restore of every claimed data class, ownership, retention, retries, RPO/RTO, cost and cleanup. | Full G18 / CR-G18-BACKUP. | | C22 | C19, C21 | Supported SSH/Kubernetes/console access request, approval, grant, use, expiry/revoke and recovery without reviving grants. | Full G19 / CR-G19-ACCESS. | | C23 | C22 | Durable support case, consent, bounded redacted bundle, controlled access, routing outage, escalation, resolve and restore. | Full G20 / CR-G20-SUPPORT. | -| C24 | C23 | Real authorized remote/API-only system with complete lifecycle/usage and two independent replaceable adapters; independent developer uses public core only. | Full G21 / CR-G21-EXTERNAL. Unimplemented external templates remain templates. | -| C25 | All C01–C24; functional G22 entry below | Adjacent signed release upgrade, failed canary, rollback/restore boundary and integrated supported failure campaign under continuous identity/operation/data/billing probes. | Full G23 / CR-G23-RESILIENCE and CR-G23-OPERATIONS-ENTRY. Functional G22 proof precedes campaign; human G22 qualification follows in C26. | +| C24 | C14 | Real authorized remote/API-only system with complete lifecycle/usage and two independent replaceable adapters; independent developer uses public core only. | Full G21 / CR-G21-EXTERNAL. Enters after C14 instead of C23 (Fork 2A). Unimplemented external templates remain templates. | +| SP01 | C01 | SafePush admission, provenance verification and two-slot LKG rollback guardrails (`S1–S6`). | Preparatory G00 / CR-G00-DELIVERY and autonomous RSI safety gate before full C02/C08 promotion. | +| DBAAS | CORE: C11, C16; QUAL: C13, C14, C21 | Standalone OCS Managed PostgreSQL (CNPG) lifecycle, HA failover, point-in-time recovery, quota/metering and tenant isolation. | Mandatory 1.0 service category (Fork 3 2026-10-14); core starts after C11 and C16, full qualification after C13, C14 and C21 (Fork 2A). | +| DNSAAS | CORE: C11, C07; QUAL: C13, C14, C15 | Standalone OCS Managed DNS lifecycle, authoritative zone/record reconciliation, tenant delegation, quota/metering and DNSSEC/split-horizon verification. | Mandatory 1.0 service category (Fork 3 2026-10-14); core starts after C11 and C07, full qualification after C13, C14 and C15 (Fork 2A). | +| FILESAAS | CORE: C11, C16; QUAL: C13, C14 | Standalone OCS Shared File Storage (RWX NFS/CephFS) lifecycle, mount isolation, snapshot/restore, quota/metering and cleanup. | Table-stakes 1.0 service category (6 of 7 competitors, Wave 6); core starts after C11 and C16, full qualification after C13 and C14 (Fork 2A). | +| C25 | All C01–C24, SP01, DBAAS, DNSAAS, FILESAAS; functional G22 entry below | Adjacent signed release upgrade, failed canary, rollback/restore boundary and integrated supported failure campaign under continuous identity/operation/data/billing probes. | Full G23 / CR-G23-RESILIENCE and CR-G23-OPERATIONS-ENTRY. Functional G22 proof precedes campaign; human G22 qualification follows in C26. | | C26 | C25 | Independent engineer completes full incident set, diagnosis, repair, rotation, maintenance and 14-day toil measurement on the accepted candidate. | Full G22 / CR-G22-OPERATIONS. Same release as C25; later runtime change creates a new candidate and repeats affected G23 checks. | | C27 | C26 | Independently administered bare-metal provider and second engineer reproduce exact public release, install/use/upgrade/failure/restore/cleanup and original product development. | Full G24 / CR-G24-PORTABILITY. Separate outstanding PhoenixNAP site/IaC/GitLab CI commitment must reach accepted protected provider main with its exact core pin and Stage 9; it remains configuration-only and does not claim a PhoenixNAP deployment. Another independently controlled live provider is allowed but does not cancel this obligation. | -| C28 | All C01–C27 | Final adversarial audit, upstream fixes, propagated regressions, public immutable platform/OCS 1.0, support/compatibility/runbooks and exact independent proof. | Full G27 / CR-G27-RELEASE and all remaining standalone 1.0 legacy obligations, Tasks 25–27 and F1–F5. No required unverified blocker, generic private dependency or hidden manual step. | +| C28 | All C01–C27, SP01, DBAAS, DNSAAS, FILESAAS | Final adversarial audit, upstream fixes, propagated regressions, public immutable platform/OCS 1.0, support/compatibility/runbooks and exact independent proof. | Full G27 / CR-G27-RELEASE and all remaining standalone 1.0 legacy obligations, Tasks 25–27 and F1–F5. No required unverified blocker, generic private dependency or hidden manual step. | G25 / CR-G25-CELLS and G26 / CR-G26-FEDERATION retain their complete original goal files and depend on G27. They are post-1.0 expansion tracks outside C01–C28; diff --git a/roadmap/EXECUTION_CONTRACT.md b/roadmap/EXECUTION_CONTRACT.md index 170b15d..b9864a0 100644 --- a/roadmap/EXECUTION_CONTRACT.md +++ b/roadmap/EXECUTION_CONTRACT.md @@ -96,10 +96,14 @@ complete goal, every qualification dependency and verified evidence. C01 baselin acceptance is the entry to C02/G01; it does not falsely complete G00's later clean-room/downstream/live chain. -Only one active write PR/MR per repository is allowed. Read-only research and -verification may proceed in parallel. A compatible batch of already accepted -public commits may share one downstream pin, but each requirement must remain -traceable. +At most two concurrent active write PRs/MRs per repository are allowed when they +target distinct delivery slices with non-overlapping path ownership (Fork 2A). +Read-only research and verification may proceed in parallel. Slice acceptance +executes autonomously through the pre-approved 32-oracle batch in +`contracts/slice-oracles/oracles-32.json` and the frozen RSI evaluator (`VER`, +Fork 1A), with weekly founder audit and rollback authority. A compatible batch of +already accepted public commits may share one downstream pin, but each +requirement must remain traceable. ## Universal definition of done From 9687db8039edd13bebba1453e50df1187185e0f0 Mon Sep 17 00:00:00 2001 From: Iurii Trukhin Date: Mon, 9 Nov 2026 06:48:07 +0100 Subject: [PATCH 3/3] feat(sp01): separate merge and deploy admission, measure G2 in repo The SP01 oracle could never pass. There was no evaluate command and no slice profile. Every candidate also needed a live canary, deploy history and a recovery drill, although a merge into this repository deploys nothing. pkg/rsi now separates merge admission from deploy admission. Deploy admission runs every gate and needs an S1-S6 receipt. A merge claim with deployment evidence is evaluated as deploy. Merge admission is allowed only into opencloudtech/CloudRING main. A first deploy without history needs K4. Profile keys must match field names exactly, and references must name a record. cloudringctl rsi evaluate runs a slice profile. All 32 slice oracles use it, and the declared class of each oracle covers its allowed paths. cmd/cloudring-mutation measures G2 on the changed lines. It mutates the files go build compiles, needs a green baseline before and after the mutants, and confirms each kill with a second failing run. gremlins v0.6.0 counted build failures as kills and never ran the tests of cmd/ packages, so its statuses are no longer used. EXECUTION_CONTRACT.md documents the admission rules and the G2 method. --- cmd/cloudring-mutation/diff.go | 165 +++++ cmd/cloudring-mutation/execute.go | 359 ++++++++++ cmd/cloudring-mutation/main.go | 168 +++++ cmd/cloudring-mutation/mutation_test.go | 554 +++++++++++++++ cmd/cloudring-mutation/procattr_other.go | 12 + cmd/cloudring-mutation/procattr_unix.go | 18 + cmd/cloudring-mutation/sites.go | 236 +++++++ cmd/cloudring-safepush/arguments_test.go | 90 +++ cmd/cloudring-safepush/main.go | 14 +- cmd/cloudringctl/main.go | 3 + cmd/cloudringctl/rsi.go | 65 ++ cmd/cloudringctl/rsi_test.go | 173 +++++ .../fixtures/compliant-k3-profile.json | 12 + .../fixtures/merge-only-k3-profile.json | 48 ++ contracts/slice-oracles/oracles-32.json | 80 ++- contracts/slice-oracles/profiles/SP01.json | 93 +++ pkg/rsi/admission_test.go | 359 ++++++++++ pkg/rsi/classify_limits_test.go | 226 ++++++ pkg/rsi/decision_limits_test.go | 218 ++++++ pkg/rsi/gate_limits_test.go | 163 +++++ pkg/rsi/rsi.go | 645 +++++++++++++++--- pkg/rsi/rsi_fuzz_test.go | 34 +- pkg/rsi/rsi_test.go | 10 + pkg/safepush/github.go | 4 +- pkg/safepush/github_fields_test.go | 347 ++++++++++ pkg/safepush/policy_limits_test.go | 170 +++++ pkg/sliceoracles/admission_test.go | 148 ++++ pkg/sliceoracles/oracles.go | 80 ++- pkg/sliceoracles/validate_limits_test.go | 111 +++ roadmap/EXECUTION_CONTRACT.md | 42 ++ 30 files changed, 4449 insertions(+), 198 deletions(-) create mode 100644 cmd/cloudring-mutation/diff.go create mode 100644 cmd/cloudring-mutation/execute.go create mode 100644 cmd/cloudring-mutation/main.go create mode 100644 cmd/cloudring-mutation/mutation_test.go create mode 100644 cmd/cloudring-mutation/procattr_other.go create mode 100644 cmd/cloudring-mutation/procattr_unix.go create mode 100644 cmd/cloudring-mutation/sites.go create mode 100644 cmd/cloudring-safepush/arguments_test.go create mode 100644 cmd/cloudringctl/rsi.go create mode 100644 cmd/cloudringctl/rsi_test.go create mode 100644 contracts/rsi-governance/fixtures/merge-only-k3-profile.json create mode 100644 contracts/slice-oracles/profiles/SP01.json create mode 100644 pkg/rsi/admission_test.go create mode 100644 pkg/rsi/classify_limits_test.go create mode 100644 pkg/rsi/decision_limits_test.go create mode 100644 pkg/rsi/gate_limits_test.go create mode 100644 pkg/safepush/github_fields_test.go create mode 100644 pkg/safepush/policy_limits_test.go create mode 100644 pkg/sliceoracles/admission_test.go create mode 100644 pkg/sliceoracles/validate_limits_test.go diff --git a/cmd/cloudring-mutation/diff.go b/cmd/cloudring-mutation/diff.go new file mode 100644 index 0000000..d43321a --- /dev/null +++ b/cmd/cloudring-mutation/diff.go @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +var hunkHeader = regexp.MustCompile(`^@@ -\d+(?:,(\d+))? \+(\d+)(?:,(\d+))? @@`) + +// fileHeaders are the extended header lines git prints for a file when +// renames, copies and binary output are off. +var fileHeaders = []string{"index ", "old mode ", "new mode ", "new file mode ", "deleted file mode ", "--- "} + +// candidateChanges returns the lines of the working tree under dirs that the +// candidate adds or changes against base, and its untracked new files, with +// paths relative to root (which may be below the repository top level). A +// renamed file counts as new, and every file is compared as text, so neither +// rename detection nor diff attributes can hide a changed line. +func candidateChanges(root, base string, dirs []string) (changeSet, error) { + diff, err := git(root, append([]string{"-c", "core.quotePath=false", "diff", "--no-renames", "--text", "--no-textconv", + "--no-ext-diff", "--submodule=short", "-U0", "--no-color", "--relative", "--src-prefix=a/", "--dst-prefix=b/", base, "--"}, dirs...)...) + if err != nil { + return changeSet{}, err + } + changes, err := parseUnifiedDiff(diff) + if err != nil { + return changeSet{}, err + } + untracked, err := git(root, append([]string{"ls-files", "-z", "--others", "--exclude-standard", "--"}, dirs...)...) + if err != nil { + return changeSet{}, err + } + for _, name := range strings.Split(string(untracked), "\x00") { + if name != "" { + changes.newFiles[name] = true + } + } + return changes, nil +} + +// parseUnifiedDiff reads the output of candidateChanges' git diff: every line +// of a hunk's new side is a changed line. Any line it does not recognise, +// including a binary notice, a rename header or a quoted file name, is an +// error, so the measurement stops instead of skipping a file. +func parseUnifiedDiff(diff []byte) (changeSet, error) { + changes := changeSet{newFiles: map[string]bool{}, lines: map[string]map[int]bool{}} + const ( + outside = iota + header + hunk + ) + state, current, named := outside, "", false + oldLeft, newLeft := 0, 0 + scanner := bufio.NewScanner(bytes.NewReader(diff)) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for scanner.Scan() { + line := scanner.Text() + if state == hunk && (oldLeft > 0 || newLeft > 0 || strings.HasPrefix(line, `\`)) { + switch { + case strings.HasPrefix(line, "-") && oldLeft > 0: + oldLeft-- + case strings.HasPrefix(line, "+") && newLeft > 0: + newLeft-- + case !strings.HasPrefix(line, `\`): + return changeSet{}, fmt.Errorf("unexpected line in hunk %q", line) + } + continue + } + switch { + case strings.HasPrefix(line, "diff --git "): + state, current, named = header, "", false + case state != outside && strings.HasPrefix(line, "@@ "): + m := hunkHeader.FindStringSubmatch(line) + if m == nil || !named { + return changeSet{}, fmt.Errorf("unsupported hunk header %q", line) + } + var start int + var err error + if oldLeft, err = hunkCount(m[1]); err != nil { + return changeSet{}, err + } + if start, err = strconv.Atoi(m[2]); err != nil { + return changeSet{}, err + } + if newLeft, err = hunkCount(m[3]); err != nil { + return changeSet{}, err + } + if current != "" && newLeft > 0 { + if changes.lines[current] == nil { + changes.lines[current] = map[int]bool{} + } + for n := start; n < start+newLeft; n++ { + changes.lines[current][n] = true + } + } + state = hunk + case state == header && strings.HasPrefix(line, "+++ "): + name, err := newSideName(strings.TrimPrefix(line, "+++ ")) + if err != nil { + return changeSet{}, err + } + current, named = name, true + case state == header && hasPrefix(line, fileHeaders): + case strings.HasPrefix(line, "Binary files "): + return changeSet{}, fmt.Errorf("binary diff hides changed lines: %q", line) + default: + return changeSet{}, fmt.Errorf("unsupported diff line %q", line) + } + } + if oldLeft > 0 || newLeft > 0 { + return changeSet{}, fmt.Errorf("diff ends inside a hunk") + } + return changes, scanner.Err() +} + +// newSideName returns the path of a "+++ " header, "" for a deleted file. Git +// appends a TAB to a name that contains a space and quotes names with other +// special characters; a quoted name is refused. +func newSideName(name string) (string, error) { + if name == "/dev/null" { + return "", nil + } + if trimmed := strings.TrimSuffix(name, "\t"); trimmed != name && strings.Contains(trimmed, " ") { + name = trimmed + } + if !strings.HasPrefix(name, "b/") || strings.ContainsAny(name, "\t\"\\") { + return "", fmt.Errorf("unsupported diff file name %q", name) + } + return strings.TrimPrefix(name, "b/"), nil +} + +func hunkCount(s string) (int, error) { + if s == "" { + return 1, nil + } + return strconv.Atoi(s) +} + +func hasPrefix(line string, prefixes []string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(line, prefix) { + return true + } + } + return false +} + +func git(root string, args ...string) ([]byte, error) { + cmd := exec.Command("git", append([]string{"-C", root}, args...)...) // #nosec G204 -- fixed git binary; the base revision is checked not to be an option. + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return out, nil +} diff --git a/cmd/cloudring-mutation/execute.go b/cmd/cloudring-mutation/execute.go new file mode 100644 index 0000000..1d62948 --- /dev/null +++ b/cmd/cloudring-mutation/execute.go @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +// Mutant statuses. KILLED, LIVED, TIMED OUT and UNSTABLE count in G2; a mutant +// that does not build (NOT VIABLE) or that go vet rejects during go test +// (KILLED_BY_VET) is not viable and is not counted. UNSTABLE is a failure that +// did not repeat under kill confirmation; it counts as not killed. +const ( + statusKilled = "KILLED" + statusLived = "LIVED" + statusTimedOut = "TIMED OUT" + statusUnstable = "UNSTABLE" + statusNotViable = "NOT VIABLE" + statusKilledByVet = "KILLED_BY_VET" +) + +// buildAllowance is the time a mutant run may take beyond the go test +// -timeout for compiling and linking before it is stopped as TIMED OUT. +const buildAllowance = 5 * time.Minute + +type result struct { + site + Status string + Failure string // first "--- FAIL" line, if any +} + +// testRun is the outcome of one go test invocation. +type testRun struct { + err error + output string + // expired is set when the run outlived the -timeout plus buildAllowance. + expired bool +} + +// classify maps a go test run of a mutant to its status. builds reports +// whether the mutant compiles with vet off; it is consulted only for a run +// that go test reports as a build or setup failure. +func classify(run testRun, builds func() bool) string { + switch { + case run.expired || strings.Contains(run.output, "panic: test timed out after"): + return statusTimedOut + case run.err == nil: + return statusLived + case strings.Contains(run.output, "[build failed]") || strings.Contains(run.output, "[setup failed]"): + if builds() { + return statusKilledByVet + } + return statusNotViable + default: + return statusKilled + } +} + +func firstFailure(output string) string { + for _, line := range strings.Split(output, "\n") { + if trimmed := strings.TrimSpace(line); strings.HasPrefix(trimmed, "--- FAIL:") { + return trimmed + } + } + return "" +} + +// failedTest returns the name in the first "--- FAIL: name (duration)" line. +func failedTest(output string) string { + if fields := strings.Fields(firstFailure(output)); len(fields) > 2 { + return fields[2] + } + return "" +} + +// goTest runs go test in dir for the package directory pkg, in its own process +// group so that a test that signals its group cannot stop the measurement. +func goTest(ctx context.Context, dir, pkg string, timeout time.Duration, extra ...string) testRun { + ctx, cancel := context.WithTimeout(ctx, timeout+buildAllowance) + defer cancel() + args := append([]string{"test", "-count=1", "-timeout", timeout.String()}, extra...) + args = append(args, "./"+pkg+"/") + cmd := exec.CommandContext(ctx, "go", args...) // #nosec G204 -- fixed go binary; the package path is validated relative to the module root. + cmd.Dir = dir + var out bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &out + ownProcessGroup(cmd) + cmd.WaitDelay = 10 * time.Second + err := cmd.Run() + return testRun{err: err, output: out.String(), expired: errors.Is(ctx.Err(), context.DeadlineExceeded)} +} + +// execute runs every mutant once, each worker in its own copy of the module. +// Every package with a mutant must pass go test unmutated in a fresh copy +// before the mutants run and in every copy after them; otherwise a failure +// that does not depend on the mutant would count as a kill. +func execute(ctx context.Context, root string, sites []site, timeout time.Duration, workers int) ([]result, error) { + if len(sites) == 0 { + return nil, nil + } + scratch, err := os.MkdirTemp("", "cloudring-mutation-") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(scratch) }() + copies := make([]string, workers) + for i := range copies { + copies[i] = filepath.Join(scratch, fmt.Sprintf("module-%d", i)) + if err := copyModule(root, copies[i]); err != nil { + return nil, err + } + } + pkgs := sitePackages(sites) + if err := baseline(ctx, copies[:1], pkgs, timeout); err != nil { + return nil, fmt.Errorf("baseline before the mutants: %w", err) + } + results := make([]result, len(sites)) + jobs := make(chan int) + errs := make(chan error, workers) + var wg sync.WaitGroup + for _, copyDir := range copies { + wg.Add(1) + go func(copyDir string) { + defer wg.Done() + for i := range jobs { + r, err := runMutant(ctx, copyDir, sites[i], timeout) + if err != nil { + errs <- err + return + } + results[i] = r + } + }(copyDir) + } + var runErr error +feed: + for i := range sites { + select { + case jobs <- i: + case runErr = <-errs: + break feed + case <-ctx.Done(): + runErr = ctx.Err() + break feed + } + } + close(jobs) + wg.Wait() + if runErr == nil { + select { + case runErr = <-errs: + default: + } + } + if runErr != nil { + return nil, runErr + } + if err := baseline(ctx, copies, pkgs, timeout); err != nil { + return nil, fmt.Errorf("baseline after the mutants: %w", err) + } + return results, nil +} + +// sitePackages returns the package directories that have mutants, sorted. +func sitePackages(sites []site) []string { + seen := map[string]bool{} + var pkgs []string + for _, s := range sites { + if pkg := path.Dir(s.File); !seen[pkg] { + seen[pkg] = true + pkgs = append(pkgs, pkg) + } + } + sort.Strings(pkgs) + return pkgs +} + +// baseline requires every package to pass go test unmutated in each copy. +func baseline(ctx context.Context, copies, pkgs []string, timeout time.Duration) error { + errs := make([]error, len(copies)) + var wg sync.WaitGroup + for i, copyDir := range copies { + wg.Add(1) + go func(i int, copyDir string) { + defer wg.Done() + for _, pkg := range pkgs { + if run := goTest(ctx, copyDir, pkg, timeout); run.err != nil { + errs[i] = fmt.Errorf("go test ./%s/ fails without a mutant in copy %d: %s", pkg, i, failureSummary(run)) + return + } + } + }(i, copyDir) + } + wg.Wait() + if err := ctx.Err(); err != nil { + return err + } + return errors.Join(errs...) +} + +// failureSummary names why a go test run failed in one line. +func failureSummary(run testRun) string { + if classify(run, func() bool { return true }) == statusTimedOut { + return "timed out" + } + if failure := firstFailure(run.output); failure != "" { + return failure + } + for _, line := range strings.Split(run.output, "\n") { + if line = strings.TrimSpace(line); line != "" && !strings.HasPrefix(line, "# ") { + return line + } + } + return run.err.Error() +} + +func runMutant(ctx context.Context, copyDir string, s site, timeout time.Duration) (result, error) { + module, err := os.OpenRoot(copyDir) + if err != nil { + return result{}, err + } + defer func() { _ = module.Close() }() + name := filepath.FromSlash(s.File) + info, err := module.Stat(name) + if err != nil { + return result{}, err + } + original, err := module.ReadFile(name) + if err != nil { + return result{}, err + } + mutated, err := apply(original, s) + if err != nil { + return result{}, err + } + perm := info.Mode().Perm() + if err := module.WriteFile(name, mutated, perm); err != nil { + return result{}, err + } + pkg := path.Dir(s.File) + run := goTest(ctx, copyDir, pkg, timeout) + status := classify(run, func() bool { + return goTest(ctx, copyDir, pkg, timeout, "-vet=off", "-run", "^$").err == nil + }) + if status == statusKilled { + if status, err = confirmKill(ctx, module, copyDir, pkg, name, original, mutated, perm, timeout, failedTest(run.output)); err != nil { + return result{}, err + } + } + if err := module.WriteFile(name, original, perm); err != nil { + return result{}, fmt.Errorf("restore %s: %w", s.File, err) + } + if ctx.Err() != nil { + return result{}, ctx.Err() + } + return result{site: s, Status: status, Failure: firstFailure(run.output)}, nil +} + +// confirmKill reruns the package of a mutant whose first run failed in test: +// unmutated it must pass twice in a row, and the mutant must then fail again +// with the same first failing test. Otherwise the failure did not depend on +// the mutant alone and the mutant is UNSTABLE. Two consecutive unmutated runs +// break a failure that alternates between runs. +func confirmKill(ctx context.Context, module *os.Root, copyDir, pkg, name string, original, mutated []byte, perm fs.FileMode, timeout time.Duration, test string) (string, error) { + if err := module.WriteFile(name, original, perm); err != nil { + return "", err + } + for range 2 { + if goTest(ctx, copyDir, pkg, timeout).err != nil { + return statusUnstable, nil + } + } + if err := module.WriteFile(name, mutated, perm); err != nil { + return "", err + } + again := goTest(ctx, copyDir, pkg, timeout) + if classify(again, func() bool { return true }) != statusKilled || failedTest(again.output) != test { + return statusUnstable, nil + } + return statusKilled, nil +} + +// copyModule copies the module tree at root to the new directory dst without +// version control metadata. Symbolic links are recreated, not followed, and +// every access stays inside the two trees. +func copyModule(root, dst string) error { + src, err := os.OpenRoot(root) + if err != nil { + return err + } + defer func() { _ = src.Close() }() + if err := os.Mkdir(dst, 0o700); err != nil { + return err + } + out, err := os.OpenRoot(dst) + if err != nil { + return err + } + defer func() { _ = out.Close() }() + return fs.WalkDir(src.FS(), ".", func(name string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.Name() == ".git" && name != "." { + if entry.IsDir() { + return fs.SkipDir + } + return nil + } + switch mode := entry.Type(); { + case entry.IsDir(): + return out.MkdirAll(name, 0o700) + case mode&fs.ModeSymlink != 0: + link, err := src.Readlink(name) + if err != nil { + return err + } + return out.Symlink(link, name) + case mode.IsRegular(): + return copyFile(src, out, name) + } + return nil + }) +} + +func copyFile(src, dst *os.Root, name string) error { + in, err := src.Open(name) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + info, err := in.Stat() + if err != nil { + return err + } + out, err := dst.OpenFile(name, os.O_CREATE|os.O_EXCL|os.O_WRONLY, info.Mode().Perm()) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + return out.Close() +} diff --git a/cmd/cloudring-mutation/main.go b/cmd/cloudring-mutation/main.go new file mode 100644 index 0000000..c9af1c0 --- /dev/null +++ b/cmd/cloudring-mutation/main.go @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +// Command cloudring-mutation measures the G2 diff mutation score defined in +// roadmap/EXECUTION_CONTRACT.md. It generates the mutants of gremlins v0.6.0 +// (the default mutators plus INVERT_LOGICAL) in the files go build compiles, +// on the lines a candidate adds or changes against a base revision and on its +// untracked new files, runs `go test` on each mutant's package in a private +// copy of the module, and prints one row per mutant followed by the score. +// Every package with a mutant must pass `go test` unmutated in the copy before +// the mutants and in every copy after them, and a failing run counts as a kill +// only when it repeats (see confirmKill). +// +// Exit status: 0 when the score meets 4/5, 1 when it does not, 2 for invalid +// arguments or a failed measurement: a package that fails without a mutant, a +// diff the tool cannot read in full, or a compiled file it cannot mutate. +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path" + "path/filepath" + "sort" + "strings" + "time" +) + +// defaultTimeout is the go test -timeout of one mutant run. +const defaultTimeout = 2 * time.Minute + +const usage = "usage: cloudring-mutation --base REV --packages DIR[,DIR...] [--root DIR] [--timeout 2m] [--workers 4]" + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + code := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) +} + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("cloudring-mutation", flag.ContinueOnError) + flags.SetOutput(io.Discard) + root := flags.String("root", ".", "module root") + base := flags.String("base", "", "base revision, normally the merge-base of the candidate") + packages := flags.String("packages", "", "comma-separated package directories relative to the root") + timeout := flags.Duration("timeout", defaultTimeout, "go test -timeout of each mutant run") + workers := flags.Int("workers", 4, "mutants run in parallel, each in its own copy of the module") + if err := flags.Parse(args); err != nil || flags.NArg() != 0 || *base == "" || strings.HasPrefix(*base, "-") || + *packages == "" || *timeout <= 0 || *workers < 1 { + _, _ = fmt.Fprintln(stderr, usage) + return 2 + } + if info, err := os.Stat(filepath.Join(*root, "go.mod")); err != nil || !info.Mode().IsRegular() { + _, _ = fmt.Fprintln(stderr, "cloudring-mutation: --root must be a module root containing go.mod") + return 2 + } + dirs, err := packageDirs(*root, *packages) + if err != nil { + _, _ = fmt.Fprintln(stderr, "cloudring-mutation:", err) + return 2 + } + changes, err := candidateChanges(*root, *base, dirs) + if err != nil { + _, _ = fmt.Fprintln(stderr, "cloudring-mutation:", err) + return 2 + } + sites, err := enumerate(*root, dirs, changes) + if err != nil { + _, _ = fmt.Fprintln(stderr, "cloudring-mutation:", err) + return 2 + } + results, err := execute(ctx, *root, sites, *timeout, *workers) + if err != nil { + _, _ = fmt.Fprintln(stderr, "cloudring-mutation:", err) + return 2 + } + met, err := report(stdout, *base, dirs, *timeout, *workers, results) + if err != nil { + return 2 + } + if !met { + return 1 + } + return 0 +} + +// packageDirs validates the comma-separated package directories: each must be +// a relative directory inside the module root. +func packageDirs(root, list string) ([]string, error) { + var dirs []string + for _, item := range strings.Split(list, ",") { + dir := path.Clean(filepath.ToSlash(strings.TrimSpace(item))) + if dir == "." || dir == "" || path.IsAbs(dir) || dir == ".." || strings.HasPrefix(dir, "../") || strings.HasPrefix(dir, "-") { + return nil, fmt.Errorf("package directory %q must be a directory inside the module root", item) + } + info, err := os.Stat(filepath.Join(root, filepath.FromSlash(dir))) + if err != nil || !info.IsDir() { + return nil, fmt.Errorf("package directory %q not found", item) + } + dirs = append(dirs, dir) + } + sort.Strings(dirs) + return dirs, nil +} + +// score holds G2 counts: killed over KILLED + LIVED + TIMED OUT + UNSTABLE. +type score struct { + killed, total int +} + +func (s score) meets() bool { return s.total > 0 && 5*s.killed >= 4*s.total } + +func (s score) String() string { + if s.total == 0 { + return fmt.Sprintf("%d/%d", s.killed, s.total) + } + return fmt.Sprintf("%d/%d = %.4f", s.killed, s.total, float64(s.killed)/float64(s.total)) +} + +func tally(results []result, include func(result) bool) score { + var s score + for _, r := range results { + if !include(r) { + continue + } + switch r.Status { + case statusKilled: + s.killed++ + s.total++ + case statusLived, statusTimedOut, statusUnstable: + s.total++ + } + } + return s +} + +// report prints one tab-separated row per mutant and the score. +func report(w io.Writer, base string, dirs []string, timeout time.Duration, workers int, results []result) (bool, error) { + out := bufio.NewWriter(w) + _, _ = fmt.Fprintf(out, "# cloudring-mutation base=%s packages=%s timeout=%s workers=%d mutants=%d\n", base, strings.Join(dirs, ","), timeout, workers, len(results)) + if len(results) > 0 { + sites := make([]site, len(results)) + for i, r := range results { + sites[i] = r.site + } + _, _ = fmt.Fprintf(out, "# baseline: go test passed without mutants in the private copy before the mutants and in all %d copies after them: %s\n", workers, strings.Join(sitePackages(sites), ",")) + } + _, _ = fmt.Fprintln(out, "STATUS\tLOCATION\tTYPE\tCHANGE\tFIRST FAILURE") + counts := map[string]int{} + for _, r := range results { + counts[r.Status]++ + _, _ = fmt.Fprintf(out, "%s\t%s\t%s\t%s -> %s\t%s\n", r.Status, r.location(), r.Type, r.From, r.To, r.Failure) + } + all := tally(results, func(result) bool { return true }) + defaults := tally(results, func(r result) bool { return r.Type != "INVERT_LOGICAL" }) + _, _ = fmt.Fprintf(out, "# statuses: KILLED=%d LIVED=%d TIMED OUT=%d UNSTABLE=%d NOT VIABLE=%d KILLED_BY_VET=%d\n"+ + "# default mutators only: %s\n"+ + "# G2 killedMutants=%d totalMutants=%d score=%s threshold=4/5 met=%t\n", + counts[statusKilled], counts[statusLived], counts[statusTimedOut], counts[statusUnstable], counts[statusNotViable], counts[statusKilledByVet], + defaults, all.killed, all.total, all, all.meets()) + return all.meets(), out.Flush() +} diff --git a/cmd/cloudring-mutation/mutation_test.go b/cmd/cloudring-mutation/mutation_test.go new file mode 100644 index 0000000..e518428 --- /dev/null +++ b/cmd/cloudring-mutation/mutation_test.go @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "context" + "errors" + "fmt" + "go/token" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +const sample = `package p + +func f(a, b int, c, d bool, s string) (int, bool, string) { + x := a - b + x++ + x += 2 + y := -x * a / b % 3 + return x, a < b && c || d, s + "!" +} +` + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func siteKeys(sites []site) []string { + var keys []string + for _, s := range sites { + keys = append(keys, fmt.Sprintf("%d:%d %s %s->%s", s.Line, s.Col, s.Type, s.From, s.To)) + } + return keys +} + +// Sites follow the gremlins v0.6.0 token table for the default mutators plus +// INVERT_LOGICAL: assignment operators, unary NOT and loop control are not +// mutated, a minus yields INVERT_NEGATIVES before ARITHMETIC_BASE, and only +// changed lines of files go build compiles count (not tests, testdata or +// files a build constraint excludes). +func TestEnumerateUsesTheGremlinsTokenTable(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "go.mod"), "module example.test/m\n\ngo 1.21\n") + writeFile(t, filepath.Join(root, "p", "p.go"), sample) + writeFile(t, filepath.Join(root, "p", "p_test.go"), "package p\n\nvar z = 1 + 2\n") + writeFile(t, filepath.Join(root, "p", "ignored.go"), "//go:build ignore\n\npackage p\n\nvar z = 1 + 2\n") + writeFile(t, filepath.Join(root, "p", "testdata", "x.go"), "package x\n\nvar z = 1 + 2\n") + all := map[string]bool{"p/p.go": true, "p/p_test.go": true, "p/ignored.go": true, "p/testdata/x.go": true} + got, err := enumerate(root, []string{"p"}, changeSet{newFiles: all}) + if err != nil { + t.Fatal(err) + } + want := []string{ + "4:9 INVERT_NEGATIVES -->+", "4:9 ARITHMETIC_BASE -->+", + "5:3 INCREMENT_DECREMENT ++->--", + "7:7 INVERT_NEGATIVES -->+", "7:7 ARITHMETIC_BASE -->+", + "7:10 ARITHMETIC_BASE *->/", "7:14 ARITHMETIC_BASE /->*", "7:18 ARITHMETIC_BASE %->*", + "8:14 CONDITIONALS_BOUNDARY <-><=", "8:14 CONDITIONALS_NEGATION <->>=", + "8:18 INVERT_LOGICAL &&->||", "8:23 INVERT_LOGICAL ||->&&", + "8:31 ARITHMETIC_BASE +->-", + } + if keys := siteKeys(got); !reflect.DeepEqual(keys, want) { + t.Fatalf("sites:\n%s\nwant:\n%s", strings.Join(keys, "\n"), strings.Join(want, "\n")) + } + for _, s := range got { + if s.File != "p/p.go" { + t.Fatalf("site outside the package file: %+v", s) + } + } + got, err = enumerate(root, []string{"p"}, changeSet{lines: map[string]map[int]bool{"p/p.go": {5: true}}}) + if err != nil { + t.Fatal(err) + } + if keys := siteKeys(got); !reflect.DeepEqual(keys, []string{"5:3 INCREMENT_DECREMENT ++->--"}) { + t.Fatalf("changed-line sites: %v", keys) + } +} + +// A mutant changes one AST node; printing keeps that node's meaning, so the +// outer operator of a chain becomes (c || d) && e, not c || d && e. +func TestApplyChangesOneNodeOfTheTree(t *testing.T) { + src := []byte("package p\n\nfunc g(c, d, e bool) bool {\n\treturn c || d || e\n}\n") + for col, want := range map[int]string{11: "return c && d || e", 16: "return (c || d) && e"} { + out, err := apply(src, site{File: "p.go", Line: 4, Col: col, Type: "INVERT_LOGICAL", From: token.LOR, To: token.LAND}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), want) { + t.Fatalf("column %d: %s", col, out) + } + } + if _, err := apply(src, site{File: "p.go", Line: 4, Col: 13, Type: "INVERT_LOGICAL", From: token.LOR, To: token.LAND}); err == nil { + t.Fatal("a position without the token was mutated") + } +} + +func TestParseUnifiedDiffKeepsOnlyNewSideLines(t *testing.T) { + diff := strings.Join([]string{ + "diff --git a/p/a.go b/p/a.go", + "index 1..2 100644", + "--- a/p/a.go", + "+++ b/p/a.go", + "@@ -3,0 +4,2 @@ func f() {", + "+++ x", + "+\ty++", + "@@ -10 +12 @@", + "-\told", + "+\tnew", + `\ No newline at end of file`, + "@@ -20,2 +21,0 @@", + "-\tgone", + "-\tgone", + "diff --git a/p/b.go b/p/b.go", + "deleted file mode 100644", + "--- a/p/b.go", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-package p", + "diff --git a/p/sp ace.go b/p/sp ace.go", + "new file mode 100644", + "--- /dev/null", + "+++ b/p/sp ace.go\t", + "@@ -0,0 +1 @@", + "+package p", + }, "\n") + got, err := parseUnifiedDiff([]byte(diff)) + if err != nil { + t.Fatal(err) + } + want := map[string]map[int]bool{"p/a.go": {4: true, 5: true, 12: true}, "p/sp ace.go": {1: true}} + if !reflect.DeepEqual(got.lines, want) || len(got.newFiles) != 0 { + t.Fatalf("changed lines %v, new files %v", got.lines, got.newFiles) + } + head := "diff --git a/p/x.go b/p/x.go\n" + for name, bad := range map[string]string{ + "unknown prefix": head + "--- a/p/x.go\n+++ w/p/x.go\n", + "quoted name": head + "--- a/p/x.go\n+++ \"b/p/q\\\"uote.go\"\n", + "binary notice": head + "index 1..2 100644\nBinary files a/p/x.go and b/p/x.go differ\n", + "rename header": head + "similarity index 100%\nrename from p/x.go\nrename to p/y.go\n", + "hunk before name": head + "@@ -1 +1 @@\n-a\n+b\n", + "short hunk": head + "--- a/p/x.go\n+++ b/p/x.go\n@@ -1 +1,2 @@\n-a\n+b\n", + "text after hunk": head + "--- a/p/x.go\n+++ b/p/x.go\n@@ -1 +1 @@\n-a\n+b\n c\n", + "text before a file": "warning: something\n", + "tab without a space": head + "--- a/p/x.go\n+++ b/p/x.go\t\n", + } { + if _, err := parseUnifiedDiff([]byte(bad)); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func TestClassify(t *testing.T) { + failed := errors.New("exit status 1") + builds := func(ok bool) func() bool { return func() bool { return ok } } + for _, tc := range []struct { + name string + run testRun + builds bool + want string + }{ + {"tests pass", testRun{}, true, statusLived}, + {"a test fails", testRun{err: failed, output: "--- FAIL: TestX (0.00s)\nFAIL\n"}, true, statusKilled}, + {"a test panics", testRun{err: failed, output: "panic: runtime error\nFAIL\tp\t0.01s\n"}, true, statusKilled}, + {"go test timeout", testRun{err: failed, output: "panic: test timed out after 2m0s\n"}, true, statusTimedOut}, + {"run outlived the allowance", testRun{err: failed, expired: true}, true, statusTimedOut}, + {"does not compile", testRun{err: failed, output: "FAIL\tp [build failed]\n"}, false, statusNotViable}, + {"go vet rejects", testRun{err: failed, output: "FAIL\tp [build failed]\n"}, true, statusKilledByVet}, + {"setup fails", testRun{err: failed, output: "FAIL\tp [setup failed]\n"}, false, statusNotViable}, + } { + if got := classify(tc.run, builds(tc.builds)); got != tc.want { + t.Errorf("%s: %s, want %s", tc.name, got, tc.want) + } + } +} + +// Real mutants of a small module, one per outcome, run through go test. +func TestExecuteClassifiesRealMutants(t *testing.T) { + if testing.Short() { + t.Skip("runs go test on generated mutants") + } + root := t.TempDir() + writeFile(t, filepath.Join(root, "go.mod"), "module example.test/m\n\ngo 1.21\n") + writeFile(t, filepath.Join(root, "p", "p.go"), `package p + +func Max(a, b int) int { + if a > b { + return a + } + return b +} + +func Greeting(s string) string { return "hello " + s } + +func Known(s string) bool { return s != "a" && s != "b" } + +func Sum(n int) int { + total := 0 + for i := 0; i < n; i++ { + total += i + } + return total +} +`) + writeFile(t, filepath.Join(root, "p", "p_test.go"), `package p + +import "testing" + +func TestP(t *testing.T) { + if Max(1, 2) != 2 || Max(2, 1) != 2 || Greeting("x") != "hello x" || !Known("c") || Sum(3) != 3 { + t.Fatal("unexpected result") + } +} +`) + sites, err := enumerate(root, []string{"p"}, changeSet{newFiles: map[string]bool{"p/p.go": true}}) + if err != nil { + t.Fatal(err) + } + pick := map[string]site{} + for _, s := range sites { + switch { + case s.Line == 4 && s.Type == "CONDITIONALS_BOUNDARY": + pick[statusLived] = s // a >= b: Max(2, 2) is not tested + case s.Line == 4 && s.Type == "CONDITIONALS_NEGATION": + pick[statusKilled] = s + case s.Line == 10 && s.Type == "ARITHMETIC_BASE": + pick[statusNotViable] = s // string - string does not compile + case s.Line == 12 && s.Type == "INVERT_LOGICAL": + pick[statusKilledByVet] = s // s != "a" || s != "b": suspect or + case s.Line == 16 && s.Type == "INCREMENT_DECREMENT": + pick[statusTimedOut] = s // i-- never reaches n + } + } + var chosen []site + var want []string + for _, status := range []string{statusLived, statusKilled, statusNotViable, statusKilledByVet, statusTimedOut} { + s, ok := pick[status] + if !ok { + t.Fatalf("no site for %s in %v", status, siteKeys(sites)) + } + chosen, want = append(chosen, s), append(want, status) + } + results, err := execute(context.Background(), root, chosen, 3*time.Second, len(chosen)) + if err != nil { + t.Fatal(err) + } + for i, r := range results { + if r.Status != want[i] { + t.Errorf("%s %s: %s, want %s", r.location(), r.Type, r.Status, want[i]) + } + } + if s := tally(results, func(result) bool { return true }); s != (score{killed: 1, total: 3}) || s.meets() { + t.Fatalf("score %+v", s) + } + if got, err := os.ReadFile(filepath.Join(root, "p", "p.go")); err != nil || !strings.Contains(string(got), "if a > b {") { + t.Fatalf("the measured tree was modified: %v", err) + } +} + +func TestRunRejectsInvalidArguments(t *testing.T) { + root, bare := t.TempDir(), t.TempDir() + writeFile(t, filepath.Join(root, "go.mod"), "module example.test/m\n") + writeFile(t, filepath.Join(root, "p", "p.go"), "package p\n") + writeFile(t, filepath.Join(bare, "p", "p.go"), "package p\n") + for _, args := range [][]string{ + {}, + {"--packages", "p"}, + {"--base", "HEAD"}, + {"--base", "--upload-pack=x", "--packages", "p"}, + {"--root", bare, "--base", "HEAD", "--packages", "p"}, + {"--root", root, "--base", "HEAD", "--packages", "/p"}, + {"--root", root, "--base", "HEAD", "--packages", "../p"}, + {"--root", root, "--base", "HEAD", "--packages", "missing"}, + {"--root", root, "--base", "HEAD", "--packages", "p", "--timeout", "0s"}, + {"--root", root, "--base", "HEAD", "--packages", "p", "--workers", "0"}, + {"--root", root, "--base", "HEAD", "--packages", "p", "extra"}, + } { + var out, errOut strings.Builder + if code := run(context.Background(), args, &out, &errOut); code != 2 || out.Len() != 0 || errOut.Len() == 0 { + t.Errorf("%v: exit=%d out=%q err=%q", args, code, out.String(), errOut.String()) + } + } +} + +func TestScoreThresholdIsInclusive(t *testing.T) { + for _, tc := range []struct { + s score + want bool + }{{score{4, 5}, true}, {score{3, 4}, false}, {score{995, 1021}, true}, {score{0, 0}, false}} { + if got := tc.s.meets(); got != tc.want { + t.Errorf("%+v: %t", tc.s, got) + } + } +} + +// Each worker mutates its own copy: version control metadata stays behind, +// links are recreated rather than followed, and file modes are kept. +func TestCopyModuleKeepsTheTreeWithoutVersionControl(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "go.mod"), "module example.test/m\n") + writeFile(t, filepath.Join(root, "p", "run.sh"), "#!/bin/sh\n") + if err := os.Chmod(filepath.Join(root, "p", "run.sh"), 0o750); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(root, ".git", "HEAD"), "ref: refs/heads/main\n") + writeFile(t, filepath.Join(root, "sub", ".git"), "gitdir: elsewhere\n") + if err := os.Symlink("../go.mod", filepath.Join(root, "p", "link")); err != nil { + t.Fatal(err) + } + dst := filepath.Join(t.TempDir(), "copy") + if err := copyModule(root, dst); err != nil { + t.Fatal(err) + } + if info, err := os.Stat(filepath.Join(dst, "p", "run.sh")); err != nil || info.Mode().Perm() != 0o750 { + t.Fatalf("regular file: %v %v", info, err) + } + if link, err := os.Readlink(filepath.Join(dst, "p", "link")); err != nil || link != "../go.mod" { + t.Fatalf("link: %q %v", link, err) + } + for _, name := range []string{".git", filepath.Join("sub", ".git")} { + if _, err := os.Lstat(filepath.Join(dst, name)); !os.IsNotExist(err) { + t.Fatalf("%s copied: %v", name, err) + } + } + if err := copyModule(root, dst); err == nil { + t.Fatal("copied into an existing directory") + } +} + +// gitIn runs git in repo with a fixed identity and no signing. +func gitIn(t *testing.T, repo string, args ...string) { + t.Helper() + if _, err := git(repo, append([]string{"-c", "user.name=t", "-c", "user.email=t@example.test", "-c", "commit.gpgsign=false"}, args...)...); err != nil { + t.Fatal(err) + } +} + +func requireGit(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } +} + +// Changed lines come from the working tree against the base revision, with +// paths relative to a module root below the repository top level, and +// untracked files count as new. +func TestCandidateChangesReadsTheWorkingTreeAgainstBase(t *testing.T) { + requireGit(t) + repo := t.TempDir() + mod := filepath.Join(repo, "mod") + writeFile(t, filepath.Join(mod, "go.mod"), "module example.test/m\n") + writeFile(t, filepath.Join(mod, "p", "p.go"), "package p\n\nvar a = 1\n\nvar b = 2\n") + writeFile(t, filepath.Join(mod, "q", "q.go"), "package q\n") + gitIn(t, repo, "init", "-q") + gitIn(t, repo, "add", ".") + gitIn(t, repo, "commit", "-q", "-m", "base") + writeFile(t, filepath.Join(mod, "p", "p.go"), "package p\n\nvar a = 1\n\nvar b = 3\nvar c = 4\n") + writeFile(t, filepath.Join(mod, "p", "new.go"), "package p\n") + writeFile(t, filepath.Join(mod, "q", "q.go"), "package q\n\nvar d = 5\n") + got, err := candidateChanges(mod, "HEAD", []string{"p"}) + if err != nil { + t.Fatal(err) + } + if want := map[string]map[int]bool{"p/p.go": {5: true, 6: true}}; !reflect.DeepEqual(got.lines, want) { + t.Fatalf("changed lines %v", got.lines) + } + if want := map[string]bool{"p/new.go": true}; !reflect.DeepEqual(got.newFiles, want) { + t.Fatalf("new files %v", got.newFiles) + } +} + +const twice = "package p\n\nfunc Twice(a int) int { return a * 2 }\n" + +// repoWithBase creates a module repository whose first commit holds files. +func repoWithBase(t *testing.T, files map[string]string) string { + t.Helper() + repo := t.TempDir() + writeFile(t, filepath.Join(repo, "go.mod"), "module example.test/m\n\ngo 1.21\n") + for name, content := range files { + writeFile(t, filepath.Join(repo, filepath.FromSlash(name)), content) + } + gitIn(t, repo, "init", "-q") + gitIn(t, repo, "add", ".") + gitIn(t, repo, "commit", "-q", "-m", "base") + return repo +} + +func runTool(t *testing.T, repo, base string) (int, string, string) { + t.Helper() + var out, errOut strings.Builder + code := run(context.Background(), []string{"--root", repo, "--base", base, "--packages", "p", "--timeout", "30s", "--workers", "1"}, &out, &errOut) + return code, out.String(), errOut.String() +} + +// A file named with a space, a file marked -diff and a renamed file are +// candidate code like any other: their mutants are measured. +func TestRunMeasuresEveryChangedFile(t *testing.T) { + if testing.Short() { + t.Skip("runs go test on generated mutants") + } + requireGit(t) + test := "package p\n\nimport \"testing\"\n\nfunc TestTwice(t *testing.T) {\n\tif Twice(3) != 6 {\n\t\tt.Fatal(Twice(3))\n\t}\n}\n" + for _, tc := range []struct { + name, file string + change func(t *testing.T) (repo, base string) + }{ + {"space in the name", "p/sp ace.go", func(t *testing.T) (string, string) { + repo := repoWithBase(t, map[string]string{"p/p_test.go": test}) + writeFile(t, filepath.Join(repo, "p", "sp ace.go"), twice) + gitIn(t, repo, "add", ".") + gitIn(t, repo, "commit", "-q", "-m", "candidate") + return repo, "HEAD~1" + }}, + {"-diff attribute", "p/p.go", func(t *testing.T) (string, string) { + repo := repoWithBase(t, map[string]string{".gitattributes": "*.go -diff\n", "p/p.go": "package p\n", "p/p_test.go": test}) + writeFile(t, filepath.Join(repo, "p", "p.go"), twice) + return repo, "HEAD" + }}, + {"renamed file", "p/new.go", func(t *testing.T) (string, string) { + repo := repoWithBase(t, map[string]string{"p/old.go": twice, "p/p_test.go": test}) + gitIn(t, repo, "mv", "p/old.go", "p/new.go") + return repo, "HEAD" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + repo, base := tc.change(t) + code, out, errOut := runTool(t, repo, base) + row := "KILLED\t" + tc.file + ":3:34\tARITHMETIC_BASE\t* -> /\t" + if code != 0 || !strings.Contains(out, "mutants=1\n") || !strings.Contains(out, row) { + t.Fatalf("exit=%d\n%s%s", code, out, errOut) + } + }) + } +} + +// A failure that does not depend on the mutant would count as a kill, so the +// tool stops when a package fails without a mutant: in the tree, only in the +// private copy (which has no .git), or only after its first run. +func TestRunStopsWhenAPackageFailsWithoutAMutant(t *testing.T) { + if testing.Short() { + t.Skip("runs go test on generated mutants") + } + requireGit(t) + for name, test := range map[string]string{ + "fails in the tree": "if Twice(3) != 7 {\n\t\tt.Fatal(\"red\")\n\t}", + "fails only in the copy": "if _, err := os.Stat(\"../.git\"); err != nil || Twice(3) != 6 {\n\t\tt.Fatal(err)\n\t}", + "fails after its first run": "if _, err := os.Stat(\"ran\"); err == nil || Twice(3) != 6 {\n\t\tt.Fatal(\"second run\")\n\t}\n" + + "\tif err := os.WriteFile(\"ran\", nil, 0o600); err != nil {\n\t\tt.Fatal(err)\n\t}", + } { + t.Run(name, func(t *testing.T) { + repo := repoWithBase(t, nil) + writeFile(t, filepath.Join(repo, "p", "p.go"), twice) + writeFile(t, filepath.Join(repo, "p", "p_test.go"), "package p\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nvar _ = os.Stat\n\nfunc TestTwice(t *testing.T) {\n\t"+test+"\n}\n") + if name != "fails in the tree" { + if run := goTest(context.Background(), repo, "p", 30*time.Second); run.err != nil { + t.Fatalf("the test must pass in the tree: %s", run.output) + } + _ = os.Remove(filepath.Join(repo, "p", "ran")) + } + code, out, errOut := runTool(t, repo, "HEAD") + if code != 2 || out != "" || !strings.Contains(errOut, "fails without a mutant") { + t.Fatalf("exit=%d\n%s%s", code, out, errOut) + } + }) + } +} + +// runCounter is test code that counts its runs in a file of the package +// directory, the state a test must not keep but a measured package might. +const runCounter = `func runs(t *testing.T) int { + b, _ := os.ReadFile("runs") + n, _ := strconv.Atoi(string(b)) + n++ + if err := os.WriteFile("runs", []byte(strconv.Itoa(n)), 0o600); err != nil { + t.Fatal(err) + } + return n +} +` + +// A failure that does not repeat is not a kill. The mutant H(a)=a-1 is never +// checked; only a test that fails on alternating runs, or on runs a fixed +// pseudo-random sequence picks (seed 170: pass, fail, pass, pass, pass, pass), +// fails while it is applied. Both mutants must be UNSTABLE and not met. +func TestRunConfirmsKills(t *testing.T) { + if testing.Short() { + t.Skip("runs go test on generated mutants") + } + requireGit(t) + for name, check := range map[string]string{ + "alternating": "if runs(t)%2 == 0 {\n\t\tt.Fatal(\"even run\")\n\t}", + "seeded random flake": "if rand.New(rand.NewSource(170*1000 + int64(runs(t)))).Intn(2) == 0 {\n\t\tt.Fatal(\"flake\")\n\t}", + } { + t.Run(name, func(t *testing.T) { + repo := repoWithBase(t, nil) + writeFile(t, filepath.Join(repo, "p", "p.go"), "package p\n\nfunc H(a int) int { return a + 1 }\n") + writeFile(t, filepath.Join(repo, "p", "p_test.go"), "package p\n\nimport (\n\t\"math/rand\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nvar _ = rand.New\n\n"+ + runCounter+"\nfunc TestH(t *testing.T) { _ = H(1) }\n\nfunc TestEnvironment(t *testing.T) {\n\t"+check+"\n}\n") + code, out, errOut := runTool(t, repo, "HEAD") + if code != 1 || !strings.Contains(out, "UNSTABLE\tp/p.go:3:30\tARITHMETIC_BASE\t+ -> -\t--- FAIL: TestEnvironment") || + !strings.Contains(out, "UNSTABLE=1 ") || !strings.Contains(out, "killedMutants=0 totalMutants=1 ") { + t.Fatalf("exit=%d\n%s%s", code, out, errOut) + } + }) + } +} + +// Code go build compiles is measured even when a //line directive renumbers +// it, and a compiled file the tool cannot mutate in place (a symbolic link) +// stops the run. +func TestRunMeasuresTheCodeGoBuildCompiles(t *testing.T) { + if testing.Short() { + t.Skip("runs go test on generated mutants") + } + requireGit(t) + base := map[string]string{"p/base.go": "package p\n", "p/p_test.go": "package p\n\nimport \"testing\"\n\nfunc TestP(t *testing.T) {}\n"} + untested := "package p\n\nfunc Untested(a int) int { return a * 2 }\n" + t.Run("line directive", func(t *testing.T) { + repo := repoWithBase(t, base) + writeFile(t, filepath.Join(repo, "p", "lined.go"), "package p\n\n//line lined.go:1000\nfunc Untested(a int) int { return a * 2 }\n") + gitIn(t, repo, "add", ".") + gitIn(t, repo, "commit", "-q", "-m", "candidate") + code, out, errOut := runTool(t, repo, "HEAD~1") + if code != 1 || !strings.Contains(out, "LIVED\tp/lined.go:4:37\tARITHMETIC_BASE\t* -> /\t") { + t.Fatalf("exit=%d\n%s%s", code, out, errOut) + } + }) + t.Run("symbolic link", func(t *testing.T) { + repo := repoWithBase(t, base) + writeFile(t, filepath.Join(repo, "p", "_impl", "h.go"), untested) + if err := os.Symlink(filepath.Join("_impl", "h.go"), filepath.Join(repo, "p", "hidden.go")); err != nil { + t.Fatal(err) + } + gitIn(t, repo, "add", ".") + gitIn(t, repo, "commit", "-q", "-m", "candidate") + code, out, errOut := runTool(t, repo, "HEAD~1") + if code != 2 || out != "" || !strings.Contains(errOut, "hidden.go") || !strings.Contains(errOut, "not a regular file") { + t.Fatalf("exit=%d\n%s%s", code, out, errOut) + } + }) +} diff --git a/cmd/cloudring-mutation/procattr_other.go b/cmd/cloudring-mutation/procattr_other.go new file mode 100644 index 0000000..843cf6f --- /dev/null +++ b/cmd/cloudring-mutation/procattr_other.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +//go:build !unix + +package main + +import "os/exec" + +// ownProcessGroup keeps the default process handling where process groups +// are not available. +func ownProcessGroup(*exec.Cmd) {} diff --git a/cmd/cloudring-mutation/procattr_unix.go b/cmd/cloudring-mutation/procattr_unix.go new file mode 100644 index 0000000..293ae90 --- /dev/null +++ b/cmd/cloudring-mutation/procattr_unix.go @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +//go:build unix + +package main + +import ( + "os/exec" + "syscall" +) + +// ownProcessGroup starts cmd in a new process group and makes cancellation +// stop the whole group, including the test binary go test started. +func ownProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } +} diff --git a/cmd/cloudring-mutation/sites.go b/cmd/cloudring-mutation/sites.go new file mode 100644 index 0000000..194ff94 --- /dev/null +++ b/cmd/cloudring-mutation/sites.go @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" +) + +// The mutant table of gremlins v0.6.0 (internal/engine/mappings.go), reduced to +// the types G2 enables: the default mutators plus INVERT_LOGICAL. A token lists +// its types in gremlins' order. +var mutantTypes = map[token.Token][]string{ + token.ADD: {"ARITHMETIC_BASE"}, + token.DEC: {"INCREMENT_DECREMENT"}, + token.EQL: {"CONDITIONALS_NEGATION"}, + token.GEQ: {"CONDITIONALS_BOUNDARY", "CONDITIONALS_NEGATION"}, + token.GTR: {"CONDITIONALS_BOUNDARY", "CONDITIONALS_NEGATION"}, + token.INC: {"INCREMENT_DECREMENT"}, + token.LAND: {"INVERT_LOGICAL"}, + token.LEQ: {"CONDITIONALS_BOUNDARY", "CONDITIONALS_NEGATION"}, + token.LOR: {"INVERT_LOGICAL"}, + token.LSS: {"CONDITIONALS_BOUNDARY", "CONDITIONALS_NEGATION"}, + token.MUL: {"ARITHMETIC_BASE"}, + token.NEQ: {"CONDITIONALS_NEGATION"}, + token.QUO: {"ARITHMETIC_BASE"}, + token.REM: {"ARITHMETIC_BASE"}, + token.SUB: {"INVERT_NEGATIVES", "ARITHMETIC_BASE"}, +} + +var replacements = map[string]map[token.Token]token.Token{ + "ARITHMETIC_BASE": {token.ADD: token.SUB, token.MUL: token.QUO, token.QUO: token.MUL, token.REM: token.MUL, token.SUB: token.ADD}, + "CONDITIONALS_BOUNDARY": {token.GEQ: token.GTR, token.GTR: token.GEQ, token.LEQ: token.LSS, token.LSS: token.LEQ}, + "CONDITIONALS_NEGATION": {token.EQL: token.NEQ, token.GEQ: token.LSS, token.GTR: token.LEQ, token.LEQ: token.GTR, token.LSS: token.GEQ, token.NEQ: token.EQL}, + "INCREMENT_DECREMENT": {token.DEC: token.INC, token.INC: token.DEC}, + "INVERT_LOGICAL": {token.LAND: token.LOR, token.LOR: token.LAND}, + "INVERT_NEGATIVES": {token.SUB: token.ADD}, +} + +// site is one mutant: a token at a source position and its replacement. +type site struct { + File string // slash path relative to the module root + Line, Col int + Type string + From, To token.Token +} + +func (s site) location() string { return fmt.Sprintf("%s:%d:%d", s.File, s.Line, s.Col) } + +// tokenOf returns the token field gremlins mutates in n, if any. +func tokenOf(n ast.Node) (*token.Token, token.Pos, bool) { + switch n := n.(type) { + case *ast.AssignStmt: + return &n.Tok, n.TokPos, true + case *ast.BinaryExpr: + return &n.Op, n.OpPos, true + case *ast.BranchStmt: + return &n.Tok, n.TokPos, true + case *ast.IncDecStmt: + return &n.Tok, n.TokPos, true + case *ast.UnaryExpr: + return &n.Op, n.OpPos, true + } + return nil, token.NoPos, false +} + +// changeSet holds the candidate's lines: whole new files and changed lines of +// existing files, keyed by slash path relative to the module root. +type changeSet struct { + newFiles map[string]bool + lines map[string]map[int]bool +} + +func (c changeSet) contains(file string, line int) bool { + return c.newFiles[file] || c.lines[file][line] +} + +// enumerate lists the mutants on changed lines of the Go files that go build +// compiles for the packages under dirs. +func enumerate(root string, dirs []string, changes changeSet) ([]site, error) { + files, err := compiledFiles(root, dirs) + if err != nil { + return nil, err + } + var sites []site + for _, file := range files { + found, err := fileSites(filepath.Join(root, filepath.FromSlash(file)), file, changes) + if err != nil { + return nil, err + } + sites = append(sites, found...) + } + sort.SliceStable(sites, func(i, j int) bool { + a, b := sites[i], sites[j] + if a.File != b.File { + return a.File < b.File + } + if a.Line != b.Line { + return a.Line < b.Line + } + return a.Col < b.Col + }) + return sites, nil +} + +// compiledFiles returns the GoFiles and CgoFiles that `go list` reports for +// the packages under dirs, as slash paths relative to root. A package outside +// the module root and a listed file that is not a regular file in its package +// directory (a symbolic link, for one) are errors: the tool would not mutate +// the code go build compiles. +func compiledFiles(root string, dirs []string) ([]string, error) { + abs, err := filepath.Abs(root) + if err != nil { + return nil, err + } + realRoot, err := filepath.EvalSymlinks(abs) + if err != nil { + return nil, err + } + args := []string{"list", "-json=Dir,GoFiles,CgoFiles"} + for _, dir := range dirs { + args = append(args, "./"+dir+"/...") + } + cmd := exec.Command("go", args...) // #nosec G204 -- fixed go binary; the package directories are validated relative to the module root. + cmd.Dir = abs + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("go list: %v: %s", err, strings.TrimSpace(stderr.String())) + } + var files []string + decoder := json.NewDecoder(bytes.NewReader(out)) + for decoder.More() { + var pkg struct { + Dir string + GoFiles, CgoFiles []string + } + if err := decoder.Decode(&pkg); err != nil { + return nil, err + } + realDir, err := filepath.EvalSymlinks(pkg.Dir) + if err != nil { + return nil, err + } + rel, err := filepath.Rel(realRoot, realDir) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("package directory %s is outside the module root", pkg.Dir) + } + for _, name := range append(pkg.GoFiles, pkg.CgoFiles...) { + info, err := os.Lstat(filepath.Join(realDir, name)) + if err != nil { + return nil, err + } + if name != filepath.Base(name) || !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s in %s is not a regular file of the package directory", name, pkg.Dir) + } + files = append(files, path.Join(filepath.ToSlash(rel), name)) + } + } + sort.Strings(files) + return files, nil +} + +// fileSites lists the mutants of one file. Positions are physical file +// positions: //line directives are ignored, so they match the diff. +func fileSites(name, rel string, changes changeSet) ([]site, error) { + src, err := os.ReadFile(name) // #nosec G304 -- a Go file of a package the operator named. + if err != nil { + return nil, err + } + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, rel, src, parser.ParseComments) + if err != nil { + return nil, err + } + var sites []site + ast.Inspect(file, func(n ast.Node) bool { + tok, pos, ok := tokenOf(n) + if !ok { + return true + } + p := fset.PositionFor(pos, false) + if !changes.contains(rel, p.Line) { + return true + } + for _, kind := range mutantTypes[*tok] { + sites = append(sites, site{File: rel, Line: p.Line, Col: p.Column, Type: kind, From: *tok, To: replacements[kind][*tok]}) + } + return true + }) + return sites, nil +} + +// apply returns src with the mutant applied the way gremlins applies it: the +// token of that one AST node is replaced and the file is printed with +// go/printer, which parenthesizes the changed tree where its meaning needs it. +func apply(src []byte, s site) ([]byte, error) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, s.File, src, parser.ParseComments) + if err != nil { + return nil, err + } + applied := false + ast.Inspect(file, func(n ast.Node) bool { + tok, pos, ok := tokenOf(n) + if !ok || applied { + return !applied + } + if p := fset.PositionFor(pos, false); p.Line == s.Line && p.Column == s.Col && *tok == s.From { + *tok = s.To + applied = true + } + return !applied + }) + if !applied { + return nil, fmt.Errorf("no %s token at %s", s.From, s.location()) + } + var out bytes.Buffer + if err := printer.Fprint(&out, fset, file); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/cmd/cloudring-safepush/arguments_test.go b/cmd/cloudring-safepush/arguments_test.go new file mode 100644 index 0000000..50440f3 --- /dev/null +++ b/cmd/cloudring-safepush/arguments_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package main + +import ( + "bytes" + "context" + "net/http" + "strings" + "testing" + "time" + + "github.com/opencloudtech/CloudRING/pkg/safepush" +) + +// validInvocation is a complete, well-formed request; extra flags override it. +func validInvocation(extra ...string) []string { + return append([]string{"github", "--repository", "synthetic/project", "--pr", "1", + "--head", strings.Repeat("a", 40), "--base", strings.Repeat("b", 40), "--branch", "main"}, extra...) +} + +func TestEachInvalidArgumentAloneMakesNoProviderCall(t *testing.T) { + for _, extra := range [][]string{ + {"extra"}, {"--unknown"}, {"--repository="}, {"--pr=0"}, {"--pr=-1"}, {"--head="}, {"--base="}, + {"--branch=develop"}, {"--branch="}, {"--timeout=0s"}, {"--timeout=-1s"}, {"--timeout=30m1ns"}, {"--timeout=31m"}, + } { + var out, errOut bytes.Buffer + read := func(context.Context, *http.Client, string, string, string) (safepush.GitHubPolicy, error) { + t.Errorf("%v: provider called for an invalid request", extra) + return safepush.GitHubPolicy{}, nil + } + code := run(context.Background(), validInvocation(extra...), func(string) string { return "synthetic-test-value" }, &out, &errOut, read, nil) + if code != 2 || out.Len() != 0 || errOut.String() != "SafePush request is invalid\n" { + t.Fatalf("%v: exit=%d out=%q err=%q", extra, code, &out, &errOut) + } + } +} + +func TestCommandAdmitsBothProtectedBranchesAndTheTimeoutBound(t *testing.T) { + for _, tc := range []struct { + extra []string + branch string + }{ + {nil, "main"}, + {[]string{"--branch=master"}, "master"}, + {[]string{"--timeout=30m"}, "main"}, + } { + var out, errOut bytes.Buffer + var clientTimeout time.Duration + var verified safepush.GitHubCandidate + read := func(_ context.Context, client *http.Client, _, repository, base string) (safepush.GitHubPolicy, error) { + clientTimeout = client.Timeout + if repository != "synthetic/project" || base != strings.Repeat("b", 40) { + t.Errorf("policy read for %s at %s", repository, base) + } + return safepush.GitHubPolicy{Repository: repository}, nil + } + verify := func(_ context.Context, _ *http.Client, _ string, _ safepush.GitHubPolicy, candidate safepush.GitHubCandidate) error { + verified = candidate + return nil + } + code := run(context.Background(), validInvocation(tc.extra...), func(string) string { return "synthetic-test-value" }, &out, &errOut, read, verify) + if code != 0 || errOut.Len() != 0 || !strings.HasPrefix(out.String(), "SafePush: the exact required CI checks passed") { + t.Fatalf("%v: exit=%d out=%q err=%q", tc.extra, code, &out, &errOut) + } + want := safepush.GitHubCandidate{PullRequest: 1, HeadSHA: strings.Repeat("a", 40), BaseSHA: strings.Repeat("b", 40), TargetBranch: tc.branch} + if verified != want || clientTimeout != 30*time.Second { + t.Fatalf("%v: verified %+v with client timeout %s", tc.extra, verified, clientTimeout) + } + } +} + +// A pending result is retried no sooner than the 30 s poll interval: within a +// one-second deadline the provider is asked exactly once. +func TestPendingAdmissionIsNotPolledBeforeTheInterval(t *testing.T) { + var out, errOut bytes.Buffer + calls := 0 + read := func(_ context.Context, _ *http.Client, _, repository, _ string) (safepush.GitHubPolicy, error) { + return safepush.GitHubPolicy{Repository: repository}, nil + } + verify := func(context.Context, *http.Client, string, safepush.GitHubPolicy, safepush.GitHubCandidate) error { + calls++ + return safepush.ErrGitHubPending + } + code := run(context.Background(), validInvocation("--timeout=1s"), func(string) string { return "synthetic-test-value" }, &out, &errOut, read, verify) + if code != 1 || calls != 1 || errOut.String() != "SafePush blocked: verification deadline exceeded\n" { + t.Fatalf("exit=%d calls=%d err=%q", code, calls, &errOut) + } +} diff --git a/cmd/cloudring-safepush/main.go b/cmd/cloudring-safepush/main.go index fb4aa4a..e5167d9 100644 --- a/cmd/cloudring-safepush/main.go +++ b/cmd/cloudring-safepush/main.go @@ -30,7 +30,7 @@ func main() { // caller-supplied check-results file, success flag, API host, or policy override. func run(ctx context.Context, args []string, getenv func(string) string, stdout, stderr io.Writer, read githubPolicyReader, verify githubVerifier) int { if len(args) == 0 || args[0] != "github" { - fmt.Fprintln(stderr, "usage: cloudring-safepush github --repository OWNER/REPO --pr NUMBER --head SHA --base SHA --branch main|master") + _, _ = fmt.Fprintln(stderr, "usage: cloudring-safepush github --repository OWNER/REPO --pr NUMBER --head SHA --base SHA --branch main|master") return 2 } flags := flag.NewFlagSet("github", flag.ContinueOnError) @@ -44,17 +44,17 @@ func run(ctx context.Context, args []string, getenv func(string) string, stdout, if flags.Parse(args[1:]) != nil || flags.NArg() != 0 || *repository == "" || *pr <= 0 || *head == "" || *base == "" || (*branch != "main" && *branch != "master") || *timeout <= 0 || *timeout > 30*time.Minute { - fmt.Fprintln(stderr, "SafePush request is invalid") + _, _ = fmt.Fprintln(stderr, "SafePush request is invalid") return 2 } candidate := safepush.GitHubCandidate{PullRequest: *pr, HeadSHA: *head, BaseSHA: *base, TargetBranch: *branch} if safepush.ValidateGitHubCandidate(*repository, candidate) != nil { - fmt.Fprintln(stderr, "SafePush request is invalid") + _, _ = fmt.Fprintln(stderr, "SafePush request is invalid") return 2 } bearer := getenv("GITHUB_TOKEN") if bearer == "" { - fmt.Fprintln(stderr, "SafePush requires the workflow's read-only GITHUB_TOKEN") + _, _ = fmt.Fprintln(stderr, "SafePush requires the workflow's read-only GITHUB_TOKEN") return 2 } ctx, cancel := context.WithTimeout(ctx, *timeout) @@ -62,16 +62,16 @@ func run(ctx context.Context, args []string, getenv func(string) string, stdout, client := &http.Client{Timeout: 30 * time.Second} policy, err := read(ctx, client, bearer, *repository, *base) if err != nil { - fmt.Fprintln(stderr, "SafePush blocked:", safeDiagnostic(err)) + _, _ = fmt.Fprintln(stderr, "SafePush blocked:", safeDiagnostic(err)) return 1 } if policy.Repository != *repository { - fmt.Fprintln(stderr, "SafePush blocked: accepted policy belongs to another repository") + _, _ = fmt.Fprintln(stderr, "SafePush blocked: accepted policy belongs to another repository") return 1 } err = waitForChecks(ctx, 30*time.Second, func() error { return verify(ctx, client, bearer, policy, candidate) }) if err != nil { - fmt.Fprintln(stderr, "SafePush blocked:", safeDiagnostic(err)) + _, _ = fmt.Fprintln(stderr, "SafePush blocked:", safeDiagnostic(err)) return 1 } if _, err := fmt.Fprintln(stdout, "SafePush: the exact required CI checks passed for the current candidate and base"); err != nil { diff --git a/cmd/cloudringctl/main.go b/cmd/cloudringctl/main.go index 2af3816..563bb37 100644 --- a/cmd/cloudringctl/main.go +++ b/cmd/cloudringctl/main.go @@ -38,6 +38,9 @@ type state struct { func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } func run(args []string, out, errOut io.Writer) int { + if len(args) > 0 && args[0] == "rsi" { + return runRSICommand(args[1:], out, errOut) + } if len(args) < 2 || args[0] != "verify" || args[1] != "goal" { return blocked(errOut, "invalid arguments") } diff --git a/cmd/cloudringctl/rsi.go b/cmd/cloudringctl/rsi.go new file mode 100644 index 0000000..ee494e5 --- /dev/null +++ b/cmd/cloudringctl/rsi.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/opencloudtech/CloudRING/pkg/rsi" +) + +// maxRSIProfileBytes matches the evaluator's own profile bound; one extra byte +// lets ParseProfile reject an oversized file instead of a silently cut one. +const maxRSIProfileBytes = 1 << 20 + +// runRSICommand runs `rsi evaluate --profile `: it prints the RSI +// governance report as JSON and exits 0 only for a pass verdict, 1 for a +// fail verdict and 2 for invalid arguments or an unreadable profile. +func runRSICommand(args []string, out, errOut io.Writer) int { + if len(args) == 0 || args[0] != "evaluate" { + _, _ = fmt.Fprintln(errOut, "cloudringctl rsi: expected evaluate --profile ") + return 2 + } + flags := flag.NewFlagSet("cloudringctl rsi evaluate", flag.ContinueOnError) + flags.SetOutput(io.Discard) + profilePath := flags.String("profile", "", "RSI candidate profile JSON file") + if err := flags.Parse(args[1:]); err != nil || flags.NArg() != 0 || strings.TrimSpace(*profilePath) == "" { + _, _ = fmt.Fprintln(errOut, "cloudringctl rsi: expected evaluate --profile without trailing arguments") + return 2 + } + file, err := os.Open(*profilePath) // #nosec G304 -- explicit local profile input, read-only and bounded below. + if err != nil { + _, _ = fmt.Fprintln(errOut, "cloudringctl rsi: cannot open profile") + return 2 + } + raw, err := io.ReadAll(io.LimitReader(file, maxRSIProfileBytes+1)) + closeErr := file.Close() + if err != nil || closeErr != nil { + _, _ = fmt.Fprintln(errOut, "cloudringctl rsi: cannot read profile") + return 2 + } + profile, err := rsi.ParseProfile(raw) + if err != nil { + _, _ = fmt.Fprintf(errOut, "cloudringctl rsi: %v\n", err) + return 2 + } + report, err := rsi.EvaluateCandidate(profile) + if err != nil { + _, _ = fmt.Fprintf(errOut, "cloudringctl rsi: %v\n", err) + return 2 + } + encoder := json.NewEncoder(out) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return 1 + } + if report.Verdict != rsi.VerdictPass { + return 1 + } + return 0 +} diff --git a/cmd/cloudringctl/rsi_test.go b/cmd/cloudringctl/rsi_test.go new file mode 100644 index 0000000..b65fb74 --- /dev/null +++ b/cmd/cloudringctl/rsi_test.go @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: Apache-2.0 +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/opencloudtech/CloudRING/pkg/rsi" +) + +func TestRSIEvaluatePropagatesVerdict(t *testing.T) { + for _, tc := range []struct { + fixture, verdict, admission, effClass, kill string + code, failed int + }{ + {"compliant-k3-profile.json", rsi.VerdictPass, rsi.AdmissionDeploy, rsi.ClassK3, rsi.KillSwitchNone, 0, 0}, + {"merge-only-k3-profile.json", rsi.VerdictPass, rsi.AdmissionMerge, rsi.ClassK3, rsi.KillSwitchNone, 0, 0}, + {"tampered-k4-profile.json", rsi.VerdictFail, rsi.AdmissionDeploy, rsi.ClassK4, rsi.KillSwitchL3, 1, 9}, + } { + t.Run(tc.fixture, func(t *testing.T) { + var out, errOut bytes.Buffer + code := run([]string{"rsi", "evaluate", "--profile", filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", tc.fixture)}, &out, &errOut) + if code != tc.code { + t.Fatalf("code=%d err=%s out=%s", code, &errOut, &out) + } + var rep rsi.Report + if err := json.Unmarshal(out.Bytes(), &rep); err != nil { + t.Fatal(err) + } + if rep.Verdict != tc.verdict || rep.Admission != tc.admission || rep.Classification.EffectiveClass != tc.effClass || + rep.KillSwitchLevel != tc.kill || rep.Failed != tc.failed { + t.Fatalf("unexpected RSI report: %+v", rep) + } + }) + } +} + +// A deploy-admission profile without an S1-S6 receipt fails the oracle. +func TestRSIEvaluateDeployRequiresLKGReceipt(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", "compliant-k3-profile.json")) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + delete(doc, "lkg") + stripped, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + profile := filepath.Join(t.TempDir(), "deploy-without-lkg.json") + if err := os.WriteFile(profile, stripped, 0o600); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"rsi", "evaluate", "--profile", profile}, &out, &errOut); code != 1 { + t.Fatalf("code=%d err=%s out=%s", code, &errOut, &out) + } + var rep rsi.Report + if err := json.Unmarshal(out.Bytes(), &rep); err != nil { + t.Fatal(err) + } + for _, c := range rep.Checks { + if c.ID == "s1_s6_lkg_promotion" { + if c.Status != rsi.StatusFail || rep.Failed != 1 || rep.Verdict != rsi.VerdictFail { + t.Fatalf("deploy without LKG receipt: %+v", rep) + } + return + } + } + t.Fatalf("report has no s1_s6_lkg_promotion check: %+v", rep.Checks) +} + +// A key that differs from a field name only in case is rejected, so the +// evaluator never reads a profile differently from other JSON readers. +func TestRSIEvaluateRejectsCaseVariantKeys(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", "merge-only-k3-profile.json")) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + doc["candidate"].(map[string]any)["DeploymentTargets"] = []any{"hub"} + variant, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + profile := filepath.Join(t.TempDir(), "case-variant.json") + if err := os.WriteFile(profile, variant, 0o600); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"rsi", "evaluate", "--profile", profile}, &out, &errOut); code != 2 || out.Len() != 0 { + t.Fatalf("code=%d err=%s out=%s", code, &errOut, &out) + } +} + +func TestRSIEvaluateRejectsInvalidInvocations(t *testing.T) { + compliant := filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", "compliant-k3-profile.json") + for _, args := range [][]string{ + {"rsi"}, + {"rsi", "--profile", compliant}, + {"rsi", "evaluate"}, + {"rsi", "evaluate", "--profile"}, + {"rsi", "evaluate", "--profile", filepath.Join(t.TempDir(), "missing.json")}, + {"rsi", "evaluate", "--profile", compliant, "extra"}, + } { + var out, errOut bytes.Buffer + if code := run(args, &out, &errOut); code != 2 || out.Len() != 0 || errOut.Len() == 0 { + t.Fatalf("args=%v code=%d out=%s err=%s", args, code, &out, &errOut) + } + } +} + +// Only a first argument "rsi" selects the RSI command; anything else, including +// no arguments, keeps the goal verification behaviour. +func TestRunDispatchesOnlyTheRSISubcommand(t *testing.T) { + for _, tc := range []struct { + args []string + reason string + }{ + {nil, "invalid_arguments"}, + {[]string{"verify"}, "invalid_arguments"}, + {[]string{"verify", "goal", "--goal", "G01", "--state", "roadmap/state/G00.json", "--output", "json"}, `unsupported_goal_"G01"`}, + } { + var out, errOut bytes.Buffer + code := run(tc.args, &out, &errOut) + want := "cloudring_goal_verification status=BLOCKED reason=" + tc.reason + "\n" + if code != 2 || out.Len() != 0 || errOut.String() != want { + t.Fatalf("args=%v code=%d out=%q err=%q", tc.args, code, &out, &errOut) + } + } +} + +// A profile of exactly the evaluator's size bound is read whole. +func TestRSIEvaluateReadsAProfileAtTheSizeBound(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", "compliant-k3-profile.json")) + if err != nil { + t.Fatal(err) + } + open := bytes.IndexByte(raw, '{') + trimmed := bytes.TrimRight(raw, " \t\r\n") + if open < 0 || len(trimmed) > maxRSIProfileBytes { + t.Fatalf("unexpected fixture") + } + padded := append(append(append([]byte{}, trimmed[:open+1]...), bytes.Repeat([]byte{' '}, maxRSIProfileBytes-len(trimmed))...), trimmed[open+1:]...) + if len(padded) != maxRSIProfileBytes || padded[len(padded)-1] != '}' { + t.Fatalf("padded profile has %d bytes", len(padded)) + } + profile := filepath.Join(t.TempDir(), "bound.json") + if err := os.WriteFile(profile, padded, 0o600); err != nil { + t.Fatal(err) + } + var out, errOut bytes.Buffer + if code := run([]string{"rsi", "evaluate", "--profile", profile}, &out, &errOut); code != 0 { + t.Fatalf("code=%d err=%s", code, &errOut) + } +} + +func TestRSIEvaluateReportsAnUnreadableProfile(t *testing.T) { + var out, errOut bytes.Buffer + if code := run([]string{"rsi", "evaluate", "--profile", t.TempDir()}, &out, &errOut); code != 2 || out.Len() != 0 || + errOut.String() != "cloudringctl rsi: cannot read profile\n" { + t.Fatalf("code=%d out=%q err=%q", code, &out, &errOut) + } +} diff --git a/contracts/rsi-governance/fixtures/compliant-k3-profile.json b/contracts/rsi-governance/fixtures/compliant-k3-profile.json index e2ee7ed..96cddbd 100644 --- a/contracts/rsi-governance/fixtures/compliant-k3-profile.json +++ b/contracts/rsi-governance/fixtures/compliant-k3-profile.json @@ -67,6 +67,18 @@ "overseerModelFamily": "family-b" } }, + "lkg": { + "immutableBootloaderVerified": true, + "pinnedLkgSlots": 2, + "soakMinutes": 30, + "webhookBypassVerified": true, + "preMigrationSnapshotVerified": true, + "metaRsiSelfCheckPassed": true, + "candidateSha": "a326123af6ca994ce876bb786dff7683f181ca36", + "quarantinedShas": [ + "deadbeef00000000000000000000000000000000" + ] + }, "voi": { "priorSuccessProb": "2/5", "successNetPayoffEur": "120000", diff --git a/contracts/rsi-governance/fixtures/merge-only-k3-profile.json b/contracts/rsi-governance/fixtures/merge-only-k3-profile.json new file mode 100644 index 0000000..847f4f3 --- /dev/null +++ b/contracts/rsi-governance/fixtures/merge-only-k3-profile.json @@ -0,0 +1,48 @@ +{ + "apiVersion": "cloudring.org/rsi-governance/v1", + "kind": "RSICandidateProfile", + "candidate": { + "id": "cand-k3-rsi-merge-01", + "declaredClass": "K3", + "changedPaths": [ + "pkg/rsi/rsi.go", + "cmd/cloudringctl/commands.go" + ], + "readOnlyOnly": false, + "reversibleConfigOnly": false, + "deletedOrSkippedTests": 0, + "modifiedExistingTests": false, + "modifiedTestdata": false, + "humanK4ApprovalReference": "", + "admission": "merge", + "targetRepository": "opencloudtech/CloudRING", + "targetBranch": "main", + "deploymentTargets": [], + "origin": "operator_directed", + "operatorReference": "review-1" + }, + "gates": { + "g1": { + "evaluatorFrozenHashVerified": true, + "suitePassed": true, + "newTestFailedBeforeFix": true, + "newTestPassedAfterFix": true, + "rerunMaskingUsed": false, + "silentErrorSwallowing": 0, + "unseenHoldoutPassed": true, + "seededErrorRejected": true + }, + "g2": { + "killedMutants": 19, + "totalMutants": 20, + "fuzzTargetsPassed": true + }, + "g3": { + "sastFindings": 0, + "vulnFindings": 0, + "externalCapabilityGate": true, + "promptDelimiterOnlySecurity": false, + "explanationOnlyApproval": false + } + } +} diff --git a/contracts/slice-oracles/oracles-32.json b/contracts/slice-oracles/oracles-32.json index 5b49eec..765e0fc 100644 --- a/contracts/slice-oracles/oracles-32.json +++ b/contracts/slice-oracles/oracles-32.json @@ -34,7 +34,7 @@ "roadmap" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C01.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C01.json", "requiredGates": [ "G1", "G2", @@ -61,12 +61,24 @@ "qualDependencies": [], "requiresFork3Hardware": false, "allowedPaths": [ + ".github/safepush.json", + "cmd/cloudring-mutation", + "cmd/cloudring-safepush", + "cmd/cloudringctl/main.go", + "cmd/cloudringctl/rsi.go", + "cmd/cloudringctl/rsi_test.go", + "contracts/rsi-governance", + "contracts/slice-oracles", + "docs/ci-checks.md", + "docs/safepush.md", "pkg/rsi", + "pkg/safepush", "pkg/sliceoracles", - "contracts/slice-oracles" + "roadmap/DELIVERY_SLICES.md", + "roadmap/EXECUTION_CONTRACT.md" ], - "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/SP01.json", + "autonomyClass": "K4", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/SP01.json", "requiredGates": [ "G1", "G2", @@ -98,7 +110,7 @@ "internal/portal" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C02.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C02.json", "requiredGates": [ "G1", "G2", @@ -129,7 +141,7 @@ "internal/api" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C03.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C03.json", "requiredGates": [ "G1", "G2", @@ -161,7 +173,7 @@ "pkg/iam" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C04.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C04.json", "requiredGates": [ "G1", "G2", @@ -196,7 +208,7 @@ "sdk/ocsv3" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C05.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C05.json", "requiredGates": [ "G1", "G2", @@ -231,7 +243,7 @@ "pkg/dataact" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C06.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C06.json", "requiredGates": [ "G1", "G2", @@ -264,7 +276,7 @@ "pkg/network" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C07.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C07.json", "requiredGates": [ "G1", "G2", @@ -298,7 +310,7 @@ "pkg/installer" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C08.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C08.json", "requiredGates": [ "G1", "G2", @@ -334,7 +346,7 @@ "pkg/directory" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C09.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C09.json", "requiredGates": [ "G1", "G2", @@ -365,7 +377,7 @@ "pkg/inventory" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C10.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C10.json", "requiredGates": [ "G1", "G2", @@ -399,7 +411,7 @@ "sdk/rust/ocs-sdk-rs" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C11.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C11.json", "requiredGates": [ "G1", "G2", @@ -430,7 +442,7 @@ "pkg/subscriptions" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C12.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C12.json", "requiredGates": [ "G1", "G2", @@ -461,7 +473,7 @@ "pkg/metering" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C13.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C13.json", "requiredGates": [ "G1", "G2", @@ -492,7 +504,7 @@ "cmd/cloudringctl" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C14.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C14.json", "requiredGates": [ "G1", "G2", @@ -526,7 +538,7 @@ "pkg/products/network" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C15.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C15.json", "requiredGates": [ "G1", "G2", @@ -560,7 +572,7 @@ "pkg/products/volume" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C16.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C16.json", "requiredGates": [ "G1", "G2", @@ -594,7 +606,7 @@ "pkg/products/image" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C17.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C17.json", "requiredGates": [ "G1", "G2", @@ -626,7 +638,7 @@ "pkg/products/compute" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C18.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C18.json", "requiredGates": [ "G1", "G2", @@ -656,7 +668,7 @@ "pkg/products/kubernetes" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C19.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C19.json", "requiredGates": [ "G1", "G2", @@ -690,7 +702,7 @@ "pkg/products/objectstorage" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C20.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C20.json", "requiredGates": [ "G1", "G2", @@ -721,7 +733,7 @@ "pkg/products/backup" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C21.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C21.json", "requiredGates": [ "G1", "G2", @@ -752,7 +764,7 @@ "pkg/products/access" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C22.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C22.json", "requiredGates": [ "G1", "G2", @@ -782,7 +794,7 @@ "pkg/products/support" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C23.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C23.json", "requiredGates": [ "G1", "G2", @@ -812,7 +824,7 @@ "pkg/products/external" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C24.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C24.json", "requiredGates": [ "G1", "G2", @@ -869,7 +881,7 @@ "pkg/resilience" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C25.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C25.json", "requiredGates": [ "G1", "G2", @@ -899,7 +911,7 @@ "pkg/operations" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C26.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C26.json", "requiredGates": [ "G1", "G2", @@ -929,7 +941,7 @@ "pkg/portability" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C27.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C27.json", "requiredGates": [ "G1", "G2", @@ -989,7 +1001,7 @@ "roadmap/state" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C28.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/C28.json", "requiredGates": [ "G1", "G2", @@ -1026,7 +1038,7 @@ "pkg/products/dbaas" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/DBAAS.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/DBAAS.json", "requiredGates": [ "G1", "G2", @@ -1063,7 +1075,7 @@ "pkg/products/dnsaas" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/DNSAAS.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/DNSAAS.json", "requiredGates": [ "G1", "G2", @@ -1099,7 +1111,7 @@ "pkg/products/filesaas" ], "autonomyClass": "K2", - "verificationCommand": "go test ./... && cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/FILESAAS.json", + "verificationCommand": "go test ./... && go run ./cmd/cloudringctl rsi evaluate --profile contracts/slice-oracles/profiles/FILESAAS.json", "requiredGates": [ "G1", "G2", diff --git a/contracts/slice-oracles/profiles/SP01.json b/contracts/slice-oracles/profiles/SP01.json new file mode 100644 index 0000000..855b944 --- /dev/null +++ b/contracts/slice-oracles/profiles/SP01.json @@ -0,0 +1,93 @@ +{ + "apiVersion": "cloudring.org/rsi-governance/v1", + "kind": "RSICandidateProfile", + "candidate": { + "id": "SP01-pr168", + "declaredClass": "K4", + "changedPaths": [ + ".github/safepush.json", + "cmd/cloudring-mutation/diff.go", + "cmd/cloudring-mutation/execute.go", + "cmd/cloudring-mutation/main.go", + "cmd/cloudring-mutation/mutation_test.go", + "cmd/cloudring-mutation/procattr_other.go", + "cmd/cloudring-mutation/procattr_unix.go", + "cmd/cloudring-mutation/sites.go", + "cmd/cloudring-safepush/arguments_test.go", + "cmd/cloudring-safepush/main.go", + "cmd/cloudring-safepush/main_test.go", + "cmd/cloudringctl/main.go", + "cmd/cloudringctl/rsi.go", + "cmd/cloudringctl/rsi_test.go", + "contracts/rsi-governance/fixtures/compliant-k3-profile.json", + "contracts/rsi-governance/fixtures/merge-only-k3-profile.json", + "contracts/rsi-governance/fixtures/tampered-k4-profile.json", + "contracts/slice-oracles/oracles-32.json", + "contracts/slice-oracles/profiles/SP01.json", + "docs/ci-checks.md", + "docs/safepush.md", + "pkg/rsi/admission_test.go", + "pkg/rsi/classify_limits_test.go", + "pkg/rsi/decision_limits_test.go", + "pkg/rsi/gate_limits_test.go", + "pkg/rsi/rsi.go", + "pkg/rsi/rsi_fuzz_test.go", + "pkg/rsi/rsi_test.go", + "pkg/safepush/checks.go", + "pkg/safepush/checks_test.go", + "pkg/safepush/doc.go", + "pkg/safepush/github.go", + "pkg/safepush/github_fields_test.go", + "pkg/safepush/github_test.go", + "pkg/safepush/policy.go", + "pkg/safepush/policy_limits_test.go", + "pkg/safepush/policy_test.go", + "pkg/safepush/repository_policy_test.go", + "pkg/safepush/workflow_source.go", + "pkg/safepush/workflow_source_test.go", + "pkg/sliceoracles/admission_test.go", + "pkg/sliceoracles/oracles.go", + "pkg/sliceoracles/oracles_test.go", + "pkg/sliceoracles/validate_limits_test.go", + "roadmap/DELIVERY_SLICES.md", + "roadmap/EXECUTION_CONTRACT.md" + ], + "readOnlyOnly": false, + "reversibleConfigOnly": false, + "deletedOrSkippedTests": 0, + "modifiedExistingTests": false, + "modifiedTestdata": false, + "humanK4ApprovalReference": "", + "admission": "merge", + "targetRepository": "opencloudtech/CloudRING", + "targetBranch": "main", + "deploymentTargets": [], + "origin": "operator_directed", + "operatorReference": "opencloudtech/CloudRING#168", + "evaluatorBootstrap": true + }, + "gates": { + "g1": { + "evaluatorFrozenHashVerified": false, + "suitePassed": true, + "newTestFailedBeforeFix": true, + "newTestPassedAfterFix": true, + "rerunMaskingUsed": false, + "silentErrorSwallowing": 0, + "unseenHoldoutPassed": true, + "seededErrorRejected": true + }, + "g2": { + "killedMutants": 996, + "totalMutants": 1022, + "fuzzTargetsPassed": true + }, + "g3": { + "sastFindings": 0, + "vulnFindings": 0, + "externalCapabilityGate": false, + "promptDelimiterOnlySecurity": false, + "explanationOnlyApproval": false + } + } +} diff --git a/pkg/rsi/admission_test.go b/pkg/rsi/admission_test.go new file mode 100644 index 0000000..342936a --- /dev/null +++ b/pkg/rsi/admission_test.go @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package rsi + +import ( + "encoding/json" + "strings" + "testing" +) + +// mergeOnlyProfile is a K3 candidate merged into public main, which deploys +// nothing, directed by an operator and carrying no deployment, drill, history, +// budget or LKG receipt. +func mergeOnlyProfile() CandidateProfile { + p := validCompliantK3Profile() + p.Candidate.Admission = AdmissionMerge + p.Candidate.TargetRepository = "opencloudtech/CloudRING" + p.Candidate.TargetBranch = "main" + p.Candidate.Origin = OriginOperatorDirected + p.Candidate.OperatorReference = "review-1" + p.Gates.G4 = G4Receipt{} + p.Gates.G5 = G5Receipt{} + p.Gates.G6 = G6Receipt{} + p.Gates.G7 = G7Receipt{} + p.LKG = nil + p.VOI, p.PassK, p.Experiment, p.Discovery = nil, nil, nil, nil + return p +} + +func evaluate(t *testing.T, p CandidateProfile) Report { + t.Helper() + rep, err := EvaluateCandidate(p) + if err != nil { + t.Fatalf("EvaluateCandidate: %v", err) + } + return rep +} + +func checkOf(rep Report, id string) CheckResult { + for _, c := range rep.Checks { + if c.ID == id { + return c + } + } + return CheckResult{ID: id, Status: "missing"} +} + +func checkStatus(rep Report, id string) string { + return checkOf(rep, id).Status +} + +func TestMergeAdmissionPassesWithoutDeploymentEvidence(t *testing.T) { + rep := evaluate(t, mergeOnlyProfile()) + if rep.Verdict != VerdictPass || rep.Admission != AdmissionMerge || rep.Failed != 0 || rep.KillSwitchLevel != KillSwitchNone { + t.Fatalf("merge-only candidate: verdict=%s admission=%s failed=%d kill=%s checks=%+v", rep.Verdict, rep.Admission, rep.Failed, rep.KillSwitchLevel, rep.Checks) + } + for _, id := range []string{"g4_canary_slo_and_stateful_downgrade", "g5_portability_and_switching_drill", + "g6_dora_cfr_circuit_breaker", "g7_budget_and_cross_family_overseer", "s1_s6_lkg_promotion"} { + if got := checkStatus(rep, id); got != StatusNotApplicable { + t.Errorf("%s: status %s, want %s", id, got, StatusNotApplicable) + } + } + if rep.NotApplicable != 5 || rep.Passed+rep.Failed+rep.NotApplicable != len(rep.Checks) { + t.Fatalf("check accounting: passed=%d failed=%d notApplicable=%d checks=%d", rep.Passed, rep.Failed, rep.NotApplicable, len(rep.Checks)) + } + if got := checkOf(rep, "admission_scope_consistency").Summary; !strings.Contains(got, "target=opencloudtech/CloudRING@main") { + t.Fatalf("scope summary does not name the merge target: %q", got) + } + + // Supplied evidence is never ignored: a failed drill or a tripped breaker fails a merge too. + p := mergeOnlyProfile() + p.Gates.G5 = G5Receipt{PortabilityDrillPassed: false, DataActVerdict: VerdictFail} + if rep := evaluate(t, p); rep.Verdict != VerdictFail || checkStatus(rep, "g5_portability_and_switching_drill") != StatusFail { + t.Fatalf("supplied failing G5 receipt was not evaluated: %+v", rep.Checks) + } + p = mergeOnlyProfile() + p.Gates.G6 = G6Receipt{AIFailedDeployments: 4, AITotalDeployments: 20} + if rep := evaluate(t, p); rep.Verdict != VerdictFail || rep.MaxAutonomousClass != ClassK1 || rep.KillSwitchLevel != KillSwitchL1 { + t.Fatalf("AI CFR 4/20 must trip the breaker for a merge: verdict=%s maxAuto=%s kill=%s", rep.Verdict, rep.MaxAutonomousClass, rep.KillSwitchLevel) + } +} + +func TestDeployAdmissionAppliesEveryGate(t *testing.T) { + for _, admission := range []string{"", AdmissionDeploy} { + p := mergeOnlyProfile() + p.Candidate.Admission = admission + p.Candidate.DeploymentTargets = []string{"hub"} + rep := evaluate(t, p) + if rep.Verdict != VerdictFail || rep.Admission != AdmissionDeploy || rep.NotApplicable != 0 { + t.Fatalf("admission %q: verdict=%s admission=%s notApplicable=%d", admission, rep.Verdict, rep.Admission, rep.NotApplicable) + } + // An operator-directed deployment without receipts fails G4, G5, G7 and S1-S6. + for _, id := range []string{"g4_canary_slo_and_stateful_downgrade", "g5_portability_and_switching_drill", + "g7_budget_and_cross_family_overseer", "s1_s6_lkg_promotion"} { + if got := checkStatus(rep, id); got != StatusFail { + t.Errorf("admission %q: %s status %s, want %s", admission, id, got, StatusFail) + } + } + } + // The full deploy receipt of the compliant profile still passes unchanged. + rep := evaluate(t, validCompliantK3Profile()) + if rep.Verdict != VerdictPass || rep.Admission != AdmissionDeploy || rep.NotApplicable != 0 || checkStatus(rep, "s1_s6_lkg_promotion") != StatusPass { + t.Fatalf("compliant deploy profile: verdict=%s admission=%s notApplicable=%d checks=%+v", rep.Verdict, rep.Admission, rep.NotApplicable, rep.Checks) + } +} + +func TestDeployAdmissionEvaluatesLKGReceipt(t *testing.T) { + p := validCompliantK3Profile() + p.LKG = nil + rep := evaluate(t, p) + if rep.Verdict != VerdictFail || checkStatus(rep, "s1_s6_lkg_promotion") != StatusFail { + t.Fatalf("deploy without LKG receipt: verdict=%s checks=%+v", rep.Verdict, rep.Checks) + } + + p = validCompliantK3Profile() + p.LKG.CandidateSHA = p.LKG.QuarantinedSHAs[0] + rep = evaluate(t, p) + if c := checkOf(rep, "s1_s6_lkg_promotion"); rep.Verdict != VerdictFail || c.Status != StatusFail || !strings.HasPrefix(c.Summary, "S6: ") { + t.Fatalf("quarantined candidate SHA: verdict=%s check=%+v", rep.Verdict, c) + } + + p = validCompliantK3Profile() + p.LKG = &LKGReceipt{} + if blockers := ValidateLKG(*p.LKG); len(blockers) != 7 { + t.Fatalf("empty LKG receipt: %d blockers, want 7: %v", len(blockers), blockers) + } + if rep := evaluate(t, p); rep.Verdict != VerdictFail || checkStatus(rep, "s1_s6_lkg_promotion") != StatusFail { + t.Fatalf("empty LKG receipt passed: %+v", rep.Checks) + } +} + +func TestMislabelledMergeAdmissionFailsClosed(t *testing.T) { + addPath := func(path string) func(*CandidateProfile) { + return func(p *CandidateProfile) { p.Candidate.ChangedPaths = append(p.Candidate.ChangedPaths, path) } + } + cases := map[string]func(*CandidateProfile){ + "deployment target": func(p *CandidateProfile) { p.Candidate.DeploymentTargets = []string{"hub"} }, + "deploy manifest": addPath("deploy/kubernetes/storage/x.yaml"), + "nested deploy dir": addPath("reference/synthetic-service/Deploy/kubernetes.yaml"), + "iac": addPath("iac/main.tf"), + "branch workflow": addPath(".github/workflows/ci.yml"), + "traversal": addPath("pkg/../deploy/x.yaml"), + "gitlink": addPath("cloudring_core"), + "submodule manifest": addPath(".gitmodules"), + "canary receipt": func(p *CandidateProfile) { p.Gates.G4 = validCompliantK3Profile().Gates.G4 }, + "lkg receipt": func(p *CandidateProfile) { p.LKG = validCompliantK3Profile().LKG }, + "no target": func(p *CandidateProfile) { p.Candidate.TargetRepository, p.Candidate.TargetBranch = "", "" }, + "gitops repository": func(p *CandidateProfile) { p.Candidate.TargetRepository = "opencloudtech/cloudring-enterprise" }, + "case-variant target": func(p *CandidateProfile) { p.Candidate.TargetRepository = "opencloudtech/cloudring" }, + "other branch": func(p *CandidateProfile) { p.Candidate.TargetBranch = "release" }, + "unknown origin": func(p *CandidateProfile) { p.Candidate.Origin = "self" }, + "unnamed operator": func(p *CandidateProfile) { p.Candidate.OperatorReference = " " }, + "zero-width operator": func(p *CandidateProfile) { p.Candidate.OperatorReference = "\u200b" }, + "nbsp operator": func(p *CandidateProfile) { p.Candidate.OperatorReference = "\u00a0" }, + "hangul filler operator": func(p *CandidateProfile) { p.Candidate.OperatorReference = "\u3164" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + p := mergeOnlyProfile() + mutate(&p) + rep := evaluate(t, p) + if rep.Verdict != VerdictFail || rep.Admission != AdmissionDeploy || checkStatus(rep, "admission_scope_consistency") != StatusFail || + checkStatus(rep, "g4_canary_slo_and_stateful_downgrade") == StatusNotApplicable || + checkStatus(rep, "g7_budget_and_cross_family_overseer") == StatusNotApplicable || + checkStatus(rep, "s1_s6_lkg_promotion") == StatusNotApplicable { + t.Fatalf("mislabelled merge was not evaluated as a failing deployment: admission=%s checks=%+v", rep.Admission, rep.Checks) + } + }) + } + for _, field := range []string{`"admission":"ship"`, `"origin":"self"`} { + raw := []byte(`{"apiVersion":"cloudring.org/rsi-governance/v1","kind":"RSICandidateProfile","candidate":{"id":"c","declaredClass":"K2","changedPaths":["pkg/x.go"],` + field + `}}`) + if _, err := ParseProfile(raw); err == nil { + t.Fatalf("ParseProfile accepted %s", field) + } + } +} + +func TestDeployColdStartRequiresHumanK4Approval(t *testing.T) { + coldDeploy := func(approval string) CandidateProfile { + p := validCompliantK3Profile() + p.Candidate.HumanK4ApprovalReference = approval + p.Gates.G6 = G6Receipt{HumanFailedDeployments: 1, HumanTotalDeployments: 10} + return p + } + for _, approval := range []string{"", "\u200b", "\u00a0\u00a0"} { + rep := evaluate(t, coldDeploy(approval)) + if rep.Classification.EffectiveClass != ClassK4 || !strings.Contains(strings.Join(rep.Classification.EscalationReasons, ","), "deploy_cold_start_requires_k4") || + rep.Verdict != VerdictFail || checkStatus(rep, "k4_human_boundary") != StatusFail { + t.Fatalf("cold-start deploy with approval %q: class=%s reasons=%v verdict=%s checks=%+v", + approval, rep.Classification.EffectiveClass, rep.Classification.EscalationReasons, rep.Verdict, rep.Checks) + } + } + rep := evaluate(t, coldDeploy("change-42")) + if rep.Verdict != VerdictPass || rep.Classification.EffectiveClass != ClassK4 || checkStatus(rep, "g6_dora_cfr_circuit_breaker") != StatusPass { + t.Fatalf("human-approved first AI deployment: verdict=%s class=%s checks=%+v", rep.Verdict, rep.Classification.EffectiveClass, rep.Checks) + } + // With AI history a human approval does not bypass the breaker, and the + // cold-start escalation no longer applies. + p := coldDeploy("change-42") + p.Gates.G6.AIFailedDeployments, p.Gates.G6.AITotalDeployments = 4, 20 + rep = evaluate(t, p) + if rep.Verdict != VerdictFail || rep.MaxAutonomousClass != ClassK1 || strings.Contains(strings.Join(rep.Classification.EscalationReasons, ","), "deploy_cold_start") { + t.Fatalf("approval bypassed an existing CFR breach: verdict=%s maxAuto=%s reasons=%v", rep.Verdict, rep.MaxAutonomousClass, rep.Classification.EscalationReasons) + } + rep = evaluate(t, mergeOnlyProfile()) + if checkStatus(rep, "g6_dora_cfr_circuit_breaker") != StatusNotApplicable || rep.Classification.EffectiveClass != ClassK3 { + t.Fatalf("merge cold start: class=%s checks=%+v", rep.Classification.EffectiveClass, rep.Checks) + } +} + +func TestG7SkipsOnlyOperatorDirectedMerge(t *testing.T) { + for _, origin := range []string{"", OriginAutonomousLoop} { + p := mergeOnlyProfile() + p.Candidate.Origin = origin + if rep := evaluate(t, p); rep.Verdict != VerdictFail || checkStatus(rep, "g7_budget_and_cross_family_overseer") != StatusFail { + t.Fatalf("origin %q without budget receipt: checks=%+v", origin, rep.Checks) + } + } + // An operator-directed deployment is still bounded by G7. + p := validCompliantK3Profile() + p.Candidate.Origin = OriginOperatorDirected + p.Candidate.OperatorReference = "review-1" + p.Gates.G7 = G7Receipt{} + if rep := evaluate(t, p); rep.Verdict != VerdictFail || checkStatus(rep, "g7_budget_and_cross_family_overseer") != StatusFail { + t.Fatalf("operator-directed deploy without budget receipt: checks=%+v", rep.Checks) + } + p = mergeOnlyProfile() + p.Gates.G7 = validCompliantK3Profile().Gates.G7 + p.Gates.G7.OverseerModelFamily = p.Gates.G7.ExecutorModelFamily + if rep := evaluate(t, p); rep.Verdict != VerdictFail || checkStatus(rep, "g7_budget_and_cross_family_overseer") != StatusFail { + t.Fatalf("supplied same-family G7 receipt was not evaluated: %+v", rep.Checks) + } +} + +func TestEvaluatorBootstrapOnlyAdmitsMerge(t *testing.T) { + p := mergeOnlyProfile() + p.Candidate.EvaluatorBootstrap = true + p.Gates.G1.EvaluatorFrozenHashVerified = false + rep := evaluate(t, p) + if rep.Classification.EffectiveClass != ClassK4 || rep.Verdict != VerdictFail || rep.Failed != 1 || checkStatus(rep, "k4_human_boundary") != StatusFail { + t.Fatalf("unapproved evaluator bootstrap: class=%s verdict=%s failed=%d checks=%+v", rep.Classification.EffectiveClass, rep.Verdict, rep.Failed, rep.Checks) + } + p.Candidate.HumanK4ApprovalReference = "approval-7" + if rep := evaluate(t, p); rep.Verdict != VerdictPass { + t.Fatalf("approved evaluator bootstrap merge: checks=%+v", rep.Checks) + } + + // A deployment is judged by a frozen evaluator: bootstrap does not stand in for the hash. + d := validCompliantK3Profile() + d.Candidate.EvaluatorBootstrap = true + d.Candidate.HumanK4ApprovalReference = "approval-7" + d.Gates.G1.EvaluatorFrozenHashVerified = false + rep = evaluate(t, d) + if rep.Verdict != VerdictFail || checkStatus(rep, "g1_red_green_and_test_integrity") != StatusFail || checkStatus(rep, "k4_human_boundary") != StatusPass { + t.Fatalf("deploy with evaluator bootstrap and no frozen hash: verdict=%s checks=%+v", rep.Verdict, rep.Checks) + } + d.Gates.G1.EvaluatorFrozenHashVerified = true + if rep := evaluate(t, d); rep.Verdict != VerdictPass { + t.Fatalf("approved deploy with a frozen evaluator: checks=%+v", rep.Checks) + } +} + +// profileJSON marshals p, applies edit to its generic JSON tree and returns the bytes. +func profileJSON(t *testing.T, p CandidateProfile, edit func(map[string]any)) []byte { + t.Helper() + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + var tree map[string]any + if err := json.Unmarshal(raw, &tree); err != nil { + t.Fatal(err) + } + edit(tree) + out, err := json.Marshal(tree) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestParseProfileRejectsCaseVariantAndUnknownKeys(t *testing.T) { + obj := func(tree map[string]any, keys ...string) map[string]any { + for _, k := range keys { + tree = tree[k].(map[string]any) + } + return tree + } + deploy := validCompliantK3Profile() + cases := map[string]struct { + profile CandidateProfile + edit func(map[string]any) + }{ + "candidate case variant hides targets": {mergeOnlyProfile(), func(m map[string]any) { + obj(m, "candidate")["DeploymentTargets"] = []any{"hub"} + }}, + "case variant admission alone": {deploy, func(m map[string]any) { + c := obj(m, "candidate") + delete(c, "admission") + c["Admission"] = "merge" + }}, + "gates case variant zeroes g4": {deploy, func(m map[string]any) { obj(m, "gates")["G4"] = map[string]any{} }}, + "nested g4 field case variant": {deploy, func(m map[string]any) { obj(m, "gates", "g4")["CanaryStepPercent"] = "50" }}, + "lkg field case variant": {deploy, func(m map[string]any) { obj(m, "lkg")["CandidateSHA"] = "0" }}, + "array element case variant": {deploy, func(m map[string]any) { + obj(m, "passK")["tasks"].([]any)[0].(map[string]any)["TaskId"] = "t9" + }}, + "top-level case variant": {deploy, func(m map[string]any) { m["LKG"] = map[string]any{} }}, + "unknown key": {deploy, func(m map[string]any) { obj(m, "candidate")["notes"] = "x" }}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if _, err := ParseProfile(profileJSON(t, tc.profile, tc.edit)); err == nil { + t.Fatal("ParseProfile accepted a key that is not an exact field name") + } + }) + } + for _, p := range []CandidateProfile{deploy, mergeOnlyProfile()} { + if _, err := ParseProfile(profileJSON(t, p, func(map[string]any) {})); err != nil { + t.Fatalf("ParseProfile rejected an exact profile: %v", err) + } + } +} + +func TestReferencesMustNameARecord(t *testing.T) { + // Invisible text: zero-width and bidi format characters, no-break spaces, + // control characters, Hangul fillers (default-ignorable letters), the + // combining grapheme joiner, variation selectors and the Egyptian + // hieroglyph blanks (letters that render as empty space). + invisible := []string{ + "\u200b", "\u00a0", "\u00a0\u00a0", "a\u200bb", "x\u0007", "\u202eabc", "-", " ", + "\u115f", "\u1160", "\u3164", "\uffa0", "\u3164\u3164", "a\u3164", "\u034f", "\ufe0f", "a\ufe0f", "\u180b", "\U000e0100", + "\U00013441", "\U00013442", "a\U00013441", "\U00013441\U00013442", + } + for _, ref := range invisible { + for _, field := range []string{"operatorReference", "humanK4ApprovalReference"} { + raw := profileJSON(t, mergeOnlyProfile(), func(m map[string]any) { m["candidate"].(map[string]any)[field] = ref }) + if _, err := ParseProfile(raw); err == nil { + t.Errorf("ParseProfile accepted %s %q", field, ref) + } + } + p := mergeOnlyProfile() + p.Candidate.EvaluatorBootstrap = true + p.Candidate.HumanK4ApprovalReference = ref + if rep := evaluate(t, p); checkStatus(rep, "k4_human_boundary") != StatusFail { + t.Errorf("K4 approved by reference %q", ref) + } + } + for _, ref := range []string{"opencloudtech/CloudRING#168", "change-42", "Ж1"} { + raw := profileJSON(t, mergeOnlyProfile(), func(m map[string]any) { + m["candidate"].(map[string]any)["operatorReference"] = ref + m["candidate"].(map[string]any)["humanK4ApprovalReference"] = ref + }) + if _, err := ParseProfile(raw); err != nil { + t.Errorf("ParseProfile rejected reference %q: %v", ref, err) + } + } +} diff --git a/pkg/rsi/classify_limits_test.go b/pkg/rsi/classify_limits_test.go new file mode 100644 index 0000000..fbb25a9 --- /dev/null +++ b/pkg/rsi/classify_limits_test.go @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package rsi + +import ( + "bytes" + "encoding/json" + "errors" + "slices" + "strings" + "testing" +) + +// Each case changes one path or flag so that every escalation rule decides the +// class (and the reasons) on its own. +func TestClassifyCandidateEscalationRules(t *testing.T) { + cases := []struct { + name string + meta CandidateMetadata + want string + wantWhy []string + readOnly bool + }{ + {"backslash path", CandidateMetadata{DeclaredClass: ClassK1, ChangedPaths: []string{`a\b.go`}}, ClassK4, []string{`invalid_or_backslash_path:a\b.go`}, false}, + {"blank path", CandidateMetadata{DeclaredClass: ClassK1, ChangedPaths: []string{" "}}, ClassK4, []string{"invalid_or_backslash_path: "}, false}, + {"absolute path", CandidateMetadata{DeclaredClass: ClassK1, ChangedPaths: []string{"/etc/passwd"}}, ClassK4, []string{"path_traversal_or_absolute:/etc/passwd"}, false}, + {"repository root", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"./"}}, ClassK4, []string{"path_traversal_or_absolute:./"}, false}, + {"rsi package root", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"pkg/rsi"}}, ClassK3, []string{"touches_ai_or_rsi_layer_requires_k3"}, false}, + {"nested rsi dir", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"docs/rsi/notes.md"}}, ClassK3, []string{"touches_ai_or_rsi_layer_requires_k3"}, false}, + {"skills", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"skills/a.md"}}, ClassK3, []string{"touches_ai_or_rsi_layer_requires_k3"}, false}, + {"agents", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"agents/a.md"}}, ClassK3, []string{"touches_ai_or_rsi_layer_requires_k3"}, false}, + {"prompts", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"prompts/a.txt"}}, ClassK3, []string{"touches_ai_or_rsi_layer_requires_k3"}, false}, + {"reversible docs", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"docs/a.md"}}, ClassK1, nil, false}, + {"testdata file", CandidateMetadata{DeclaredClass: ClassK2, ModifiedTestdata: true, ChangedPaths: []string{"pkg/x/testdata/a.json"}}, ClassK4, []string{"modified_testdata_fixtures_requires_k4"}, false}, + {"testdata dir", CandidateMetadata{DeclaredClass: ClassK2, ModifiedTestdata: true, ChangedPaths: []string{"pkg/x/testdata"}}, ClassK4, []string{"modified_testdata_fixtures_requires_k4"}, false}, + {"test file with testdata", CandidateMetadata{DeclaredClass: ClassK2, ModifiedTestdata: true, ChangedPaths: []string{"pkg/x/a_test.go"}}, ClassK4, []string{"modified_testdata_fixtures_requires_k4"}, false}, + {"test file only", CandidateMetadata{DeclaredClass: ClassK2, ChangedPaths: []string{"pkg/x/a_test.go"}}, ClassK2, nil, false}, + {"go file outside code roots", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"tools/x.go"}}, ClassK2, nil, false}, + {"cmd config", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"cmd/x.yaml"}}, ClassK2, nil, false}, + {"pkg config", CandidateMetadata{DeclaredClass: ClassK1, ReversibleConfigOnly: true, ChangedPaths: []string{"pkg/x.yaml"}}, ClassK2, nil, false}, + {"read-only production", CandidateMetadata{DeclaredClass: ClassK0, ChangedPaths: []string{"pkg/x.go"}}, ClassK0, nil, true}, + {"writable K0 docs", CandidateMetadata{DeclaredClass: ClassK0, ReversibleConfigOnly: true, ChangedPaths: []string{"docs/a.md"}}, ClassK1, nil, false}, + {"declared K3 docs", CandidateMetadata{DeclaredClass: ClassK3, ReversibleConfigOnly: true, ChangedPaths: []string{"docs/a.md"}}, ClassK3, nil, false}, + {"irreversible docs", CandidateMetadata{DeclaredClass: ClassK1, ChangedPaths: []string{"docs/a.md"}}, ClassK2, nil, false}, + {"declared K3 production", CandidateMetadata{DeclaredClass: ClassK3, ChangedPaths: []string{"pkg/x.go"}}, ClassK3, nil, false}, + {"declared K2 docs", CandidateMetadata{DeclaredClass: ClassK2, ChangedPaths: []string{"docs/a.md"}}, ClassK2, nil, false}, + {"declared K3 ai path", CandidateMetadata{DeclaredClass: ClassK3, ChangedPaths: []string{"pkg/rsi/x.go"}}, ClassK3, nil, false}, + {"declared K4 ai path", CandidateMetadata{DeclaredClass: ClassK4, ChangedPaths: []string{"pkg/rsi/x.go"}}, ClassK4, nil, false}, + {"duplicate protected path", CandidateMetadata{DeclaredClass: ClassK1, ChangedPaths: []string{".github/x", ".github/x"}}, ClassK4, []string{"protected_k4_path:.github/x"}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.meta.ID = "c" + tc.meta.ReadOnlyOnly = tc.readOnly + cls, err := ClassifyCandidate(tc.meta) + if err != nil { + t.Fatal(err) + } + want := tc.wantWhy + if want == nil { + want = []string{} + } + if cls.EffectiveClass != tc.want || !slices.Equal(cls.EscalationReasons, want) { + t.Fatalf("got %s %q, want %s %q", cls.EffectiveClass, cls.EscalationReasons, tc.want, want) + } + }) + } +} + +func TestParseProfileSizeAndKind(t *testing.T) { + raw, err := json.Marshal(validCompliantK3Profile()) + if err != nil { + t.Fatal(err) + } + if _, err := ParseProfile(nil); !errors.Is(err, ErrInvalidProfile) || !strings.Contains(err.Error(), "size out of bounds") { + t.Fatalf("empty profile: %v", err) + } + exact := append(append([]byte(nil), raw...), bytes.Repeat([]byte(" "), maxProfileBytes-len(raw))...) + if _, err := ParseProfile(exact); err != nil { + t.Fatalf("profile of exactly %d bytes rejected: %v", maxProfileBytes, err) + } + if _, err := ParseProfile(append(exact, ' ')); err == nil || !strings.Contains(err.Error(), "size out of bounds") { + t.Fatalf("profile of %d bytes: %v", maxProfileBytes+1, err) + } + for name, edit := range map[string]func(*CandidateProfile){ + "wrong kind": func(p *CandidateProfile) { p.Kind = "Other" }, + "wrong apiVersion": func(p *CandidateProfile) { p.APIVersion = "other/v1" }, + } { + p := validCompliantK3Profile() + edit(&p) + data, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + if _, err := ParseProfile(data); err == nil || !strings.Contains(err.Error(), "expected apiVersion") { + t.Errorf("%s: %v", name, err) + } + } +} + +// k1Profile and k2Profile are deploy candidates whose effective class is K1 +// (reversible docs) and K2 (non-AI production code). +func k1Profile() CandidateProfile { + p := validCompliantK3Profile() + p.Candidate.DeclaredClass = ClassK1 + p.Candidate.ReversibleConfigOnly = true + p.Candidate.ChangedPaths = []string{"docs/a.md"} + return p +} + +func k2Profile() CandidateProfile { + p := validCompliantK3Profile() + p.Candidate.DeclaredClass = ClassK2 + p.Candidate.ChangedPaths = []string{"pkg/x/y.go"} + return p +} + +func TestRedGreenRequiredFromK2(t *testing.T) { + for name, p := range map[string]CandidateProfile{"K1": k1Profile(), "K2": k2Profile()} { + p.Gates.G1.NewTestFailedBeforeFix, p.Gates.G1.NewTestPassedAfterFix = false, false + rep := evaluate(t, p) + if rep.Classification.EffectiveClass != name { + t.Fatalf("%s profile classified %s", name, rep.Classification.EffectiveClass) + } + want := StatusPass + if name == "K2" { + want = StatusFail + } + if got := checkStatus(rep, "g1_red_green_and_test_integrity"); got != want { + t.Errorf("%s without red-green: g1 %s, want %s", name, got, want) + } + } +} + +func TestMutationScoreLimits(t *testing.T) { + cases := []struct { + name string + profile CandidateProfile + killed, total int + status, ratio string + }{ + {"K2 without mutants", k2Profile(), 0, 0, StatusFail, "0"}, + {"K3 without mutants", validCompliantK3Profile(), 0, 0, StatusFail, "0"}, + {"K3 negative killed", validCompliantK3Profile(), -1, 10, StatusFail, "0"}, + {"K3 killed above total", validCompliantK3Profile(), 12, 10, StatusFail, "0"}, + {"K3 all killed", validCompliantK3Profile(), 10, 10, StatusPass, "1"}, + {"K3 at threshold", validCompliantK3Profile(), 4, 5, StatusPass, "4/5"}, + {"K3 below threshold", validCompliantK3Profile(), 79, 100, StatusFail, "79/100"}, + {"K1 half killed", k1Profile(), 5, 10, StatusPass, "1/2"}, + {"K1 all killed", k1Profile(), 10, 10, StatusPass, "1"}, + {"K1 none killed", k1Profile(), 0, 10, StatusPass, "0"}, + {"K1 without mutants", k1Profile(), 0, 0, StatusPass, "0"}, + {"K1 negative killed", k1Profile(), -1, 10, StatusPass, "0"}, + {"K1 killed above total", k1Profile(), 11, 10, StatusPass, "0"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := tc.profile + p.Gates.G2.KilledMutants, p.Gates.G2.TotalMutants = tc.killed, tc.total + rep := evaluate(t, p) + if got := checkStatus(rep, "g2_diff_mutation_and_fuzz"); got != tc.status || rep.MutationKillRatio != tc.ratio { + t.Fatalf("g2 %s ratio %s, want %s ratio %s", got, rep.MutationKillRatio, tc.status, tc.ratio) + } + }) + } +} + +func TestG3EveryConditionIsRequired(t *testing.T) { + for name, edit := range map[string]func(*G3Receipt){ + "one SAST finding": func(g *G3Receipt) { g.SASTFindings = 1 }, + "one vulnerability": func(g *G3Receipt) { g.VulnFindings = 1 }, + "no external capability": func(g *G3Receipt) { g.ExternalCapabilityGate = false }, + "delimiter-only security": func(g *G3Receipt) { g.PromptDelimiterOnlySecurity = true }, + "explanation-only approval": func(g *G3Receipt) { g.ExplanationOnlyApproval = true }, + } { + p := validCompliantK3Profile() + edit(&p.Gates.G3) + if got := checkStatus(evaluate(t, p), "g3_security_and_capability_boundary"); got != StatusFail { + t.Errorf("%s: g3 %s", name, got) + } + } + if got := checkStatus(evaluate(t, validCompliantK3Profile()), "g3_security_and_capability_boundary"); got != StatusPass { + t.Fatalf("clean G3 receipt: %s", got) + } +} + +func TestUnapprovedK4WithoutTestTamperingTripsNoKillSwitch(t *testing.T) { + p := validCompliantK3Profile() + p.Candidate.ChangedPaths = append(p.Candidate.ChangedPaths, ".github/x.json") + rep := evaluate(t, p) + if checkStatus(rep, "k4_human_boundary") != StatusFail || rep.KillSwitchLevel != KillSwitchNone || len(rep.KillSwitchReasons) != 0 { + t.Fatalf("k4=%s killSwitch=%s %v", checkStatus(rep, "k4_human_boundary"), rep.KillSwitchLevel, rep.KillSwitchReasons) + } + p.Candidate.DeletedOrSkippedTests = 1 + if rep = evaluate(t, p); rep.KillSwitchLevel != KillSwitchL3 { + t.Fatalf("deleted test under unapproved K4: killSwitch=%s", rep.KillSwitchLevel) + } +} + +func TestRationalFieldsRejectNegativeAndMalformedValues(t *testing.T) { + for _, value := range []string{"-1", "-0.5", "abc", "1/0"} { + p := validCompliantK3Profile() + p.Gates.G4.CanaryStepPercent = value + if _, err := EvaluateCandidate(p); !errors.Is(err, ErrInvalidProfile) { + t.Errorf("canaryStepPercent %q: %v", value, err) + } + } + for _, value := range []string{"1", "1/1", "0"} { + p := validCompliantK3Profile() + p.VOI.TestSensitivity = value + p.VOI.TestSpecificity = "3/4" + if value == "0" { + // A test that never fires is degenerate; only the bound is checked here. + p.VOI.TestSpecificity = "1/2" + } + if _, err := EvaluateCandidate(p); err != nil { + t.Errorf("testSensitivity %q rejected: %v", value, err) + } + } + p := validCompliantK3Profile() + p.VOI.TestSensitivity = "11/10" + if _, err := EvaluateCandidate(p); !errors.Is(err, ErrInvalidProfile) { + t.Errorf("testSensitivity above 1 accepted: %v", err) + } +} diff --git a/pkg/rsi/decision_limits_test.go b/pkg/rsi/decision_limits_test.go new file mode 100644 index 0000000..ffab111 --- /dev/null +++ b/pkg/rsi/decision_limits_test.go @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package rsi + +import ( + "errors" + "math" + "strconv" + "testing" +) + +func baseVOI() VOIInput { return *validCompliantK3Profile().VOI } + +func TestVOIInputBounds(t *testing.T) { + for name, edit := range map[string]func(*VOIInput){ + "prior zero": func(v *VOIInput) { v.PriorSuccessProb = "0" }, + "prior one": func(v *VOIInput) { v.PriorSuccessProb = "1" }, + "test never fires": func(v *VOIInput) { v.TestSensitivity, v.TestSpecificity = "0", "1" }, + "test always fires": func(v *VOIInput) { v.TestSensitivity, v.TestSpecificity = "1", "0" }, + } { + in := baseVOI() + edit(&in) + if _, err := EvaluateVOI(in); !errors.Is(err, ErrInvalidProfile) { + t.Errorf("%s: %v", name, err) + } + } +} + +func TestVOIDecisionBoundaries(t *testing.T) { + // Launching is worth exactly as much as not acting: no reason to launch. + tie := VOIInput{PriorSuccessProb: "1/2", SuccessNetPayoffEUR: "100", FailureLossEUR: "100", NoActionPayoffEUR: "0", + TestCostEUR: "0", TestSensitivity: "4/5", TestSpecificity: "3/4", HardLossCapEUR: "1000"} + if r, err := EvaluateVOI(tie); err != nil || r.NoTestBestAction != "do_not_launch" { + t.Fatalf("launch EV equal to no action: %+v %v", r, err) + } + // With no possible loss the test cannot change the decision: zero net EVSI + // keeps the no-test action. + free := VOIInput{PriorSuccessProb: "1/2", SuccessNetPayoffEUR: "100", FailureLossEUR: "0", NoActionPayoffEUR: "0", + TestCostEUR: "0", TestSensitivity: "4/5", TestSpecificity: "3/4", HardLossCapEUR: "1000"} + if r, err := EvaluateVOI(free); err != nil || r.NetIncrementalEVEUR != "0" || r.RecommendedAction != "launch_now" { + t.Fatalf("zero net EVSI: %+v %v", r, err) + } + atCap := baseVOI() + atCap.HardLossCapEUR = atCap.FailureLossEUR + if r, err := EvaluateVOI(atCap); err != nil || !r.FeasibleUnderLossCap { + t.Fatalf("loss equal to the cap: %+v %v", r, err) + } + if r, err := EvaluateVOI(VOIInput{PriorSuccessProb: "2/5", SuccessNetPayoffEUR: "120000", FailureLossEUR: "60000", NoActionPayoffEUR: "0", + TestCostEUR: "4000", TestSensitivity: "4/5", TestSpecificity: "3/4", HardLossCapEUR: "80000", CorrelatedLLMRunsCount: 0}); err != nil || !r.ExtremizationRejected { + t.Fatalf("zero correlated runs: %+v %v", r, err) + } +} + +func TestVOICheckVerdicts(t *testing.T) { + infeasible := validCompliantK3Profile() + infeasible.VOI.HardLossCapEUR = "50000" + rep := evaluate(t, infeasible) + if rep.VOI.RecommendedAction != "do_not_launch_or_test_until_loss_restructured" || checkStatus(rep, "c20_voi_and_loss_cap") != StatusPass { + t.Fatalf("restructure recommendation over the cap: action=%s c20=%s", rep.VOI.RecommendedAction, checkStatus(rep, "c20_voi_and_loss_cap")) + } + extremized := validCompliantK3Profile() + extremized.VOI.CorrelatedLLMRunsCount = -1 + if got := checkStatus(evaluate(t, extremized), "c20_voi_and_loss_cap"); got != StatusFail { + t.Fatalf("extremized correlated runs: c20 %s", got) + } +} + +func basePassK() PassKInput { return *validCompliantK3Profile().PassK } + +func TestPassKInputBounds(t *testing.T) { + bad := map[string]func(*PassKInput){ + "k zero": func(in *PassKInput) { in.K = 0 }, + "no tasks": func(in *PassKInput) { in.Tasks = nil }, + "fewer trials than k": func(in *PassKInput) { in.Tasks[0] = TaskTrialCount{TaskID: "t", Trials: 3, Successes: 1} }, + "negative successes": func(in *PassKInput) { in.Tasks[0] = TaskTrialCount{TaskID: "t", Trials: 5, Successes: -1} }, + "successes above trials": func(in *PassKInput) { in.Tasks[0] = TaskTrialCount{TaskID: "t", Trials: 5, Successes: 6} }, + "no accepted tasks": func(in *PassKInput) { in.AcceptedTasks, in.AttemptedTasks = 0, 5 }, + "fewer attempted than ok": func(in *PassKInput) { in.AcceptedTasks, in.AttemptedTasks = 5, 3 }, + } + for name, edit := range bad { + in := basePassK() + in.Tasks = append([]TaskTrialCount(nil), in.Tasks...) + edit(&in) + if _, err := EvaluatePassK(in); !errors.Is(err, ErrInvalidProfile) { + t.Errorf("%s: %v", name, err) + } + } + good := basePassK() + good.Tasks = []TaskTrialCount{{TaskID: "t", Trials: 4, Successes: 2}} + good.AcceptedTasks, good.AttemptedTasks = 5, 5 + r, err := EvaluatePassK(good) + if err != nil || r.UnbiasedCombinatorialPassK != "0" || r.UnbiasedCombinatorialPassAt != "1" || r.AcceptanceRate != "1" { + t.Fatalf("trials equal to k: %+v %v", r, err) + } +} + +func baseExperiment() ExperimentInput { return *validCompliantK3Profile().Experiment } + +func TestExperimentInputBounds(t *testing.T) { + bad := map[string]func(*ExperimentInput){ + "no control units": func(in *ExperimentInput) { in.ControlAssignedUnits = 0 }, + "no treatment units": func(in *ExperimentInput) { in.TreatmentAssignedUnits = 0 }, + "one cluster per arm": func(in *ExperimentInput) { in.ClustersPerArm = 1 }, + "empty clusters": func(in *ExperimentInput) { in.UnitsPerCluster = 0 }, + "zero standard error": func(in *ExperimentInput) { in.DiffStandardError = "0" }, + "bad standard error": func(in *ExperimentInput) { in.DiffStandardError = "x" }, + "zero margin": func(in *ExperimentInput) { in.NonInferiorityMargin = "0" }, + "bad margin": func(in *ExperimentInput) { in.NonInferiorityMargin = "x" }, + "negative correlation": func(in *ExperimentInput) { in.IntraclusterCorrelation = "-0.1" }, + } + for name, edit := range bad { + in := baseExperiment() + edit(&in) + if _, err := EvaluateExperiment(in); !errors.Is(err, ErrInvalidProfile) { + t.Errorf("%s: %v", name, err) + } + } + minimal := baseExperiment() + minimal.ControlAssignedUnits, minimal.TreatmentAssignedUnits, minimal.ClustersPerArm, minimal.UnitsPerCluster = 1, 1, 2, 1 + if _, err := EvaluateExperiment(minimal); err != nil { + t.Fatalf("smallest valid design rejected: %v", err) + } +} + +func TestExperimentDecisionBoundaries(t *testing.T) { + run := func(edit func(*ExperimentInput)) ExperimentReport { + t.Helper() + in := baseExperiment() + edit(&in) + r, err := EvaluateExperiment(in) + if err != nil { + t.Fatal(err) + } + return r + } + // chi^2 = d^2/N equals the 10.8276 critical value exactly: d = 54138, + // N = 270690000. One unit less imbalance stays below it. + if r := run(func(in *ExperimentInput) { in.ControlAssignedUnits, in.TreatmentAssignedUnits = 135372069, 135317931 }); !r.SRMDetected { + t.Fatalf("SRM at the critical value not detected: chi2=%s", r.SRMChiSquare) + } + if r := run(func(in *ExperimentInput) { in.ControlAssignedUnits, in.TreatmentAssignedUnits = 135372068, 135317932 }); r.SRMDetected { + t.Fatalf("SRM detected below the critical value: chi2=%s", r.SRMChiSquare) + } + // Two clusters per arm leave df = 2, where the two-sided 5% critical t is + // 4.30: t = 3.845 (p = 0.061) is not significant, t = 4.5 (p = 0.046) is. + for z, want := range map[string]bool{"3.845": false, "-3.845": false, "4.5": true, "-4.5": true} { + r := run(func(in *ExperimentInput) { in.ClustersPerArm, in.UnitsPerCluster, in.NaiveZStat = 2, 1, z }) + if r.ClusterSignificantAt005 != want { + t.Errorf("df=2 t=%s: significant=%t, want %t", z, r.ClusterSignificantAt005, want) + } + } + // The decision is inclusive at the critical value itself: for df = 2 the + // Cornish-Fisher expansion z + g1/d + g2/d^2 of t(0.975, df). + z, d := 1.959963984540054, 2.0 + g1 := (z*z*z + z) / 4.0 + g2 := (5*math.Pow(z, 5) + 16*z*z*z + 3*z) / 96.0 + crit := z + g1/d + g2/(d*d) + if r := run(func(in *ExperimentInput) { + in.ClustersPerArm, in.UnitsPerCluster, in.NaiveZStat = 2, 1, strconv.FormatFloat(crit, 'g', -1, 64) + }); !r.ClusterSignificantAt005 { + t.Errorf("t exactly at the df=2 critical value %v is not significant", crit) + } + // DEFF = 1 + (4 - 1) * 1 = 4 halves a naive z of 2. + if r := run(func(in *ExperimentInput) { in.UnitsPerCluster, in.IntraclusterCorrelation, in.NaiveZStat = 4, "1", "2" }); r.ClusterAdjustedTStat != "1.0000" || r.DesignEffectDEFF != "4.0000" { + t.Fatalf("cluster-adjusted t: %s deff %s", r.ClusterAdjustedTStat, r.DesignEffectDEFF) + } + // One-sided alpha = 0.05 bounds are inclusive at z = 1.6448536. + if r := run(func(in *ExperimentInput) { + in.ObservedDiff, in.NonInferiorityMargin, in.DiffStandardError = "0.6448536", "1", "1" + }); !r.NonInferiorityProvenAt005 { + t.Fatalf("non-inferiority at the bound: z=%s", r.NonInferiorityZ) + } + if r := run(func(in *ExperimentInput) { + in.ObservedDiff, in.NonInferiorityMargin, in.DiffStandardError = "0", "1.6448536", "1" + }); !r.TOSTEquivalenceProven005 { + t.Fatalf("TOST at both bounds: zNI=%s zUpper=%s", r.NonInferiorityZ, r.TOSTUpperZ) + } + if r := run(func(in *ExperimentInput) { + in.ObservedDiff, in.NonInferiorityMargin, in.DiffStandardError = "0.5", "2", "1" + }); r.TOSTEquivalenceProven005 || !r.NonInferiorityProvenAt005 { + t.Fatalf("upper bound not met: tost=%t ni=%t", r.TOSTEquivalenceProven005, r.NonInferiorityProvenAt005) + } +} + +func TestExperimentCheckNeedsBalanceAndValidCUPED(t *testing.T) { + srm := validCompliantK3Profile() + srm.Experiment.ControlAssignedUnits, srm.Experiment.TreatmentAssignedUnits = 11000, 9000 + cuped := validCompliantK3Profile() + cuped.Experiment.HasPostTreatmentCovariate = true + for name, p := range map[string]CandidateProfile{"sample ratio mismatch": srm, "post-treatment covariate": cuped} { + if got := checkStatus(evaluate(t, p), "c19_experiment_srm_deff_tost"); got != StatusFail { + t.Errorf("%s: c19 %s", name, got) + } + } +} + +func TestDiscoveryCountBounds(t *testing.T) { + base := *validCompliantK3Profile().Discovery + for _, c := range []struct{ people, accounts int }{{5, -1}, {3, 5}, {-1, -1}} { + in := base + in.TotalInterviewedPeople, in.IndependentAccounts = c.people, c.accounts + if _, err := EvaluateDiscovery(in); !errors.Is(err, ErrInvalidProfile) { + t.Errorf("people=%d accounts=%d: %v", c.people, c.accounts, err) + } + } + for _, c := range []struct { + people, accounts int + inflation bool + }{{0, 0, false}, {5, 0, true}, {3, 3, false}, {6, 5, false}, {6, 4, true}} { + in := base + in.TotalInterviewedPeople, in.IndependentAccounts = c.people, c.accounts + r, err := EvaluateDiscovery(in) + if err != nil || r.AccountInflationDetected != c.inflation { + t.Errorf("people=%d accounts=%d: inflation=%t err=%v, want %t", c.people, c.accounts, r.AccountInflationDetected, err, c.inflation) + } + } +} diff --git a/pkg/rsi/gate_limits_test.go b/pkg/rsi/gate_limits_test.go new file mode 100644 index 0000000..4516d53 --- /dev/null +++ b/pkg/rsi/gate_limits_test.go @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package rsi + +import ( + "strings" + "testing" +) + +func TestTouchesDeploySurface(t *testing.T) { + cases := map[string]bool{ + "pkg/rsi/rsi.go": false, + "docs/deploy.md": false, + "pkg/deployer/x.go": false, + "cmd/cloudring_core_tool/x": false, + "deploy/kubernetes/x.yaml": true, + "a/IAC/b.tf": true, + "cloudring_core": true, + ".gitmodules": true, + ".github/workflows": true, + ".github/workflows/ci.yml": true, + ".github/workflows-note.md": false, + "": true, + " ": true, + ".": true, + "/abs/x.go": true, + "pkg/../x.go": true, + "pkg/x%41.go": true, + `pkg\x.go`: true, + "pkg/x\x00.go": true, + " deploy/x.yaml ": true, + "reference/x/Deploy/k8s.yml": true, + } + for raw, want := range cases { + if got := touchesDeploySurface(raw); got != want { + t.Errorf("touchesDeploySurface(%q) = %t, want %t", raw, got, want) + } + } +} + +func TestG4CanaryLimits(t *testing.T) { + cases := []struct { + name string + edit func(*G4Receipt) + pass bool + }{ + {"compliant", func(*G4Receipt) {}, true}, + {"two active canaries", func(g *G4Receipt) { g.ActiveCanaries = 2 }, false}, + {"step above 10%", func(g *G4Receipt) { g.CanaryStepPercent = "10.1" }, false}, + {"max weight at 50%", func(g *G4Receipt) { g.CanaryMaxPercent = "50" }, true}, + {"max weight above 50%", func(g *G4Receipt) { g.CanaryMaxPercent = "50.1" }, false}, + {"success at 99%", func(g *G4Receipt) { g.RequestSuccessRatePercent = "99" }, true}, + {"success below 99%", func(g *G4Receipt) { g.RequestSuccessRatePercent = "98.9" }, false}, + {"p99 at 500 ms", func(g *G4Receipt) { g.LatencyP99Ms = "500" }, true}, + {"p99 above 500 ms", func(g *G4Receipt) { g.LatencyP99Ms = "500.1" }, false}, + {"burn at 10% of budget", func(g *G4Receipt) { g.CanaryDurationMinutes = "43.2" }, true}, + {"burn above 10% of budget", func(g *G4Receipt) { g.CanaryDurationMinutes = "43.3" }, false}, + {"two failures in 24h", func(g *G4Receipt) { g.CanaryFailuresLast24h = 2 }, true}, + {"three failures in 24h", func(g *G4Receipt) { g.CanaryFailuresLast24h = 3 }, false}, + {"stateful without downgrade proof", func(g *G4Receipt) { g.DowngradeContinuityVerified = false }, false}, + {"stateless without downgrade proof", func(g *G4Receipt) { + g.TouchesPersistentState, g.DowngradeContinuityVerified = false, false + }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := validCompliantK3Profile() + p.Gates.G4.CanaryStepPercent = "10" + tc.edit(&p.Gates.G4) + rep := evaluate(t, p) + want := StatusFail + if tc.pass { + want = StatusPass + } + if got := checkStatus(rep, "g4_canary_slo_and_stateful_downgrade"); got != want { + t.Fatalf("g4 status %s, want %s: %+v", got, want, checkOf(rep, "g4_canary_slo_and_stateful_downgrade")) + } + wantKill := KillSwitchNone + if p.Gates.G4.CanaryFailuresLast24h >= 3 { + wantKill = KillSwitchL1 + } + if rep.KillSwitchLevel != wantKill { + t.Fatalf("kill switch %s, want %s", rep.KillSwitchLevel, wantKill) + } + }) + } +} + +func TestG6RejectsInvalidCounts(t *testing.T) { + cases := map[string]G6Receipt{ + "negative AI failures": {AIFailedDeployments: -1, AITotalDeployments: 20, HumanFailedDeployments: 2, HumanTotalDeployments: 20}, + "negative AI total": {AIFailedDeployments: 0, AITotalDeployments: -1, HumanFailedDeployments: 2, HumanTotalDeployments: 20}, + "AI failures above total": {AIFailedDeployments: 3, AITotalDeployments: 2, HumanFailedDeployments: 2, HumanTotalDeployments: 2}, + "negative human failures": {AIFailedDeployments: 1, AITotalDeployments: 20, HumanFailedDeployments: -1, HumanTotalDeployments: 20}, + "negative human total": {AIFailedDeployments: 1, AITotalDeployments: 20, HumanFailedDeployments: 0, HumanTotalDeployments: -1}, + "human failures above total": {AIFailedDeployments: 1, AITotalDeployments: 20, HumanFailedDeployments: 3, HumanTotalDeployments: 2}, + "cold start with bad history": {AIFailedDeployments: 0, AITotalDeployments: 0, HumanFailedDeployments: 3, HumanTotalDeployments: 2}, + } + for name, g6 := range cases { + t.Run(name, func(t *testing.T) { + p := validCompliantK3Profile() + p.Gates.G6 = g6 + rep := evaluate(t, p) + if checkStatus(rep, "g6_dora_cfr_circuit_breaker") != StatusFail || rep.AIChangeFailRate != "0" || + rep.MaxAutonomousClass != ClassK1 || rep.KillSwitchLevel != KillSwitchL1 { + t.Fatalf("invalid G6 counts were accepted: aiCFR=%s maxAuto=%s kill=%s check=%+v", + rep.AIChangeFailRate, rep.MaxAutonomousClass, rep.KillSwitchLevel, checkOf(rep, "g6_dora_cfr_circuit_breaker")) + } + }) + } +} + +func TestG7BudgetLimits(t *testing.T) { + cases := []struct { + name string + cost, median, duration string + pass bool + kill string + }{ + {"compliant", "1.80", "1.50", "12", true, KillSwitchNone}, + {"cost at cap", "10", "6", "12", true, KillSwitchNone}, + {"cost above cap", "11", "6", "12", false, KillSwitchNone}, + {"duration at cap", "1.80", "1.50", "30", true, KillSwitchNone}, + {"duration above cap", "1.80", "1.50", "30.1", false, KillSwitchNone}, + {"cost at twice median", "3", "1.50", "12", true, KillSwitchNone}, + {"cost above twice median", "3.01", "1.50", "12", false, KillSwitchL2}, + {"zero median", "0", "0", "12", false, KillSwitchL2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := validCompliantK3Profile() + p.Gates.G7.TaskCostUSD, p.Gates.G7.MedianTaskCostUSD, p.Gates.G7.TaskDurationMinutes = tc.cost, tc.median, tc.duration + rep := evaluate(t, p) + want := StatusFail + if tc.pass { + want = StatusPass + } + if got := checkStatus(rep, "g7_budget_and_cross_family_overseer"); got != want || rep.KillSwitchLevel != tc.kill { + t.Fatalf("g7 status %s (want %s) kill %s (want %s): %+v", got, want, rep.KillSwitchLevel, tc.kill, checkOf(rep, "g7_budget_and_cross_family_overseer")) + } + }) + } + for _, fam := range [][2]string{{"", "family-b"}, {"family-a", ""}, {"family-a", " FAMILY-A "}} { + p := validCompliantK3Profile() + p.Gates.G7.ExecutorModelFamily, p.Gates.G7.OverseerModelFamily = fam[0], fam[1] + if got := checkStatus(evaluate(t, p), "g7_budget_and_cross_family_overseer"); got != StatusFail { + t.Errorf("executor %q overseer %q: g7 %s, want %s", fam[0], fam[1], got, StatusFail) + } + } +} + +func TestLKGCheckSummary(t *testing.T) { + rep := evaluate(t, validCompliantK3Profile()) + if c := checkOf(rep, "s1_s6_lkg_promotion"); c.Status != StatusPass || !strings.HasPrefix(c.Summary, "S1-S6 hold: pinnedSlots=2 soak=30min") { + t.Fatalf("valid LKG receipt: %+v", c) + } + p := validCompliantK3Profile() + p.LKG.SoakMinutes = MinSoakMinutes - 1 + if c := checkOf(evaluate(t, p), "s1_s6_lkg_promotion"); c.Status != StatusFail || !strings.HasPrefix(c.Summary, "S2: ") { + t.Fatalf("short soak: %+v", c) + } +} diff --git a/pkg/rsi/rsi.go b/pkg/rsi/rsi.go index f2d20d8..24e6435 100644 --- a/pkg/rsi/rsi.go +++ b/pkg/rsi/rsi.go @@ -12,13 +12,18 @@ package rsi import ( + "encoding/json" "errors" "fmt" "math" "math/big" "path" + "reflect" + "slices" "sort" "strings" + "unicode" + "unicode/utf8" "github.com/opencloudtech/CloudRING/internal/strictjson" ) @@ -37,14 +42,29 @@ const ( VerdictPass = "pass" VerdictFail = "fail" - StatusPass = "pass" - StatusFail = "fail" + StatusPass = "pass" + StatusFail = "fail" + StatusNotApplicable = "not_applicable" + + // AdmissionMerge admits code into a branch that deploys nothing; + // AdmissionDeploy admits code into a running environment. + AdmissionMerge = "merge" + AdmissionDeploy = "deploy" + + // OriginAutonomousLoop is a change produced by the metered unattended loop; + // OriginOperatorDirected is a change whose task a human operator directed. + OriginAutonomousLoop = "autonomous_loop" + OriginOperatorDirected = "operator_directed" KillSwitchNone = "none" KillSwitchL1 = "L1_suspend_gitops_canary" KillSwitchL2 = "L2_revoke_gateway_openbao_leases" KillSwitchL3 = "L3_vap_ai_freeze_deny" + // MinPinnedLKGSlots and MinSoakMinutes bound S2 of the LKG invariants. + MinPinnedLKGSlots = 2 + MinSoakMinutes = 30 + maxProfileBytes = 1 << 20 ) @@ -72,10 +92,13 @@ type CandidateProfile struct { Kind string `json:"kind"` Candidate CandidateMetadata `json:"candidate"` Gates GateReceipts `json:"gates"` - VOI *VOIInput `json:"voi,omitempty"` - PassK *PassKInput `json:"passK,omitempty"` - Experiment *ExperimentInput `json:"experiment,omitempty"` - Discovery *DiscoveryInput `json:"discovery,omitempty"` + // LKG is the S1-S6 receipt of the deployment slot the candidate would be + // promoted into; deploy admission requires it. + LKG *LKGReceipt `json:"lkg,omitempty"` + VOI *VOIInput `json:"voi,omitempty"` + PassK *PassKInput `json:"passK,omitempty"` + Experiment *ExperimentInput `json:"experiment,omitempty"` + Discovery *DiscoveryInput `json:"discovery,omitempty"` } // CandidateMetadata identifies the diff and declared target class. @@ -89,6 +112,24 @@ type CandidateMetadata struct { ModifiedExistingTests bool `json:"modifiedExistingTests"` ModifiedTestdata bool `json:"modifiedTestdata"` HumanK4ApprovalReference string `json:"humanK4ApprovalReference"` + // Admission is "merge" or "deploy"; empty means "deploy", so every gate applies. + Admission string `json:"admission"` + // TargetRepository and TargetBranch name where a merge admission lands + // (owner/name and branch); merge admission requires a known non-GitOps target. + TargetRepository string `json:"targetRepository"` + TargetBranch string `json:"targetBranch"` + // DeploymentTargets names the running environments this admission changes. + DeploymentTargets []string `json:"deploymentTargets"` + // Origin is "autonomous_loop" or "operator_directed"; empty means "autonomous_loop". + Origin string `json:"origin"` + // OperatorReference is the auditable record of the operator who directed + // the task; required for an operator_directed origin. + OperatorReference string `json:"operatorReference"` + // EvaluatorBootstrap records that the merge base has no evaluator, so no + // frozen predecessor can judge the candidate. It escalates to K4 and stands + // in for the frozen-hash check only at merge admission: once merged, the + // evaluator exists and every deployment is judged by a frozen copy. + EvaluatorBootstrap bool `json:"evaluatorBootstrap"` } // GateReceipts holds externally measured G1-G7 evidence (never self-reported by the candidate). @@ -115,6 +156,12 @@ type G1Receipt struct { } // G2Receipt captures diff-based mutation testing on changed non-arid lines and fuzzing. +// For guardrail code (pkg/rsi, pkg/sliceoracles, pkg/safepush, cmd/cloudring-safepush +// and the rsi command of cmd/cloudringctl) cmd/cloudring-mutation measures it: the +// gremlins v0.6.0 mutants (default mutators plus INVERT_LOGICAL) on lines changed against +// the merge-base, each run alone with go test on its package. KilledMutants counts failing +// runs; TotalMutants counts every mutant that builds and passes go vet, less equivalent +// mutants proven in the evidence (roadmap/EXECUTION_CONTRACT.md). type G2Receipt struct { KilledMutants int `json:"killedMutants"` TotalMutants int `json:"totalMutants"` @@ -157,6 +204,19 @@ type G6Receipt struct { HumanTotalDeployments int `json:"humanTotalDeployments"` } +// LKGReceipt captures the six self-evolution liveness invariants (S1-S6) of +// the last-known-good slot a deployed candidate would be promoted into. +type LKGReceipt struct { + ImmutableBootloaderVerified bool `json:"immutableBootloaderVerified"` + PinnedLKGSlots int `json:"pinnedLkgSlots"` + SoakMinutes int `json:"soakMinutes"` + WebhookBypassVerified bool `json:"webhookBypassVerified"` + PreMigrationSnapshotVerified bool `json:"preMigrationSnapshotVerified"` + MetaRSISelfCheckPassed bool `json:"metaRsiSelfCheckPassed"` + CandidateSHA string `json:"candidateSha"` + QuarantinedSHAs []string `json:"quarantinedShas"` +} + // G7Receipt captures compute budget, wall-clock duration, and cross-family overseer isolation. type G7Receipt struct { TaskCostUSD string `json:"taskCostUsd"` @@ -311,10 +371,12 @@ type Report struct { SchemaVersion string `json:"schemaVersion"` APIVersion string `json:"apiVersion"` CandidateID string `json:"candidateId"` + Admission string `json:"admission"` Classification Classification `json:"classification"` Checks []CheckResult `json:"checks"` Passed int `json:"passed"` Failed int `json:"failed"` + NotApplicable int `json:"notApplicable"` MaxAutonomousClass string `json:"maxAutonomousClass"` KillSwitchLevel string `json:"killSwitchLevel"` KillSwitchReasons []string `json:"killSwitchReasons"` @@ -338,6 +400,13 @@ func ParseProfile(raw []byte) (CandidateProfile, error) { if err := strictjson.DecodeExact(raw, &profile); err != nil { return CandidateProfile{}, fmt.Errorf("%w: %v", ErrInvalidProfile, err) } + var tree any + if err := json.Unmarshal(raw, &tree); err != nil { + return CandidateProfile{}, fmt.Errorf("%w: %v", ErrInvalidProfile, err) + } + if err := checkExactKeys(tree, reflect.TypeFor[CandidateProfile](), "profile"); err != nil { + return CandidateProfile{}, fmt.Errorf("%w: %v", ErrInvalidProfile, err) + } if profile.APIVersion != ProfileAPIVersion || profile.Kind != ProfileKind { return CandidateProfile{}, fmt.Errorf("%w: expected apiVersion=%q kind=%q", ErrInvalidProfile, ProfileAPIVersion, ProfileKind) } @@ -350,9 +419,101 @@ func ParseProfile(raw []byte) (CandidateProfile, error) { if len(profile.Candidate.ChangedPaths) == 0 && !profile.Candidate.ReadOnlyOnly { return CandidateProfile{}, fmt.Errorf("%w: changedPaths required for non-K0 candidate", ErrInvalidProfile) } + switch profile.Candidate.Admission { + case "", AdmissionMerge, AdmissionDeploy: + default: + return CandidateProfile{}, fmt.Errorf("%w: invalid admission %q", ErrInvalidProfile, profile.Candidate.Admission) + } + switch profile.Candidate.Origin { + case "", OriginAutonomousLoop, OriginOperatorDirected: + default: + return CandidateProfile{}, fmt.Errorf("%w: invalid origin %q", ErrInvalidProfile, profile.Candidate.Origin) + } + if ref := profile.Candidate.OperatorReference; ref != "" && !validReference(ref) { + return CandidateProfile{}, fmt.Errorf("%w: candidate.operatorReference %q names no record", ErrInvalidProfile, ref) + } + if ref := profile.Candidate.HumanK4ApprovalReference; ref != "" && !validReference(ref) { + return CandidateProfile{}, fmt.Errorf("%w: candidate.humanK4ApprovalReference %q names no record", ErrInvalidProfile, ref) + } return profile, nil } +// checkExactKeys rejects every object key that is not exactly the JSON name +// of a field of the struct it decodes into, at any depth. encoding/json maps +// keys onto fields case-insensitively, so a case variant of a known key would +// otherwise decode into that field while other readers of the same profile +// see a different value. +func checkExactKeys(v any, t reflect.Type, at string) error { + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + switch t.Kind() { + case reflect.Struct: + obj, ok := v.(map[string]any) + if !ok { + return nil // the strict decoder already rejected a type mismatch + } + fields := make(map[string]reflect.Type, t.NumField()) + for i := range t.NumField() { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + if name != "" && name != "-" { + fields[name] = t.Field(i).Type + } + } + keys := make([]string, 0, len(obj)) + for key := range obj { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + fieldType, known := fields[key] + if !known { + return fmt.Errorf("%s: unknown or case-variant field %q", at, key) + } + if err := checkExactKeys(obj[key], fieldType, at+"."+key); err != nil { + return err + } + } + case reflect.Slice, reflect.Array: + items, ok := v.([]any) + if !ok { + return nil + } + for i, item := range items { + if err := checkExactKeys(item, t.Elem(), fmt.Sprintf("%s[%d]", at, i)); err != nil { + return err + } + } + } + return nil +} + +// validReference reports whether s can name an auditable record. It rejects +// control, format, private-use, surrogate, variation-selector and other +// default-ignorable characters (such as the Hangul fillers) and blank letters, +// and requires a visible letter or digit, so invisible text never counts as a +// reference. +func validReference(s string) bool { + named := false + for _, r := range s { + if r == utf8.RuneError || unicode.Is(unicode.C, r) || blankLetters[r] || + unicode.Is(unicode.Other_Default_Ignorable_Code_Point, r) || unicode.Is(unicode.Variation_Selector, r) { + return false + } + if unicode.IsGraphic(r) && (unicode.IsLetter(r) || unicode.IsDigit(r)) { + named = true + } + } + return named +} + +// blankLetters are letters (category Lo) that render as empty space but are +// not default-ignorable, so no Unicode property excludes them. +var blankLetters = map[rune]bool{ + 0x13441: true, // EGYPTIAN HIEROGLYPH FULL BLANK + 0x13442: true, // EGYPTIAN HIEROGLYPH HALF BLANK +} + func validClass(c string) bool { switch c { case ClassK0, ClassK1, ClassK2, ClassK3, ClassK4: @@ -457,6 +618,12 @@ func ClassifyCandidate(meta CandidateMetadata) (Classification, error) { touchesK4 = true reasons = append(reasons, "modified_testdata_fixtures_requires_k4") } + if meta.EvaluatorBootstrap { + // No frozen predecessor exists to judge the evaluator's own introduction, + // so only a human can admit it. + touchesK4 = true + reasons = append(reasons, "evaluator_bootstrap_requires_k4") + } if touchesK4 { effective = ClassK4 } @@ -521,9 +688,126 @@ func parseProbabilityRat(raw string, field string) (*big.Rat, error) { return r, nil } -// EvaluateCandidate runs the complete G1-G7 governance evaluation and any -// attached decision-science checks (VOI, PassK, Experiment, Discovery). +// deploySurfaceSegments name repository paths whose change takes effect on a +// running environment: deployment manifests, infrastructure code, and gitlinks +// (submodules), which pin one repository's code into another that deploys it. +var deploySurfaceSegments = map[string]bool{"deploy": true, "iac": true, ".gitmodules": true, "cloudring_core": true} + +// touchesDeploySurface reports whether a changed path takes effect on a running +// environment once merged: a deploy, iac or gitlink path segment anywhere, or a +// workflow that runs on the target branch. A path that cannot be normalized +// counts as deploy surface, so it cannot support a merge-only claim. +func touchesDeploySurface(raw string) bool { + trimmed := strings.TrimSpace(raw) + if trimmed == "" || strings.ContainsAny(trimmed, "\x00\\%") || strings.Contains(trimmed, "..") { + return true + } + lower := strings.ToLower(path.Clean(trimmed)) + if lower == "." || strings.HasPrefix(lower, "/") || lower == ".github/workflows" || strings.HasPrefix(lower, ".github/workflows/") { + return true + } + for _, seg := range strings.Split(lower, "/") { + if deploySurfaceSegments[seg] { + return true + } + } + return false +} + +type mergeTarget struct{ repository, branch string } + +// mergeOnlyTargets lists the repository branches that no environment tracks as +// a GitOps source, so merging into them deploys nothing. Checked 2026-11-09: +// the hub's only Flux GitRepository is opencloudtech/cloudring-enterprise main; +// public opencloudtech/CloudRING code reaches an environment only through a +// gitlink bump in cloudring-enterprise, which is a deploy admission there. +var mergeOnlyTargets = map[mergeTarget]bool{{"opencloudtech/CloudRING", "main"}: true} + +// admissionScope derives the admission that governs gate applicability from +// the declared admission and origin and the evidence that can contradict them. +// A merge-only claim must name a known non-GitOps target branch and carry no +// deployment targets, no deploy-surface path, no canary receipt and no LKG +// receipt; an operator-directed origin must name its operator. Any +// contradiction resolves to deploy admission from the autonomous loop, where +// every gate applies, and is returned so the caller fails the candidate. +func admissionScope(profile CandidateProfile) (string, bool, []string) { + meta := profile.Candidate + admission := AdmissionDeploy + var contradictions []string + switch meta.Admission { + case "", AdmissionDeploy: + case AdmissionMerge: + admission = AdmissionMerge + if !mergeOnlyTargets[mergeTarget{meta.TargetRepository, meta.TargetBranch}] { + contradictions = append(contradictions, fmt.Sprintf("merge_admission_target_not_known_non_gitops:%q@%q", meta.TargetRepository, meta.TargetBranch)) + } + if len(meta.DeploymentTargets) > 0 { + contradictions = append(contradictions, "merge_admission_with_deployment_targets") + } + for _, raw := range meta.ChangedPaths { + if touchesDeploySurface(raw) { + contradictions = append(contradictions, fmt.Sprintf("merge_admission_touches_deploy_surface:%s", raw)) + } + } + if profile.Gates.G4 != (G4Receipt{}) { + contradictions = append(contradictions, "merge_admission_with_canary_receipt") + } + if profile.LKG != nil { + contradictions = append(contradictions, "merge_admission_with_lkg_receipt") + } + default: + contradictions = append(contradictions, fmt.Sprintf("unknown_admission:%s", meta.Admission)) + } + operatorDirected := false + switch meta.Origin { + case "", OriginAutonomousLoop: + case OriginOperatorDirected: + operatorDirected = validReference(meta.OperatorReference) + if !operatorDirected { + contradictions = append(contradictions, "operator_directed_without_operator_reference") + } + default: + contradictions = append(contradictions, fmt.Sprintf("unknown_origin:%s", meta.Origin)) + } + if len(contradictions) > 0 { + return AdmissionDeploy, false, contradictions + } + return admission, operatorDirected, nil +} + +// escalateToK4 raises a classification to K4 for a reason found during +// evaluation rather than in the changed paths. +func escalateToK4(cls Classification, reason string) Classification { + cls.EffectiveClass = ClassK4 + cls.HumanApprovalRequired = true + cls.HoldoutRequired = true + cls.EscalationReasons = append(cls.EscalationReasons, reason) + sort.Strings(cls.EscalationReasons) + cls.EscalationReasons = compactStrings(cls.EscalationReasons) + return cls +} + +// EvaluateCandidate runs the G1-G7 governance evaluation for the candidate's +// admission and any attached decision-science checks (VOI, PassK, Experiment, +// Discovery). +// +// Deploy admission (the default) applies every gate, including the S1-S6 LKG +// invariants of the receipt in profile.LKG. Merge admission admits code into a +// branch that deploys nothing: G4 and S1-S6 are not applicable there, because +// no running environment hosts a canary and nothing is promoted to an LKG slot. func EvaluateCandidate(profile CandidateProfile) (Report, error) { + return evaluateCandidate(profile, profile.LKG) +} + +// EvaluateCandidateForSlot evaluates profile for promotion into the LKG slot +// described by slot. Whenever the derived admission is deploy, including a +// merge claim that contradicts its own scope, S1-S6 are checked against slot +// and never against a receipt the profile carries about itself. +func EvaluateCandidateForSlot(profile CandidateProfile, slot LKGReceipt) (Report, error) { + return evaluateCandidate(profile, &slot) +} + +func evaluateCandidate(profile CandidateProfile, lkg *LKGReceipt) (Report, error) { cls, err := ClassifyCandidate(profile.Candidate) if err != nil { return Report{}, err @@ -534,11 +818,33 @@ func EvaluateCandidate(profile CandidateProfile) (Report, error) { killLevel := KillSwitchNone maxAutoClass := ClassK3 + admission, operatorDirected, contradictions := admissionScope(profile) + g6 := profile.Gates.G6 + g6Invalid := g6.AITotalDeployments < 0 || g6.AIFailedDeployments < 0 || g6.AIFailedDeployments > g6.AITotalDeployments || + g6.HumanTotalDeployments < 0 || g6.HumanFailedDeployments < 0 || g6.HumanFailedDeployments > g6.HumanTotalDeployments + coldStart := !g6Invalid && g6.AITotalDeployments == 0 + if admission == AdmissionDeploy && coldStart { + // The AI line has no change-fail history to bound its first deployment, + // so only a human can admit it. + cls = escalateToK4(cls, "deploy_cold_start_requires_k4") + } + scopeSummary := fmt.Sprintf("admission=%s operatorDirected=%t deploymentTargets=%d", + admission, operatorDirected, len(profile.Candidate.DeploymentTargets)) + if admission == AdmissionMerge { + scopeSummary = fmt.Sprintf("admission=merge target=%s@%s operatorDirected=%t deploymentTargets=0", + profile.Candidate.TargetRepository, profile.Candidate.TargetBranch, operatorDirected) + } + if len(contradictions) > 0 { + scopeSummary = fmt.Sprintf("declared admission=%q origin=%q contradicts evidence (%s); evaluated as deploy admission from the autonomous loop", + profile.Candidate.Admission, profile.Candidate.Origin, strings.Join(contradictions, ", ")) + } + checks = append(checks, makeCheck("admission_scope_consistency", len(contradictions) == 0, scopeSummary)) + // Gate K4: Human approval check when escalated or declared K4. k4Pass := true k4Summary := fmt.Sprintf("effective class %s within autonomous boundary", cls.EffectiveClass) if cls.EffectiveClass == ClassK4 { - if strings.TrimSpace(profile.Candidate.HumanK4ApprovalReference) == "" { + if !validReference(profile.Candidate.HumanK4ApprovalReference) { k4Pass = false k4Summary = fmt.Sprintf("effective class K4 requires explicit human approval (%s)", strings.Join(cls.EscalationReasons, ", ")) if profile.Candidate.DeletedOrSkippedTests > 0 || profile.Candidate.ModifiedExistingTests || profile.Candidate.ModifiedTestdata { @@ -551,17 +857,22 @@ func EvaluateCandidate(profile CandidateProfile) (Report, error) { } checks = append(checks, makeCheck("k4_human_boundary", k4Pass, k4Summary)) - // Gate G1: Red->Green test & evaluator hash integrity. + // Gate G1: Red->Green test & evaluator hash integrity. An evaluator + // bootstrap has no frozen predecessor to verify against, so at merge + // admission it stands in for the frozen hash; it is always K4 and admitted + // only through the human boundary above. A deployment is always judged by + // a frozen evaluator. g1 := profile.Gates.G1 - g1Pass := g1.EvaluatorFrozenHashVerified && g1.SuitePassed && !g1.RerunMaskingUsed && g1.SilentErrorSwallowing == 0 + frozenOK := g1.EvaluatorFrozenHashVerified || (profile.Candidate.EvaluatorBootstrap && admission == AdmissionMerge) + g1Pass := frozenOK && g1.SuitePassed && !g1.RerunMaskingUsed && g1.SilentErrorSwallowing == 0 if classRank(cls.EffectiveClass) >= classRank(ClassK2) { g1Pass = g1Pass && g1.NewTestFailedBeforeFix && g1.NewTestPassedAfterFix } if cls.HoldoutRequired { g1Pass = g1Pass && g1.UnseenHoldoutPassed && g1.SeededErrorRejected } - g1Summary := fmt.Sprintf("frozenHash=%t suite=%t redGreen=%t/%t holdout=%t seededError=%t rerunMask=%t silentSwallow=%d", - g1.EvaluatorFrozenHashVerified, g1.SuitePassed, g1.NewTestFailedBeforeFix, g1.NewTestPassedAfterFix, + g1Summary := fmt.Sprintf("frozenHash=%t evaluatorBootstrap=%t suite=%t redGreen=%t/%t holdout=%t seededError=%t rerunMask=%t silentSwallow=%d", + g1.EvaluatorFrozenHashVerified, profile.Candidate.EvaluatorBootstrap, g1.SuitePassed, g1.NewTestFailedBeforeFix, g1.NewTestPassedAfterFix, g1.UnseenHoldoutPassed, g1.SeededErrorRejected, g1.RerunMaskingUsed, g1.SilentErrorSwallowing) checks = append(checks, makeCheck("g1_red_green_and_test_integrity", g1Pass, g1Summary)) @@ -593,116 +904,142 @@ func EvaluateCandidate(profile CandidateProfile) (Report, error) { // Gate G4: Canary SLO burn and stateful downgrade continuity. g4 := profile.Gates.G4 - stepPct, err := parseNonNegativeRat(g4.CanaryStepPercent, "gates.g4.canaryStepPercent") - if err != nil { - return Report{}, err - } - maxPct, err := parseNonNegativeRat(g4.CanaryMaxPercent, "gates.g4.canaryMaxPercent") - if err != nil { - return Report{}, err - } - succPct, err := parseNonNegativeRat(g4.RequestSuccessRatePercent, "gates.g4.requestSuccessRatePercent") - if err != nil { - return Report{}, err - } - latMs, err := parseNonNegativeRat(g4.LatencyP99Ms, "gates.g4.latencyP99Ms") - if err != nil { - return Report{}, err - } - durMin, err := parseNonNegativeRat(g4.CanaryDurationMinutes, "gates.g4.canaryDurationMinutes") - if err != nil { - return Report{}, err - } - stepFrac := new(big.Rat).Quo(stepPct, big.NewRat(100, 1)) - maxFrac := new(big.Rat).Quo(maxPct, big.NewRat(100, 1)) - succFrac := new(big.Rat).Quo(succPct, big.NewRat(100, 1)) - // Worst-case error-minutes during one canary step = stepFrac * durMin; burn fraction = (stepFrac * durMin) / 43.2 - errMinutes := new(big.Rat).Mul(stepFrac, durMin) - burnFrac := new(big.Rat).Quo(errMinutes, sloMonthlyBudgetMin) - - g4Pass := g4.ActiveCanaries >= 1 && g4.ActiveCanaries <= 1 && - stepFrac.Cmp(maxCanaryStepPct) <= 0 && - maxFrac.Cmp(maxCanaryWeightPct) <= 0 && - succFrac.Cmp(minCanarySuccessRate) >= 0 && - latMs.Cmp(maxCanaryLatencyMs) <= 0 && - burnFrac.Cmp(maxCanarySLOBurnFrac) <= 0 && - g4.CanaryFailuresLast24h < 3 && - (!g4.TouchesPersistentState || g4.DowngradeContinuityVerified) - - if g4.CanaryFailuresLast24h >= 3 { - killLevel = escalateKillSwitch(killLevel, KillSwitchL1) - killReasons = append(killReasons, "three_or_more_canary_failures_in_24h") + burnFrac := big.NewRat(0, 1) + switch { + case admission == AdmissionMerge: + checks = append(checks, notApplicableCheck("g4_canary_slo_and_stateful_downgrade", + "merge admission: no running environment hosts a canary; G4 applies at deploy admission")) + case g4 == (G4Receipt{}): + checks = append(checks, makeCheck("g4_canary_slo_and_stateful_downgrade", false, + "deploy admission requires one live canary receipt; none supplied")) + default: + g4Pass, g4Summary, frac, err := evaluateG4(g4) + if err != nil { + return Report{}, err + } + burnFrac = frac + if g4.CanaryFailuresLast24h >= 3 { + killLevel = escalateKillSwitch(killLevel, KillSwitchL1) + killReasons = append(killReasons, "three_or_more_canary_failures_in_24h") + } + checks = append(checks, makeCheck("g4_canary_slo_and_stateful_downgrade", g4Pass, g4Summary)) } - g4Summary := fmt.Sprintf("active=%d step=%s%% max=%s%% success=%s%% p99=%sms burnFrac=%s stateful=%t downgradeOK=%t", - g4.ActiveCanaries, stepPct.FloatString(1), maxPct.FloatString(1), succPct.FloatString(2), latMs.FloatString(1), - burnFrac.FloatString(4), g4.TouchesPersistentState, g4.DowngradeContinuityVerified) - checks = append(checks, makeCheck("g4_canary_slo_and_stateful_downgrade", g4Pass, g4Summary)) - // Gate G5: Portability and EU Data Act switching drill. + // Gate G5: Portability and EU Data Act switching drill. The drill exercises + // tenant export and provider switching on a running installation; merged + // code reaches tenants only through a later deploy admission, where G5 is + // mandatory. A drill receipt supplied with a merge is still evaluated. g5 := profile.Gates.G5 - g5Pass := g5.PortabilityDrillPassed && g5.DataActVerdict == VerdictPass - g5Summary := fmt.Sprintf("portabilityDrill=%t dataActVerdict=%s", g5.PortabilityDrillPassed, g5.DataActVerdict) - checks = append(checks, makeCheck("g5_portability_and_switching_drill", g5Pass, g5Summary)) + if admission == AdmissionMerge && g5 == (G5Receipt{}) { + checks = append(checks, notApplicableCheck("g5_portability_and_switching_drill", + "merge admission: no tenant installation runs the merged code; G5 applies at deploy admission")) + } else { + g5Pass := g5.PortabilityDrillPassed && g5.DataActVerdict == VerdictPass + g5Summary := fmt.Sprintf("portabilityDrill=%t dataActVerdict=%s", g5.PortabilityDrillPassed, g5.DataActVerdict) + checks = append(checks, makeCheck("g5_portability_and_switching_drill", g5Pass, g5Summary)) + } - // Gate G6: Rolling DORA Change Fail Rate (CFR) circuit breaker. - g6 := profile.Gates.G6 + // Gate G6: Rolling DORA Change Fail Rate (CFR) circuit breaker. With AI + // deployment history the breaker applies to every admission. Without it + // (cold start) there is no rate to compute: a merge adds no deployment and + // is not applicable; a first deployment was escalated to K4 above, so only + // the human boundary can admit it. aiCFR := big.NewRat(0, 1) humanCFR := big.NewRat(0, 1) - g6Pass := true - if g6.AITotalDeployments <= 0 || g6.AIFailedDeployments < 0 || g6.AIFailedDeployments > g6.AITotalDeployments || - g6.HumanTotalDeployments < 0 || g6.HumanFailedDeployments < 0 || g6.HumanFailedDeployments > g6.HumanTotalDeployments { - g6Pass = false - } else { - aiCFR = big.NewRat(int64(g6.AIFailedDeployments), int64(g6.AITotalDeployments)) - if aiCFR.Cmp(maxAICFR) > 0 { - g6Pass = false - } - if g6.HumanTotalDeployments > 0 { - humanCFR = big.NewRat(int64(g6.HumanFailedDeployments), int64(g6.HumanTotalDeployments)) - if aiCFR.Cmp(humanCFR) > 0 { + switch { + case coldStart && admission == AdmissionMerge: + checks = append(checks, notApplicableCheck("g6_dora_cfr_circuit_breaker", + "cold start: the AI line has no deployment history and a merge adds no deployment; the breaker applies from the first deploy admission")) + case coldStart: + checks = append(checks, makeCheck("g6_dora_cfr_circuit_breaker", true, + "cold start: the AI line has no deployment history; the first deployment is admitted only through k4_human_boundary (deploy_cold_start_requires_k4)")) + default: + g6Pass := !g6Invalid + if g6Pass { + aiCFR = big.NewRat(int64(g6.AIFailedDeployments), int64(g6.AITotalDeployments)) + if aiCFR.Cmp(maxAICFR) > 0 { g6Pass = false } + if g6.HumanTotalDeployments > 0 { + humanCFR = big.NewRat(int64(g6.HumanFailedDeployments), int64(g6.HumanTotalDeployments)) + if aiCFR.Cmp(humanCFR) > 0 { + g6Pass = false + } + } } + if !g6Pass { + maxAutoClass = ClassK1 + killLevel = escalateKillSwitch(killLevel, KillSwitchL1) + killReasons = append(killReasons, "g6_cfr_budget_exceeded_autonomy_downgraded_to_k1") + } + g6Summary := fmt.Sprintf("aiCFR=%d/%d (%s) humanCFR=%d/%d (%s) maxAutoClass=%s", + g6.AIFailedDeployments, g6.AITotalDeployments, aiCFR.FloatString(4), + g6.HumanFailedDeployments, g6.HumanTotalDeployments, humanCFR.FloatString(4), maxAutoClass) + checks = append(checks, makeCheck("g6_dora_cfr_circuit_breaker", g6Pass, g6Summary)) } - if !g6Pass { - maxAutoClass = ClassK1 - killLevel = escalateKillSwitch(killLevel, KillSwitchL1) - killReasons = append(killReasons, "g6_cfr_budget_exceeded_autonomy_downgraded_to_k1") - } - g6Summary := fmt.Sprintf("aiCFR=%d/%d (%s) humanCFR=%d/%d (%s) maxAutoClass=%s", - g6.AIFailedDeployments, g6.AITotalDeployments, aiCFR.FloatString(4), - g6.HumanFailedDeployments, g6.HumanTotalDeployments, humanCFR.FloatString(4), maxAutoClass) - checks = append(checks, makeCheck("g6_dora_cfr_circuit_breaker", g6Pass, g6Summary)) // Gate G7: Task budget, duration, burn multiplier, and cross-family overseer. + // G7 bounds the metered autonomous loop: its per-task spend and duration, + // a different-family overseer for its work, and kill switch L2 revoking its + // leases. An operator-directed merge is directed and reviewed by a named + // human operator, holds no loop budget or leases and is not metered per + // task, so the gate has no subject. Every deployment, and any supplied + // budget receipt, is evaluated. g7 := profile.Gates.G7 - costUSD, err := parseNonNegativeRat(g7.TaskCostUSD, "gates.g7.taskCostUsd") - if err != nil { - return Report{}, err - } - medCostUSD, err := parseNonNegativeRat(g7.MedianTaskCostUSD, "gates.g7.medianTaskCostUsd") - if err != nil { - return Report{}, err - } - taskDurMin, err := parseNonNegativeRat(g7.TaskDurationMinutes, "gates.g7.taskDurationMinutes") - if err != nil { - return Report{}, err + switch { + case operatorDirected && admission == AdmissionMerge && g7 == (G7Receipt{}): + checks = append(checks, notApplicableCheck("g7_budget_and_cross_family_overseer", + fmt.Sprintf("operator-directed merge (%s): no metered autonomous loop budget or overseer pairing to bound", profile.Candidate.OperatorReference))) + case g7 == (G7Receipt{}): + checks = append(checks, makeCheck("g7_budget_and_cross_family_overseer", false, + "a deployment or autonomous loop candidate requires a budget and overseer receipt; none supplied")) + default: + costUSD, err := parseNonNegativeRat(g7.TaskCostUSD, "gates.g7.taskCostUsd") + if err != nil { + return Report{}, err + } + medCostUSD, err := parseNonNegativeRat(g7.MedianTaskCostUSD, "gates.g7.medianTaskCostUsd") + if err != nil { + return Report{}, err + } + taskDurMin, err := parseNonNegativeRat(g7.TaskDurationMinutes, "gates.g7.taskDurationMinutes") + if err != nil { + return Report{}, err + } + execFam := strings.ToLower(strings.TrimSpace(g7.ExecutorModelFamily)) + overFam := strings.ToLower(strings.TrimSpace(g7.OverseerModelFamily)) + burnLimit := new(big.Rat).Mul(medCostUSD, maxCostBurnMult) + burnExceeded := medCostUSD.Sign() <= 0 || costUSD.Cmp(burnLimit) > 0 + if burnExceeded { + killLevel = escalateKillSwitch(killLevel, KillSwitchL2) + killReasons = append(killReasons, "g7_task_cost_burn_exceeded_2x_median") + } + g7Pass := costUSD.Cmp(maxTaskCostUSD) <= 0 && + taskDurMin.Cmp(maxTaskDurationMin) <= 0 && + !burnExceeded && + execFam != "" && overFam != "" && execFam != overFam + g7Summary := fmt.Sprintf("cost=$%s (median=$%s) duration=%smin executor=%s overseer=%s", + costUSD.FloatString(2), medCostUSD.FloatString(2), taskDurMin.FloatString(1), execFam, overFam) + checks = append(checks, makeCheck("g7_budget_and_cross_family_overseer", g7Pass, g7Summary)) + } + + // S1-S6: LKG promotion invariants of the slot a deployment is promoted into. + switch { + case admission == AdmissionMerge: + checks = append(checks, notApplicableCheck("s1_s6_lkg_promotion", + "merge admission promotes nothing to an LKG slot; S1-S6 apply at deploy admission")) + case lkg == nil: + checks = append(checks, makeCheck("s1_s6_lkg_promotion", false, + "deploy admission requires an S1-S6 LKG receipt; none supplied")) + default: + lkgBlockers := ValidateLKG(*lkg) + lkgSummary := fmt.Sprintf("S1-S6 hold: pinnedSlots=%d soak=%dmin candidateSha=%s", + lkg.PinnedLKGSlots, lkg.SoakMinutes, lkg.CandidateSHA) + if len(lkgBlockers) > 0 { + lkgSummary = strings.Join(lkgBlockers, "; ") + } + checks = append(checks, makeCheck("s1_s6_lkg_promotion", len(lkgBlockers) == 0, lkgSummary)) } - execFam := strings.ToLower(strings.TrimSpace(g7.ExecutorModelFamily)) - overFam := strings.ToLower(strings.TrimSpace(g7.OverseerModelFamily)) - burnLimit := new(big.Rat).Mul(medCostUSD, maxCostBurnMult) - burnExceeded := medCostUSD.Sign() <= 0 || costUSD.Cmp(burnLimit) > 0 - if burnExceeded { - killLevel = escalateKillSwitch(killLevel, KillSwitchL2) - killReasons = append(killReasons, "g7_task_cost_burn_exceeded_2x_median") - } - g7Pass := costUSD.Cmp(maxTaskCostUSD) <= 0 && - taskDurMin.Cmp(maxTaskDurationMin) <= 0 && - !burnExceeded && - execFam != "" && overFam != "" && execFam != overFam - g7Summary := fmt.Sprintf("cost=$%s (median=$%s) duration=%smin executor=%s overseer=%s", - costUSD.FloatString(2), medCostUSD.FloatString(2), taskDurMin.FloatString(1), execFam, overFam) - checks = append(checks, makeCheck("g7_budget_and_cross_family_overseer", g7Pass, g7Summary)) var voiReport *VOIReport if profile.VOI != nil { @@ -752,12 +1089,14 @@ func EvaluateCandidate(profile CandidateProfile) (Report, error) { fmt.Sprintf("valid=%t authorized=%t reasons=%v", dr.MethodologicallyValid, dr.StageGateAuthorized, dr.Reasons))) } - passed := 0 - failed := 0 + passed, failed, notApplicable := 0, 0, 0 for _, c := range checks { - if c.Status == StatusPass { + switch c.Status { + case StatusPass: passed++ - } else { + case StatusNotApplicable: + notApplicable++ + default: failed++ } } @@ -778,10 +1117,12 @@ func EvaluateCandidate(profile CandidateProfile) (Report, error) { SchemaVersion: ReportSchema, APIVersion: ProfileAPIVersion, CandidateID: profile.Candidate.ID, + Admission: admission, Classification: cls, Checks: checks, Passed: passed, Failed: failed, + NotApplicable: notApplicable, MaxAutonomousClass: maxAutoClass, KillSwitchLevel: killLevel, KillSwitchReasons: killReasons, @@ -805,6 +1146,84 @@ func makeCheck(id string, pass bool, summary string) CheckResult { return CheckResult{ID: id, Status: status, Summary: summary} } +func notApplicableCheck(id string, summary string) CheckResult { + return CheckResult{ID: id, Status: StatusNotApplicable, Summary: summary} +} + +// ValidateLKG returns the S1-S6 invariants an LKG receipt violates; an empty +// result means the deployed candidate may be promoted to an LKG slot. +func ValidateLKG(s LKGReceipt) []string { + var blockers []string + if !s.ImmutableBootloaderVerified { + blockers = append(blockers, "S1: immutable bootloader supervisor checksum not verified") + } + if s.PinnedLKGSlots < MinPinnedLKGSlots { + blockers = append(blockers, fmt.Sprintf("S2: requires at least %d locally pinned LKG slots, got %d", MinPinnedLKGSlots, s.PinnedLKGSlots)) + } + if s.SoakMinutes < MinSoakMinutes { + blockers = append(blockers, fmt.Sprintf("S2: requires >= %d minutes soak before LKG promotion, got %d", MinSoakMinutes, s.SoakMinutes)) + } + if !s.WebhookBypassVerified { + blockers = append(blockers, "S3: rollback namespace/channel bypass of admission/conversion webhooks not verified") + } + if !s.PreMigrationSnapshotVerified { + blockers = append(blockers, "S4: pre-migration state snapshot and V_old/V_new round-trip not verified") + } + if !s.MetaRSISelfCheckPassed { + blockers = append(blockers, "S5: meta-RSI noop self-check canary did not pass") + } + if strings.TrimSpace(s.CandidateSHA) == "" { + blockers = append(blockers, "S6: candidateSha is empty") + } else if slices.Contains(s.QuarantinedSHAs, s.CandidateSHA) { + blockers = append(blockers, fmt.Sprintf("S6: candidateSha %s is in incident quarantine and requires a new regression test", s.CandidateSHA)) + } + return blockers +} + +// evaluateG4 checks one live canary step against the SLO limits and returns +// the verdict, its summary and the canary's share of the monthly error budget. +func evaluateG4(g4 G4Receipt) (bool, string, *big.Rat, error) { + stepPct, err := parseNonNegativeRat(g4.CanaryStepPercent, "gates.g4.canaryStepPercent") + if err != nil { + return false, "", nil, err + } + maxPct, err := parseNonNegativeRat(g4.CanaryMaxPercent, "gates.g4.canaryMaxPercent") + if err != nil { + return false, "", nil, err + } + succPct, err := parseNonNegativeRat(g4.RequestSuccessRatePercent, "gates.g4.requestSuccessRatePercent") + if err != nil { + return false, "", nil, err + } + latMs, err := parseNonNegativeRat(g4.LatencyP99Ms, "gates.g4.latencyP99Ms") + if err != nil { + return false, "", nil, err + } + durMin, err := parseNonNegativeRat(g4.CanaryDurationMinutes, "gates.g4.canaryDurationMinutes") + if err != nil { + return false, "", nil, err + } + stepFrac := new(big.Rat).Quo(stepPct, big.NewRat(100, 1)) + maxFrac := new(big.Rat).Quo(maxPct, big.NewRat(100, 1)) + succFrac := new(big.Rat).Quo(succPct, big.NewRat(100, 1)) + // Worst-case error-minutes during one canary step = stepFrac * durMin; burn fraction = (stepFrac * durMin) / 43.2 + errMinutes := new(big.Rat).Mul(stepFrac, durMin) + burnFrac := new(big.Rat).Quo(errMinutes, sloMonthlyBudgetMin) + + pass := g4.ActiveCanaries == 1 && + stepFrac.Cmp(maxCanaryStepPct) <= 0 && + maxFrac.Cmp(maxCanaryWeightPct) <= 0 && + succFrac.Cmp(minCanarySuccessRate) >= 0 && + latMs.Cmp(maxCanaryLatencyMs) <= 0 && + burnFrac.Cmp(maxCanarySLOBurnFrac) <= 0 && + g4.CanaryFailuresLast24h < 3 && + (!g4.TouchesPersistentState || g4.DowngradeContinuityVerified) + summary := fmt.Sprintf("active=%d step=%s%% max=%s%% success=%s%% p99=%sms burnFrac=%s stateful=%t downgradeOK=%t", + g4.ActiveCanaries, stepPct.FloatString(1), maxPct.FloatString(1), succPct.FloatString(2), latMs.FloatString(1), + burnFrac.FloatString(4), g4.TouchesPersistentState, g4.DowngradeContinuityVerified) + return pass, summary, burnFrac, nil +} + func escalateKillSwitch(current, candidate string) string { rank := map[string]int{ KillSwitchNone: 0, diff --git a/pkg/rsi/rsi_fuzz_test.go b/pkg/rsi/rsi_fuzz_test.go index cc63a9b..d9ea4a0 100644 --- a/pkg/rsi/rsi_fuzz_test.go +++ b/pkg/rsi/rsi_fuzz_test.go @@ -24,6 +24,11 @@ func FuzzEvaluateCandidate(f *testing.F) { f.Fatal(err) } f.Add(seedFail) + seedMerge, err := json.Marshal(mergeOnlyProfile()) + if err != nil { + f.Fatal(err) + } + f.Add(seedMerge) f.Add([]byte(`{"apiVersion":"cloudring.org/rsi-governance/v1","kind":"RSICandidateProfile"}`)) f.Fuzz(func(t *testing.T, raw []byte) { @@ -35,8 +40,33 @@ func FuzzEvaluateCandidate(f *testing.F) { if err != nil { return } - if rep.Passed+rep.Failed != len(rep.Checks) { - t.Fatalf("check count mismatch: passed=%d failed=%d total=%d", rep.Passed, rep.Failed, len(rep.Checks)) + if rep.Passed+rep.Failed+rep.NotApplicable != len(rep.Checks) { + t.Fatalf("check count mismatch: passed=%d failed=%d notApplicable=%d total=%d", rep.Passed, rep.Failed, rep.NotApplicable, len(rep.Checks)) + } + merge := rep.Admission == AdmissionMerge + c := parsed.Candidate + if merge && (c.Admission != AdmissionMerge || len(c.DeploymentTargets) > 0 || parsed.Gates.G4 != (G4Receipt{}) || parsed.LKG != nil || + !mergeOnlyTargets[mergeTarget{c.TargetRepository, c.TargetBranch}]) { + t.Fatalf("candidate with deployment evidence evaluated as merge: %+v", c) + } + seen := map[string]bool{} + for _, check := range rep.Checks { + seen[check.ID] = true + if check.Status == StatusNotApplicable && !merge && + (check.ID == "g4_canary_slo_and_stateful_downgrade" || check.ID == "s1_s6_lkg_promotion" || check.ID == "g7_budget_and_cross_family_overseer") { + t.Fatalf("deploy-time check %s skipped for admission %s", check.ID, rep.Admission) + } + } + if !seen["s1_s6_lkg_promotion"] { + t.Fatalf("report has no s1_s6_lkg_promotion check: %+v", rep.Checks) + } + g6 := parsed.Gates.G6 + g6Valid := g6.AIFailedDeployments >= 0 && g6.HumanFailedDeployments >= 0 && g6.HumanFailedDeployments <= g6.HumanTotalDeployments + if !merge && g6Valid && g6.AITotalDeployments == 0 && g6.AIFailedDeployments == 0 && rep.Classification.EffectiveClass != ClassK4 { + t.Fatalf("deploy cold start not escalated to K4: %+v", rep.Classification) + } + if rep.Verdict == VerdictPass && rep.Classification.EffectiveClass == ClassK4 && !validReference(c.HumanK4ApprovalReference) { + t.Fatalf("K4 candidate passed without a valid human reference %q", c.HumanK4ApprovalReference) } if rep.Failed > 0 && rep.Verdict != VerdictFail { t.Fatalf("failed checks=%d but verdict=%s", rep.Failed, rep.Verdict) diff --git a/pkg/rsi/rsi_test.go b/pkg/rsi/rsi_test.go index 98e25c0..1231711 100644 --- a/pkg/rsi/rsi_test.go +++ b/pkg/rsi/rsi_test.go @@ -75,6 +75,16 @@ func validCompliantK3Profile() CandidateProfile { OverseerModelFamily: "family-b", }, }, + LKG: &LKGReceipt{ + ImmutableBootloaderVerified: true, + PinnedLKGSlots: 2, + SoakMinutes: 30, + WebhookBypassVerified: true, + PreMigrationSnapshotVerified: true, + MetaRSISelfCheckPassed: true, + CandidateSHA: "a326123af6ca994ce876bb786dff7683f181ca36", + QuarantinedSHAs: []string{"deadbeef00000000000000000000000000000000"}, + }, VOI: &VOIInput{ PriorSuccessProb: "2/5", SuccessNetPayoffEUR: "120000", diff --git a/pkg/safepush/github.go b/pkg/safepush/github.go index c039cdd..fcb4055 100644 --- a/pkg/safepush/github.go +++ b/pkg/safepush/github.go @@ -362,7 +362,9 @@ func (api *githubAPI) get(ctx context.Context, suffix string, query url.Values, if err != nil { return ErrGitHubAPI } - defer resp.Body.Close() + // Every path either returns ErrGitHubAPI already or has read the body to the + // end through strictjson.Read, so a Close error cannot change the result. + defer func() { _ = resp.Body.Close() }() mediaType, _, mediaErr := mime.ParseMediaType(resp.Header.Get("Content-Type")) if resp.StatusCode != http.StatusOK || mediaErr != nil || (mediaType != "application/json" && mediaType != "application/vnd.github+json") || diff --git a/pkg/safepush/github_fields_test.go b/pkg/safepush/github_fields_test.go new file mode 100644 index 0000000..53eeef6 --- /dev/null +++ b/pkg/safepush/github_fields_test.go @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "context" + "errors" + "fmt" + "maps" + "net/http" + "strings" + "testing" + "time" + + "github.com/opencloudtech/CloudRING/internal/strictjson" +) + +const ghTestEmptyBlob = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + +// These cases change one provider field consistently on every read that +// carries it, so the named check alone must reject the snapshot, with the +// error class that check returns. + +func (fixture *githubHTTPSFixture) ghRun() map[string]any { + return fixture.responses["/actions/runs/201"].(map[string]any) +} + +func (fixture *githubHTTPSFixture) ghList(route, key string) []any { + return fixture.responses[route].(map[string]any)[key].([]any) +} + +func (fixture *githubHTTPSFixture) ghJob(index int) map[string]any { + return fixture.ghList("/actions/runs/201/attempts/1/jobs", "jobs")[index].(map[string]any) +} + +func (fixture *githubHTTPSFixture) ghEditTrees(edit func(treeID string, entries []any) []any) { + for _, treeID := range []string{ghTestBaseTree, ghTestHeadTree} { + tree := fixture.responses["/git/trees/"+treeID].(map[string]any) + tree["tree"] = edit(treeID, tree["tree"].([]any)) + } +} + +func (fixture *githubHTTPSFixture) ghOnlyJob(name string) { + fixture.policy.Workflows[0].Jobs = []string{name} + fixture.setJobs([]string{name}) +} + +func ghTreeEntry(path, entryType, mode, sha string) map[string]any { + return map[string]any{"path": path, "type": entryType, "mode": mode, "sha": sha} +} + +func TestGitHubSingleFieldSpoofing(t *testing.T) { + cases := []struct { + name string + want error + edit func(*githubHTTPSFixture) + }{ + {"pr id zero", ErrGitHubState, func(f *githubHTTPSFixture) { f.responses["/pulls/7"].(map[string]any)["id"] = 0 }}, + {"pr head repository id zero", ErrGitHubState, func(f *githubHTTPSFixture) { + f.responses["/pulls/7"].(map[string]any)["head"].(map[string]any)["repo"] = map[string]any{"id": 0, "full_name": "owner/repo"} + }}, + {"comparison diverged", ErrGitHubState, func(f *githubHTTPSFixture) { + f.responses["/compare/"+ghTestBase+"..."+ghTestHead].(map[string]any)["status"] = "diverged" + }}, + {"comparison without ahead_by", ErrGitHubState, func(f *githubHTTPSFixture) { + delete(f.responses["/compare/"+ghTestBase+"..."+ghTestHead].(map[string]any), "ahead_by") + }}, + {"comparison ahead_by zero", ErrGitHubState, func(f *githubHTTPSFixture) { + f.responses["/compare/"+ghTestBase+"..."+ghTestHead].(map[string]any)["ahead_by"] = 0 + }}, + {"tree entry with invalid object id", ErrGitHubSource, func(f *githubHTTPSFixture) { + f.ghEditTrees(func(_ string, e []any) []any { return append(e, ghTreeEntry("README.md", "blob", "100644", "zz")) }) + }}, + {"tree entry with padded path", ErrGitHubSource, func(f *githubHTTPSFixture) { + f.ghEditTrees(func(_ string, e []any) []any { + return append(e, ghTreeEntry(" README.md", "blob", "100644", ghTestOther)) + }) + }}, + {"guarded directory with file mode", ErrGitHubSource, func(f *githubHTTPSFixture) { + f.ghEditTrees(func(_ string, e []any) []any { + return append(e, ghTreeEntry(".github/scripts/lib", "tree", "100644", ghTestOther)) + }) + }}, + {"guarded entry of another object type", ErrGitHubSource, func(f *githubHTTPSFixture) { + f.ghEditTrees(func(_ string, e []any) []any { + return append(e, ghTreeEntry(".github/scripts/lib", "commit", "040000", ghTestOther)) + }) + }}, + {"excluded workflow replaced by an unlisted one", ErrGitHubSource, func(f *githubHTTPSFixture) { + f.ghEditTrees(func(_ string, e []any) []any { + e[1].(map[string]any)["path"] = ".github/workflows/spoof.yml" + return e + }) + }}, + {"workflow id zero", ErrGitHubSource, func(f *githubHTTPSFixture) { + f.ghList("/actions/workflows", "workflows")[2].(map[string]any)["id"] = 0 + }}, + {"null total_count", ErrGitHubAPI, func(f *githubHTTPSFixture) { + f.responses["/actions/workflows"] = map[string]any{"total_count": nil, "workflows": []any{}} + }}, + {"string total_count", ErrGitHubAPI, func(f *githubHTTPSFixture) { + f.responses["/actions/workflows"] = map[string]any{"total_count": "0", "workflows": []any{}} + }}, + {"null item list", ErrGitHubAPI, func(f *githubHTTPSFixture) { + f.responses["/actions/workflows"] = map[string]any{"total_count": 0, "workflows": nil} + }}, + {"run id zero", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghRun()["id"] = 0 }}, + {"run number zero", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghRun()["run_number"] = 0 }}, + {"run attempt zero", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghRun()["run_attempt"] = 0 }}, + {"run check suite zero", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghRun()["check_suite_id"] = 0 }}, + {"duplicate run id with a newer number", ErrGitHubRun, func(f *githubHTTPSFixture) { + newer := maps.Clone(f.ghRun()) + newer["run_number"] = 12 + f.responses["/actions/runs"] = ghTestEnvelope("workflow_runs", []any{f.ghRun(), newer}) + }}, + {"run of another workflow id", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghRun()["workflow_id"] = 103 }}, + {"push run", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghRun()["event"] = "push" }}, + {"run of another branch", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["head_branch"] = "other" + f.ghJob(0)["head_branch"], f.ghJob(1)["head_branch"] = "other", "other" + }}, + {"run of another head", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["head_sha"] = ghTestOther + for _, route := range []string{"/actions/runs/201/attempts/1/jobs", "/check-suites/301/check-runs"} { + key := map[bool]string{true: "jobs", false: "check_runs"}[strings.HasSuffix(route, "/jobs")] + for _, item := range f.ghList(route, key) { + item.(map[string]any)["head_sha"] = ghTestOther + } + } + }}, + {"run in another repository id", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["repository"] = map[string]any{"id": 43, "full_name": "owner/repo"} + }}, + {"run in another repository name", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["repository"] = map[string]any{"id": 42, "full_name": "owner/fork"} + }}, + {"run from another head repository id", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["head_repository"] = map[string]any{"id": 43, "full_name": "owner/repo"} + }}, + {"run from another head repository name", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["head_repository"] = map[string]any{"id": 42, "full_name": "owner/fork"} + }}, + {"run with referenced workflows", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghRun()["referenced_workflows"] = []any{map[string]any{"path": "owner/other/workflow.yml"}} + }}, + {"run linked to another pr id", ErrGitHubRun, func(f *githubHTTPSFixture) { + ghTestFirst(f.ghRun(), "pull_requests")["id"] = 72 + }}, + {"job id zero", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghJob(0)["id"] = 0 }}, + {"check url without origin", ErrGitHubRun, func(f *githubHTTPSFixture) { f.ghJob(0)["check_run_url"] = "401" }}, + {"check url with leading zero", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghJob(0)["check_run_url"] = "https://api.github.com/repos/owner/repo/check-runs/0401" + }}, + {"step number zero", ErrGitHubRun, func(f *githubHTTPSFixture) { ghTestFirst(f.ghJob(0), "steps")["number"] = 0 }}, + {"duplicate step number", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghJob(0)["steps"].([]any)[1].(map[string]any)["number"] = 1 + }}, + {"blank step name", ErrGitHubRun, func(f *githubHTTPSFixture) { ghTestFirst(f.ghJob(0), "steps")["name"] = "" }}, + {"duplicate check run", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghOnlyJob("unit") + checks := f.ghList("/check-suites/301/check-runs", "check_runs") + f.responses["/check-suites/301/check-runs"] = ghTestEnvelope("check_runs", []any{checks[0], checks[0]}) + }}, + {"two jobs share one check run", ErrGitHubRun, func(f *githubHTTPSFixture) { + f.ghOnlyJob("unit") + second := maps.Clone(f.ghJob(0)) + second["id"] = 402 + f.responses["/actions/runs/201/attempts/1/jobs"] = ghTestEnvelope("jobs", []any{f.ghJob(0), second}) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + tc.edit(fixture) + if err := fixture.verify(); !errors.Is(err, tc.want) { + t.Fatalf("got %v, want %v", err, tc.want) + } + }) + } +} + +func TestGitHubAdmitsOrdinaryRepositoryContent(t *testing.T) { + cases := map[string]func(*githubHTTPSFixture){ + // A PR changes code outside .github; trees list directory entries; a + // .yml outside the workflow directory or a non-YAML file inside it is + // not a workflow. + "code change and directory entries": func(f *githubHTTPSFixture) { + f.ghEditTrees(func(treeID string, e []any) []any { + code := map[string]string{ghTestBaseTree: "3333333333333333333333333333333333333333", ghTestHeadTree: ghTestOther}[treeID] + return append(e, + ghTreeEntry("pkg/app.go", "blob", "100644", code), + ghTreeEntry("docs/mkdocs.yml", "blob", "100644", "6666666666666666666666666666666666666666"), + ghTreeEntry(".github/workflows/README.md", "blob", "100644", "7777777777777777777777777777777777777777"), + ghTreeEntry(".github", "tree", "040000", "8888888888888888888888888888888888888888"), + ghTreeEntry(".github/workflows", "tree", "040000", "abababababababababababababababababababab"), + ghTreeEntry(".github/scripts", "tree", "040000", "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd"), + ) + }) + }, + "executable workflow file": func(f *githubHTTPSFixture) { + f.ghEditTrees(func(_ string, e []any) []any { + e[0].(map[string]any)["mode"] = "100755" + return e + }) + }, + "explicitly complete lists": func(f *githubHTTPSFixture) { + envelope := f.responses["/actions/workflows"].(map[string]any) + envelope["incomplete_results"], envelope["truncated"] = false, false + }, + "workflow source at the size limit": func(f *githubHTTPSFixture) { + contents := ghTestWorkflow + "#" + contents += strings.Repeat("#", maxWorkflowSourceBytes-len(contents)-1) + "\n" + f.setRequiredWorkflow(contents) + }, + } + for name, edit := range cases { + t.Run(name, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + edit(fixture) + if err := fixture.verify(); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestGitHubListLimits(t *testing.T) { + names := func(n int) []string { + out := make([]string, n) + for index := range out { + out[index] = fmt.Sprintf("matrix-%d", index) + } + return out + } + // The provider limit is inclusive: 1000 jobs span exactly ten pages. + fixture := newGitHubHTTPSFixture(t) + fixture.policy.Workflows[0].Jobs = names(1000) + fixture.setJobs(names(1000)) + if err := fixture.verify(); err != nil { + t.Fatalf("1000 jobs: %v", err) + } + // An over-limit total is rejected on the first page, before more reads. + fixture = newGitHubHTTPSFixture(t) + fixture.policy.Workflows[0].Jobs = names(101) + fixture.setJobs(names(101)) + fixture.responses["/actions/runs/201/attempts/1/jobs"].(map[string]any)["total_count"] = 1001 + if err := fixture.verify(); !errors.Is(err, ErrGitHubAPI) { + t.Fatalf("over-limit total: %v", err) + } + if got := fixture.counts["/actions/runs/201/attempts/1/jobs"]; got != 1 { + t.Fatalf("over-limit total read %d pages, want 1", got) + } +} + +func TestReadGitHubPolicyRepositoryState(t *testing.T) { + for name, edit := range map[string]func(map[string]any){ + "id zero": func(r map[string]any) { r["id"] = 0 }, + "renamed": func(r map[string]any) { r["full_name"] = "owner/other" }, + "archived": func(r map[string]any) { r["archived"] = true }, + "disabled": func(r map[string]any) { r["disabled"] = true }, + "id zero alone": func(r map[string]any) { r["id"], r["full_name"] = 0, "owner/repo" }, + } { + t.Run(name, func(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + edit(fixture.responses[""].(map[string]any)) + if _, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/repo", ghTestBase); !errors.Is(err, ErrGitHubState) { + t.Fatalf("got %v", err) + } + }) + } +} + +func TestReadGitHubPolicyRejectsEmptyBlob(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + fixture.responses["/git/trees/"+ghTestBaseTree].(map[string]any)["tree"].([]any)[3].(map[string]any)["sha"] = ghTestEmptyBlob + fixture.responses["/git/blobs/"+ghTestEmptyBlob] = map[string]any{"sha": ghTestEmptyBlob, "encoding": "base64", "size": 0, "content": ""} + if _, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/repo", ghTestBase); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("got %v", err) + } +} + +func TestGitHubResponseAtTheDocumentLimit(t *testing.T) { + fixture := newGitHubHTTPSFixture(t) + body := `{"id":42,"full_name":"owner/repo","archived":false,"disabled":false}` + body += strings.Repeat(" ", strictjson.MaxDocumentBytes-len(body)) + fixture.reply = func(writer http.ResponseWriter, request *http.Request, _ int) bool { + if request.URL.Path != "/repos/owner/repo" { + return false + } + writer.Header().Set("Content-Length", fmt.Sprint(len(body))) + _, _ = fmt.Fprint(writer, body) + return true + } + if _, err := ReadGitHubPolicy(context.Background(), fixture.client, "test-token", "owner/repo", ghTestBase); err != nil { + t.Fatalf("response of exactly %d bytes: %v", strictjson.MaxDocumentBytes, err) + } +} + +// githubRecordingTransport records how much time each request may take and +// stops it before any network access. +type githubRecordingTransport struct { + calls int + remaining time.Duration +} + +func (transport *githubRecordingTransport) RoundTrip(request *http.Request) (*http.Response, error) { + transport.calls++ + if deadline, ok := request.Context().Deadline(); ok { + transport.remaining = time.Until(deadline) + } + return nil, errors.New("stopped by test transport") +} + +func TestGitHubTokenIsCheckedBeforeTransport(t *testing.T) { + for _, malformed := range []string{"test token", "test\x7ftoken", "test\x01token", "tést"} { + transport := &githubRecordingTransport{} + client := &http.Client{Transport: transport} + if _, err := ReadGitHubPolicy(context.Background(), client, malformed, "owner/repo", ghTestBase); !errors.Is(err, ErrGitHubAPI) { + t.Errorf("token %q: %v", malformed, err) + } + if transport.calls != 0 { + t.Errorf("token %q reached the transport", malformed) + } + } +} + +func TestGitHubRequestTimeoutIsCapped(t *testing.T) { + cases := []struct { + timeout time.Duration + min, max time.Duration + }{ + {0, 25 * time.Second, 30 * time.Second}, + {time.Hour, 25 * time.Second, 30 * time.Second}, + {50 * time.Millisecond, 0, 50 * time.Millisecond}, + } + for _, tc := range cases { + transport := &githubRecordingTransport{} + client := &http.Client{Transport: transport, Timeout: tc.timeout} + if _, err := ReadGitHubPolicy(context.Background(), client, "test-token", "owner/repo", ghTestBase); !errors.Is(err, ErrGitHubAPI) { + t.Fatalf("timeout %v: %v", tc.timeout, err) + } + if transport.calls != 1 || transport.remaining <= tc.min || transport.remaining > tc.max { + t.Errorf("client timeout %v: request may take %v, want (%v, %v]", tc.timeout, transport.remaining, tc.min, tc.max) + } + } +} diff --git a/pkg/safepush/policy_limits_test.go b/pkg/safepush/policy_limits_test.go new file mode 100644 index 0000000..b618759 --- /dev/null +++ b/pkg/safepush/policy_limits_test.go @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package safepush + +import ( + "errors" + "fmt" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestValidateGitHubCandidateBounds(t *testing.T) { + valid := GitHubCandidate{PullRequest: 1, HeadSHA: ghTestHead, BaseSHA: ghTestBase, TargetBranch: "main"} + for repository, want := range map[string]bool{ + "owner/repo": true, "noslash": false, "owner/.": false, "owner/..": false, "owner/.x": true, + } { + if err := ValidateGitHubCandidate(repository, valid); (err == nil) != want { + t.Errorf("repository %q: %v", repository, err) + } + } + for name, c := range map[string]struct { + edit func(*GitHubCandidate) + ok bool + }{ + "pull request zero": {func(c *GitHubCandidate) { c.PullRequest = 0 }, false}, + "pull request one": {func(c *GitHubCandidate) {}, true}, + "master target": {func(c *GitHubCandidate) { c.TargetBranch = "master" }, true}, + "other target": {func(c *GitHubCandidate) { c.TargetBranch = "dev" }, false}, + "same head and base": {func(c *GitHubCandidate) { c.BaseSHA = c.HeadSHA }, false}, + } { + candidate := valid + c.edit(&candidate) + if err := ValidateGitHubCandidate("owner/repo", candidate); (err == nil) != c.ok || (err != nil && !errors.Is(err, ErrRequest)) { + t.Errorf("%s: %v", name, err) + } + } +} + +func TestValidateGitHubPolicySizeLimits(t *testing.T) { + workflowPath := func(i int) string { return fmt.Sprintf(".github/workflows/w%03d.yml", i) } + names := func(n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("job-%d", i) + } + return out + } + cases := map[string]struct { + edit func(*GitHubPolicy) + ok bool + }{ + "99 workflows and one exclusion": {func(p *GitHubPolicy) { + p.Workflows = nil + for i := range 99 { + p.Workflows = append(p.Workflows, GitHubWorkflow{Path: workflowPath(i), Jobs: []string{"test"}}) + } + }, true}, + "one workflow and 100 exclusions": {func(p *GitHubPolicy) { + p.ExcludedWorkflows = nil + for i := range 100 { + p.ExcludedWorkflows = append(p.ExcludedWorkflows, workflowPath(i)) + } + }, false}, + "1000 jobs": {func(p *GitHubPolicy) { p.Workflows[0].Jobs = names(1000) }, true}, + "1001 jobs": {func(p *GitHubPolicy) { p.Workflows[0].Jobs = names(1001) }, false}, + "255-byte workflow path": {func(p *GitHubPolicy) { + p.Workflows[0].Path = ".github/workflows/" + strings.Repeat("w", 255-len(".github/workflows/")-len(".yml")) + ".yml" + }, true}, + "256-byte workflow path": {func(p *GitHubPolicy) { + p.Workflows[0].Path = ".github/workflows/" + strings.Repeat("w", 256-len(".github/workflows/")-len(".yml")) + ".yml" + }, false}, + "workflow outside the directory": {func(p *GitHubPolicy) { p.Workflows[0].Path = "ci.yml" }, false}, + "empty job name": {func(p *GitHubPolicy) { p.Workflows[0].Jobs = []string{""} }, false}, + "512-byte job name": {func(p *GitHubPolicy) { p.Workflows[0].Jobs = []string{strings.Repeat("j", 512)} }, true}, + "513-byte job name": {func(p *GitHubPolicy) { p.Workflows[0].Jobs = []string{strings.Repeat("j", 513)} }, false}, + } + for name, c := range cases { + policy := policyFixture() + c.edit(&policy) + if err := ValidateGitHubPolicy(policy); (err == nil) != c.ok { + t.Errorf("%s: %v", name, err) + } + } +} + +// ghYAMLShape counts nodes and the deepest level of a parsed document the way +// a reader of the YAML tree sees it: the document is level 0 and every child +// is one level below its parent. +func ghYAMLShape(node *yaml.Node, depth int) (nodes, deepest int) { + nodes, deepest = 1, depth + for _, child := range node.Content { + n, d := ghYAMLShape(child, depth+1) + nodes += n + deepest = max(deepest, d) + } + return nodes, deepest +} + +func ghParse(t *testing.T, source string) *yaml.Node { + t.Helper() + var document yaml.Node + if err := yaml.Unmarshal([]byte(source), &document); err != nil { + t.Fatal(err) + } + return &document +} + +func TestWorkflowSourceLimitsAreInclusive(t *testing.T) { + // Depth: env: {nested: [[...false...]]} with k sequences. + nested := func(k int) string { + return "env: {nested: " + strings.Repeat("[", k) + "false" + strings.Repeat("]", k) + "}\n" + ghTestWorkflow + } + k := 1 + for _, deepest := ghYAMLShape(ghParse(t, nested(k)), 0); deepest < maxWorkflowYAMLDepth; _, deepest = ghYAMLShape(ghParse(t, nested(k)), 0) { + k++ + } + if _, deepest := ghYAMLShape(ghParse(t, nested(k)), 0); deepest != maxWorkflowYAMLDepth { + t.Fatalf("could not build a document %d levels deep", maxWorkflowYAMLDepth) + } + if err := verifyWorkflowSource([]byte(nested(k))); err != nil { + t.Fatalf("document exactly %d levels deep: %v", maxWorkflowYAMLDepth, err) + } + if err := verifyWorkflowSource([]byte(nested(k + 1))); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("document %d levels deep accepted", maxWorkflowYAMLDepth+1) + } + + // Nodes: env: {k0: 0, ...} plus, for odd remainders, one x: [0] entry. + wide := func(pairs int, extra bool) string { + var b strings.Builder + b.WriteString("env: {") + if extra { + b.WriteString("x: [0], ") + } + for i := range pairs { + fmt.Fprintf(&b, "k%d: 0, ", i) + } + b.WriteString("}\n") + return b.String() + ghTestWorkflow + } + base, _ := ghYAMLShape(ghParse(t, wide(0, false)), 0) + extra := (maxWorkflowYAMLNodes-base)%2 == 1 + pairs := (maxWorkflowYAMLNodes - base) / 2 + if extra { + pairs = (maxWorkflowYAMLNodes - base - 3) / 2 + } + exact := wide(pairs, extra) + if nodes, _ := ghYAMLShape(ghParse(t, exact), 0); nodes != maxWorkflowYAMLNodes { + t.Fatalf("built %d nodes, want %d", nodes, maxWorkflowYAMLNodes) + } + if err := verifyWorkflowSource([]byte(exact)); err != nil { + t.Fatalf("document of exactly %d nodes: %v", maxWorkflowYAMLNodes, err) + } + if err := verifyWorkflowSource([]byte(wide(pairs+1, extra))); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("document of %d nodes accepted", maxWorkflowYAMLNodes+2) + } +} + +func TestWorkflowSourceSizeAndJobs(t *testing.T) { + over := ghTestWorkflow + "#" + over += strings.Repeat("#", maxWorkflowSourceBytes-len(over)) + "\n" + if err := verifyWorkflowSource([]byte(over)); !errors.Is(err, ErrGitHubSource) { + t.Fatalf("workflow of %d bytes accepted", len(over)) + } + if err := verifyWorkflowSource([]byte("name: CI\non: pull_request\n")); !errors.Is(err, ErrGitHubSource) { + t.Fatal("workflow without jobs accepted") + } +} diff --git a/pkg/sliceoracles/admission_test.go b/pkg/sliceoracles/admission_test.go new file mode 100644 index 0000000..a7fa147 --- /dev/null +++ b/pkg/sliceoracles/admission_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package sliceoracles_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/opencloudtech/CloudRING/pkg/rsi" + "github.com/opencloudtech/CloudRING/pkg/sliceoracles" +) + +func TestDeclaredAutonomyClassCoversAllowedPaths(t *testing.T) { + batch := loadCanonicalBatch(t) + rank := map[string]int{rsi.ClassK0: 0, rsi.ClassK1: 1, rsi.ClassK2: 2, rsi.ClassK3: 3, rsi.ClassK4: 4} + for _, o := range batch.Oracles { + cls, err := rsi.ClassifyCandidate(rsi.CandidateMetadata{ID: o.SliceID, DeclaredClass: o.AutonomyClass, ChangedPaths: o.AllowedPaths}) + if err != nil { + t.Fatalf("%s: %v", o.SliceID, err) + } + if rank[o.AutonomyClass] < rank[cls.EffectiveClass] { + t.Errorf("%s declares %s but its allowedPaths are %s (%v)", o.SliceID, o.AutonomyClass, cls.EffectiveClass, cls.EscalationReasons) + } + } + + // A batch that declares a slice below its effective class is rejected. + raw, err := os.ReadFile(filepath.Join("..", "..", "contracts", "slice-oracles", "oracles-32.json")) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + for _, o := range doc["oracles"].([]any) { + if o := o.(map[string]any); o["sliceId"] == "SP01" { + o["autonomyClass"] = rsi.ClassK3 + } + } + lowered, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + if _, err := sliceoracles.ParseBatch(lowered); err == nil { + t.Fatal("batch declaring SP01 as K3 with a .github allowed path was accepted") + } +} + +func TestMergeAdmissionSkipsLKGPromotion(t *testing.T) { + batch := loadCanonicalBatch(t) + read := func(name string) rsi.CandidateProfile { + data, err := os.ReadFile(filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", name)) + if err != nil { + t.Fatal(err) + } + prof, err := rsi.ParseProfile(data) + if err != nil { + t.Fatal(err) + } + return prof + } + noLKG := sliceoracles.LKGSafetyState{} + + rec, err := batch.EvaluateSliceCandidate("C06", "FULL", read("merge-only-k3-profile.json"), noLKG) + if err != nil { + t.Fatal(err) + } + if rec.Verdict != rsi.VerdictPass || rec.PromotedToLKG || len(rec.Blockers) != 0 { + t.Fatalf("merge admission: verdict=%s promoted=%t blockers=%v", rec.Verdict, rec.PromotedToLKG, rec.Blockers) + } + + // The slot state passed to slice acceptance replaces the profile's own LKG + // receipt, so a profile cannot vouch for a slot it is not promoted into. + rec, err = batch.EvaluateSliceCandidate("C06", "FULL", read("compliant-k3-profile.json"), noLKG) + if err != nil { + t.Fatal(err) + } + if rec.Verdict != rsi.VerdictFail || rec.PromotedToLKG || len(rec.Blockers) != 8 || rec.RSIReport == nil || rec.RSIReport.Verdict != rsi.VerdictFail { + t.Fatalf("deploy admission without LKG state: verdict=%s promoted=%t blockers=%v", rec.Verdict, rec.PromotedToLKG, rec.Blockers) + } + for _, c := range rec.RSIReport.Checks { + if c.ID == "s1_s6_lkg_promotion" && c.Status != rsi.StatusFail { + t.Fatalf("s1_s6_lkg_promotion %s for an empty slot", c.Status) + } + } +} + +// A merge claim that contradicts its own scope is evaluated as deploy, and its +// S1-S6 check then runs against the slot passed to slice acceptance, never +// against the receipt the profile carries about itself. +func TestContradictedMergeIsCheckedAgainstTheSlot(t *testing.T) { + batch := loadCanonicalBatch(t) + read := func(name string) rsi.CandidateProfile { + data, err := os.ReadFile(filepath.Join("..", "..", "contracts", "rsi-governance", "fixtures", name)) + if err != nil { + t.Fatal(err) + } + prof, err := rsi.ParseProfile(data) + if err != nil { + t.Fatal(err) + } + return prof + } + goodSlot := *read("compliant-k3-profile.json").LKG + s1s6 := func(rec *sliceoracles.SliceAcceptanceReceipt) rsi.CheckResult { + t.Helper() + if rec.RSIReport == nil || rec.RSIReport.Admission != rsi.AdmissionDeploy { + t.Fatalf("contradicted merge was not evaluated as deploy: %+v", rec.RSIReport) + } + for _, c := range rec.RSIReport.Checks { + if c.ID == "s1_s6_lkg_promotion" { + return c + } + } + t.Fatal("no s1_s6_lkg_promotion check") + return rsi.CheckResult{} + } + + selfVouching := read("merge-only-k3-profile.json") + selfVouching.LKG = &goodSlot + rec, err := batch.EvaluateSliceCandidate("C06", "FULL", selfVouching, sliceoracles.LKGSafetyState{}) + if err != nil { + t.Fatal(err) + } + if c := s1s6(rec); c.Status != rsi.StatusFail || rec.Verdict != rsi.VerdictFail || rec.PromotedToLKG { + t.Fatalf("own receipt vouched for an empty slot: s1_s6=%s %q verdict=%s", c.Status, c.Summary, rec.Verdict) + } + + withTarget := read("merge-only-k3-profile.json") + withTarget.Candidate.DeploymentTargets = []string{"hub"} + rec, err = batch.EvaluateSliceCandidate("C06", "FULL", withTarget, goodSlot) + if err != nil { + t.Fatal(err) + } + if c := s1s6(rec); c.Status != rsi.StatusPass { + t.Fatalf("contradicted merge ignored the supplied slot: s1_s6=%s %q", c.Status, c.Summary) + } + rec, err = batch.EvaluateSliceCandidate("C06", "FULL", withTarget, sliceoracles.LKGSafetyState{}) + if err != nil { + t.Fatal(err) + } + if c := s1s6(rec); c.Status != rsi.StatusFail || len(rec.Blockers) < 6 { + t.Fatalf("contradicted merge with an empty slot: s1_s6=%s blockers=%v", c.Status, rec.Blockers) + } +} diff --git a/pkg/sliceoracles/oracles.go b/pkg/sliceoracles/oracles.go index 42503cc..d55e215 100644 --- a/pkg/sliceoracles/oracles.go +++ b/pkg/sliceoracles/oracles.go @@ -23,8 +23,8 @@ const ( ExpectedKind = "SliceOracleBatch" ExpectedSliceCount = 32 MaxWritePRsPerRepo = 2 - MinPinnedLKGSlots = 2 - MinSoakMinutes = 30 + MinPinnedLKGSlots = rsi.MinPinnedLKGSlots + MinSoakMinutes = rsi.MinSoakMinutes ) // Batch represents the frozen 32-oracle contract (Fork 1A + Fork 2A + Deferred Fork 3). @@ -64,16 +64,7 @@ type SliceOracle struct { } // LKGSafetyState captures the 6 necessary and sufficient self-evolution liveness invariants (S1-S6). -type LKGSafetyState struct { - ImmutableBootloaderVerified bool `json:"immutableBootloaderVerified"` - PinnedLKGSlots int `json:"pinnedLkgSlots"` - SoakMinutes int `json:"soakMinutes"` - WebhookBypassVerified bool `json:"webhookBypassVerified"` - PreMigrationSnapshotVerified bool `json:"preMigrationSnapshotVerified"` - MetaRSISelfCheckPassed bool `json:"metaRsiSelfCheckPassed"` - CandidateSHA string `json:"candidateSha"` - QuarantinedSHAs []string `json:"quarantinedShas"` -} +type LKGSafetyState = rsi.LKGReceipt // ActivePR describes an open write PR for Fork 2A concurrency validation. type ActivePR struct { @@ -146,6 +137,9 @@ func (b *Batch) Validate() error { if o.SplitMode && len(o.QualDependencies) == 0 { return fmt.Errorf("sliceoracles: split slice %s must declare qualDependencies", o.SliceID) } + if err := checkAutonomyClass(o); err != nil { + return err + } byID[o.SliceID] = o } @@ -192,6 +186,29 @@ func (b *Batch) Validate() error { return nil } +// classOrder ranks autonomy classes so a declared class can be compared with +// the class the RSI classifier derives from the slice's allowed paths. +var classOrder = map[string]int{rsi.ClassK0: 0, rsi.ClassK1: 1, rsi.ClassK2: 2, rsi.ClassK3: 3, rsi.ClassK4: 4} + +// checkAutonomyClass rejects an oracle whose declared autonomy class is lower +// than the effective class of a change touching all of its allowed paths, so a +// slice cannot be accepted under a weaker gate set than its paths require. +func checkAutonomyClass(o SliceOracle) error { + declared, ok := classOrder[o.AutonomyClass] + if !ok { + return fmt.Errorf("sliceoracles: slice %s has invalid autonomyClass %q", o.SliceID, o.AutonomyClass) + } + cls, err := rsi.ClassifyCandidate(rsi.CandidateMetadata{ID: o.SliceID, DeclaredClass: o.AutonomyClass, ChangedPaths: o.AllowedPaths}) + if err != nil { + return fmt.Errorf("sliceoracles: slice %s: %w", o.SliceID, err) + } + if classOrder[cls.EffectiveClass] > declared { + return fmt.Errorf("sliceoracles: slice %s declares %s but its allowedPaths require %s (%s)", + o.SliceID, o.AutonomyClass, cls.EffectiveClass, strings.Join(cls.EscalationReasons, ", ")) + } + return nil +} + // PartitionNoNewHardware splits the 32 oracles into immediately executable no-new-hardware slices // vs slices gated on deferred Fork 3 hardware. func (b *Batch) PartitionNoNewHardware() (noNewHW []SliceOracle, deferredHW []SliceOracle) { @@ -326,35 +343,15 @@ func pathsOverlap(a, b string) bool { // ValidateLKGSafety checks all 6 S1-S6 invariants required for dead-end-free autonomous self-evolution. func ValidateLKGSafety(s LKGSafetyState) []string { - var blockers []string - if !s.ImmutableBootloaderVerified { - blockers = append(blockers, "S1: immutable bootloader supervisor checksum not verified") - } - if s.PinnedLKGSlots < MinPinnedLKGSlots { - blockers = append(blockers, fmt.Sprintf("S2: requires at least %d locally pinned LKG slots, got %d", MinPinnedLKGSlots, s.PinnedLKGSlots)) - } - if s.SoakMinutes < MinSoakMinutes { - blockers = append(blockers, fmt.Sprintf("S2: requires >= %d minutes soak before LKG promotion, got %d", MinSoakMinutes, s.SoakMinutes)) - } - if !s.WebhookBypassVerified { - blockers = append(blockers, "S3: rollback namespace/channel bypass of admission/conversion webhooks not verified") - } - if !s.PreMigrationSnapshotVerified { - blockers = append(blockers, "S4: pre-migration state snapshot and V_old/V_new round-trip not verified") - } - if !s.MetaRSISelfCheckPassed { - blockers = append(blockers, "S5: meta-RSI noop self-check canary did not pass") - } - if strings.TrimSpace(s.CandidateSHA) == "" { - blockers = append(blockers, "S6: candidateSha is empty") - } else if slices.Contains(s.QuarantinedSHAs, s.CandidateSHA) { - blockers = append(blockers, fmt.Sprintf("S6: candidateSha %s is in incident quarantine and requires a new regression test", s.CandidateSHA)) - } - return blockers + return rsi.ValidateLKG(s) } // EvaluateSliceCandidate runs autonomous Fork 1A acceptance for a slice candidate by combining // oracle membership, hardware boundary check, S1-S6 LKG safety, and the frozen RSI G1-G7 evaluator. +// The lkg state is the slot the candidate would be promoted into: whenever the derived admission is +// deploy, including a merge claim that contradicts its own scope, the RSI S1-S6 check runs against +// this slot and never against a receipt in the profile. A consistent merge promotes nothing, so a +// passing merge receipt never reports PromotedToLKG. func (b *Batch) EvaluateSliceCandidate(sliceID, phase string, profile rsi.CandidateProfile, lkg LKGSafetyState) (*SliceAcceptanceReceipt, error) { if err := b.Validate(); err != nil { return nil, err @@ -380,20 +377,21 @@ func (b *Batch) EvaluateSliceCandidate(sliceID, phase string, profile rsi.Candid receipt.Blockers = append(receipt.Blockers, fmt.Sprintf("Fork 3 deferred: slice %s requires additional multi-node/second-provider hardware", sliceID)) } - receipt.Blockers = append(receipt.Blockers, ValidateLKGSafety(lkg)...) - - rep, err := rsi.EvaluateCandidate(profile) + rep, err := rsi.EvaluateCandidateForSlot(profile, lkg) if err != nil { return nil, err } receipt.RSIReport = &rep + if rep.Admission != rsi.AdmissionMerge { + receipt.Blockers = append(receipt.Blockers, ValidateLKGSafety(lkg)...) + } if rep.Verdict != rsi.VerdictPass { receipt.Blockers = append(receipt.Blockers, fmt.Sprintf("RSI G1-G7 evaluator returned verdict=%s (failed=%d, killSwitch=%s)", rep.Verdict, rep.Failed, rep.KillSwitchLevel)) } if len(receipt.Blockers) == 0 { receipt.Verdict = rsi.VerdictPass - receipt.PromotedToLKG = true + receipt.PromotedToLKG = rep.Admission != rsi.AdmissionMerge } return receipt, nil } diff --git a/pkg/sliceoracles/validate_limits_test.go b/pkg/sliceoracles/validate_limits_test.go new file mode 100644 index 0000000..347505d --- /dev/null +++ b/pkg/sliceoracles/validate_limits_test.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) Iurii Trukhin. + +package sliceoracles_test + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/opencloudtech/CloudRING/pkg/sliceoracles" +) + +// editedBatch returns the canonical batch document with edit applied to the +// oracle sliceID. +func editedBatch(t *testing.T, sliceID string, edit func(map[string]any)) []byte { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "contracts", "slice-oracles", "oracles-32.json")) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(raw, &doc); err != nil { + t.Fatal(err) + } + found := false + for _, o := range doc["oracles"].([]any) { + if o := o.(map[string]any); o["sliceId"] == sliceID { + edit(o) + found = true + } + } + if !found { + t.Fatalf("no oracle %s", sliceID) + } + data, err := json.Marshal(doc) + if err != nil { + t.Fatal(err) + } + return data +} + +func TestBatchRejectsIncompleteOrMisorderedOracles(t *testing.T) { + cases := []struct { + name, slice, wantErr string + edit func(map[string]any) + }{ + {"no goals", "C06", "missing goals", func(o map[string]any) { o["goals"] = []any{} }}, + {"no allowed paths", "C06", "missing goals", func(o map[string]any) { o["allowedPaths"] = []any{} }}, + {"no required gates", "C06", "missing goals", func(o map[string]any) { o["requiredGates"] = []any{} }}, + {"C24 waits for C23", "C24", "C24 coreDependencies", func(o map[string]any) { o["coreDependencies"] = []any{"C23"} }}, + {"C20 waits for C19", "C20", "C20 coreDependencies", func(o map[string]any) { o["coreDependencies"] = []any{"C11", "C16", "C19"} }}, + {"unknown dependency", "C12", "unknown dependency", func(o map[string]any) { o["coreDependencies"] = []any{"C99"} }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := sliceoracles.ParseBatch(editedBatch(t, tc.slice, tc.edit)) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("got %v, want an error containing %q", err, tc.wantErr) + } + }) + } +} + +func TestReadyQueueSplitDependencies(t *testing.T) { + batch := loadCanonicalBatch(t) + ready := func(core, full []string) []string { + set := func(ids []string) map[string]bool { + m := make(map[string]bool, len(ids)) + for _, id := range ids { + m[id] = true + } + return m + } + return batch.ReadyNoNewHardwareQueue(set(core), set(full)) + } + // C16 unlocks dependent CORE builds once its own CORE build is accepted. + got := ready([]string{"C16"}, []string{"C11"}) + for _, want := range []string{"C17:CORE", "C20:CORE", "DBAAS:CORE", "FILESAAS:CORE"} { + if !slices.Contains(got, want) { + t.Errorf("C16 CORE + C11 FULL: %s not ready in %v", want, got) + } + } + // Any other dependency must be FULL: a CORE-only C07 does not unlock C15. + if got := ready([]string{"C07"}, []string{"C11"}); slices.Contains(got, "C15:CORE") || slices.Contains(got, "DNSAAS:CORE") { + t.Errorf("CORE-only C07 unlocked %v", got) + } + // A split slice whose CORE is accepted qualifies once its QUAL dependencies are FULL. + if got := ready([]string{"C17"}, []string{"C13", "C14"}); !slices.Contains(got, "C17:QUAL") { + t.Errorf("C17 CORE + C13/C14 FULL: no C17:QUAL in %v", got) + } + if got := ready([]string{"C17"}, []string{"C13"}); slices.Contains(got, "C17:QUAL") { + t.Errorf("C17:QUAL ready without C14 FULL: %v", got) + } +} + +func TestConcurrentWritePRNeedsSliceAndPaths(t *testing.T) { + batch := loadCanonicalBatch(t) + for name, active := range map[string][]sliceoracles.ActivePR{ + "no slice id": {{SliceID: " ", TouchedPaths: []string{"pkg/rsi"}}}, + "no paths": {{SliceID: "SP01"}}, + } { + err := batch.ValidateConcurrentWritePRs(active) + if err == nil || !strings.Contains(err.Error(), "must declare sliceId and touchedPaths") { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/roadmap/EXECUTION_CONTRACT.md b/roadmap/EXECUTION_CONTRACT.md index b9864a0..930ed14 100644 --- a/roadmap/EXECUTION_CONTRACT.md +++ b/roadmap/EXECUTION_CONTRACT.md @@ -105,6 +105,48 @@ Fork 1A), with weekly founder audit and rollback authority. A compatible batch o already accepted public commits may share one downstream pin, but each requirement must remain traceable. +The evaluator distinguishes two admissions. Merge admission covers a change that +enters a branch no environment tracks as a GitOps source. Today only +`opencloudtech/CloudRING` `main` qualifies, and the profile names it. Merge +admission runs G1–G3, G5 when a drill receipt is supplied and G6 when deployment +history exists. The live canary (G4) and LKG promotion (S1–S6) do not apply. G7 +does not apply only to an operator-directed merge. Deploy admission covers a +change that reaches a running environment, including a downstream pin bump. It +runs every gate and requires an S1–S6 receipt. A first AI deployment without +history is K4. A merge claim is evaluated as deploy when it names another target, +carries deployment targets, touches a `deploy`, `iac` or gitlink path, +`.gitmodules` or a workflow file, carries a canary or LKG receipt, or claims an +operator-directed origin without a valid reference. A profile without an +admission field is evaluated as deploy. A key that is not an exact field name is +rejected. + +G2 for guardrail code (`pkg/rsi`, `pkg/sliceoracles`, `pkg/safepush`, +`cmd/cloudring-safepush`, and the `rsi` command files and changed dispatch lines +of `cmd/cloudringctl`) is the mutation score measured by +`go run ./cmd/cloudring-mutation --base --packages +./pkg/rsi,./pkg/sliceoracles,./pkg/safepush,./cmd/cloudring-safepush,./cmd/cloudringctl`. +The tool generates the mutants of gremlins v0.6.0 with the default mutators plus +`INVERT_LOGICAL` in the files `go build` compiles for those packages, on lines +the candidate adds or changes against its merge-base (every file compared as +text, a renamed file counting as new, physical lines regardless of `//line` +directives) and on its new files. It applies each mutant alone in a private copy +of the module and runs `go test -count=1 -timeout 2m` (its `--timeout` default) +on the mutant's package. The tool stops with exit 2 unless every package with a +mutant passes `go test` unmutated in the copy, before the mutants and in every +copy after them, when it cannot read the diff in full, and when a compiled file +is not a regular file of its package directory. A passing run is LIVED and a +run that hits the time limit TIMED OUT. A failing run is KILLED only after +confirmation in the same copy: the unmutated package then passes twice in a row +and the mutant fails again with the same first failing test; otherwise the +mutant is UNSTABLE. A mutant that does not build, including one that `go vet` +rejects during `go test`, is not viable and is not counted. +`killedMutants` counts KILLED mutants. `totalMutants` counts KILLED, LIVED, +TIMED OUT and UNSTABLE mutants. A mutant leaves the denominator only as an +equivalent mutant whose written proof that no input distinguishes it is in the +evidence. The threshold is 4/5. Tests in the measured packages must not read the +package's source files or keep state across runs; such a test invalidates G2 and +is a review finding. + ## Universal definition of done A goal is complete only when all applicable cells are green: