From b712e8fede805a076592868a29d40c981e3ac428 Mon Sep 17 00:00:00 2001 From: Alexandre Daubois Date: Tue, 10 Nov 2026 16:58:27 +0100 Subject: [PATCH] Confirm exact duplicates in the sandbox only, drop the in-process engine and the reference worktrees, and eval an unpublished duplicate --- cmd/tamis/main.go | 35 +-- cmd/tamis/main_test.go | 22 +- docs/configuration.md | 14 +- docs/development.md | 6 +- docs/how-it-works.md | 4 +- docs/security.md | 2 +- go.mod | 22 +- go.sum | 131 -------- internal/engine/claude.go | 103 ------- internal/engine/engine.go | 69 ----- internal/engine/engine_test.go | 289 ------------------ internal/engine/openai.go | 97 ------ internal/github/client.go | 11 +- internal/github/client_test.go | 14 +- internal/repo/repo.go | 54 +--- internal/repo/repo_test.go | 36 --- internal/report/report.go | 13 +- internal/report/report_test.go | 1 - internal/tools/tools.go | 127 -------- internal/tools/tools_test.go | 104 ------- internal/triage/eval_test.go | 43 ++- internal/triage/revisions.go | 100 +++--- .../testdata/evals/libmini/include/mini.h | 1 + .../triage/testdata/evals/libmini/src/keys.c | 19 ++ .../evals/reports/query-keys-overflow.md | 22 ++ internal/triage/triage.go | 127 +++----- internal/triage/triage_test.go | 163 +++------- 27 files changed, 257 insertions(+), 1372 deletions(-) delete mode 100644 internal/engine/claude.go delete mode 100644 internal/engine/engine.go delete mode 100644 internal/engine/engine_test.go delete mode 100644 internal/engine/openai.go delete mode 100644 internal/tools/tools.go delete mode 100644 internal/tools/tools_test.go create mode 100644 internal/triage/testdata/evals/libmini/src/keys.c create mode 100644 internal/triage/testdata/evals/reports/query-keys-overflow.md diff --git a/cmd/tamis/main.go b/cmd/tamis/main.go index 0143751..a7bc1b4 100644 --- a/cmd/tamis/main.go +++ b/cmd/tamis/main.go @@ -17,7 +17,6 @@ import ( "time" "github.com/php/tamis/internal/board" - "github.com/php/tamis/internal/engine" "github.com/php/tamis/internal/github" "github.com/php/tamis/internal/sandbox" "github.com/php/tamis/internal/setup" @@ -25,8 +24,12 @@ import ( "github.com/php/tamis/skill" ) -// defaultCodexTokenModel is Codex's default with a ChatGPT account, which does not serve the API models. -const defaultCodexTokenModel = "gpt-6.1-sol" +const ( + defaultClaudeModel = "claude-opus-5-5" + defaultOpenAIModel = "gpt-5.5" + // defaultCodexTokenModel is Codex's default with a ChatGPT account, which does not serve the API models. + defaultCodexTokenModel = "gpt-6.1-sol" +) func main() { log := slog.New(slog.NewTextHandler(os.Stderr, nil)) @@ -211,7 +214,7 @@ func run(ctx context.Context, log *slog.Logger) error { if cfg.CloseDuplicates && cfg.IssuesRepository == "" { return errors.New("CLOSE_DUPLICATES needs ISSUES_REPOSITORY") } - eng, agent, model, err := newEngine(os.Getenv("ENGINE"), os.Getenv("MODEL"), os.Getenv("LLM_API_KEY")) + agent, model, err := selectAgent(os.Getenv("ENGINE"), os.Getenv("MODEL"), os.Getenv("LLM_API_KEY")) if err != nil { return err } @@ -239,7 +242,7 @@ func run(ctx context.Context, log *slog.Logger) error { defer reader.Close() reader.Daybreak = daybreak - return triage.Run(ctx, cfg, github.New(token), reader, eng, log) + return triage.Run(ctx, cfg, github.New(token), reader, log) } func parseSince(v string) (time.Time, error) { @@ -257,28 +260,22 @@ func parseSince(v string) (time.Time, error) { return t, nil } -// newEngine returns the comparator engine, and the sandboxed agent and model the reader runs with. A Codex -// access token cannot call the OpenAI API, so the comparator runs as a sandboxed agent too: no engine. -func newEngine(name, model, apiKey string) (engine.Engine, string, string, error) { +// selectAgent returns the sandboxed agent every model step runs as, and its model. A Codex access token cannot +// call the OpenAI API, so it gets an agent of its own. +func selectAgent(name, model, apiKey string) (string, string, error) { if apiKey == "" { - return nil, "", "", errors.New("LLM_API_KEY is required") + return "", "", errors.New("LLM_API_KEY is required") } switch name { case "", "claude": - if model == "" { - model = engine.DefaultClaudeModel - } - return engine.NewClaude(apiKey, model), "claude", model, nil + return "claude", cmp.Or(model, defaultClaudeModel), nil case "openai": if strings.HasPrefix(apiKey, "at-") { - return nil, "codex-token", cmp.Or(model, defaultCodexTokenModel), nil - } - if model == "" { - model = engine.DefaultOpenAIModel + return "codex-token", cmp.Or(model, defaultCodexTokenModel), nil } - return engine.NewOpenAI(apiKey, model), "codex", model, nil + return "codex", cmp.Or(model, defaultOpenAIModel), nil default: - return nil, "", "", fmt.Errorf("unknown ENGINE %q, expected claude or openai", name) + return "", "", fmt.Errorf("unknown ENGINE %q, expected claude or openai", name) } } diff --git a/cmd/tamis/main_test.go b/cmd/tamis/main_test.go index 0c65f7d..239a8ea 100644 --- a/cmd/tamis/main_test.go +++ b/cmd/tamis/main_test.go @@ -28,30 +28,28 @@ func TestParseSince(t *testing.T) { assert.Error(t, err, "an invalid date should fail") } -func TestNewEngine(t *testing.T) { - for name, want := range map[string]string{"": "claude", "claude": "claude", "openai": "codex"} { - _, agent, model, err := newEngine(name, "", "key") +func TestSelectAgent(t *testing.T) { + for name, want := range map[string][2]string{"": {"claude", defaultClaudeModel}, "claude": {"claude", defaultClaudeModel}, "openai": {"codex", defaultOpenAIModel}} { + agent, model, err := selectAgent(name, "", "key") assert.NoError(t, err, "%q", name) - assert.Equal(t, want, agent, "%q", name) - assert.NotEmpty(t, model, "%q", name) + assert.Equal(t, want[0], agent, "%q", name) + assert.Equal(t, want[1], model, "%q", name) } - eng, agent, model, err := newEngine("openai", "", "at-token") + agent, model, err := selectAgent("openai", "", "at-token") assert.NoError(t, err, "a Codex access token") - assert.Nil(t, eng, "a Codex access token") assert.Equal(t, "codex-token", agent, "a Codex access token") assert.Equal(t, defaultCodexTokenModel, model, "a Codex access token") - eng, agent, _, _ = newEngine("claude", "", "at-key") - assert.NotNil(t, eng, "only the openai engine reads a Codex access token") + agent, _, _ = selectAgent("claude", "", "at-key") assert.Equal(t, "claude", agent, "only the openai engine reads a Codex access token") - _, _, model, _ = newEngine("claude", "claude-sonnet-5-5", "key") + _, model, _ = selectAgent("claude", "claude-sonnet-5-5", "key") assert.Equal(t, "claude-sonnet-5-5", model, "an explicit model should be kept") - _, _, _, err = newEngine("gemini", "", "key") + _, _, err = selectAgent("gemini", "", "key") assert.Error(t, err, "an unknown engine should fail") - _, _, _, err = newEngine("claude", "", "") + _, _, err = selectAgent("claude", "", "") assert.Error(t, err, "a missing API key should fail") } diff --git a/docs/configuration.md b/docs/configuration.md index 39662a6..1a5ba41 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,7 +56,7 @@ Each report names the source commit and the input revision it covers. | Request | 1 MiB of body. API responses ask for at most 16,000 output tokens, and the ChatGPT backend applies its own output limits. | | Reports | One per known reporter and per execution | -tamis starts a report only while the execution still holds a whole advisory budget and thirty minutes of its time. Otherwise the report waits for the next execution. SDK retries are off, but tamis retries an agent run that fails without a valid report once, after a minute, unless the model account was refused for good or the advisory budget is spent. A GitHub read answered with 429, 502, 503 or 504 is tried up to three times. When a budget runs out, the report stays retriable and tamis does not mark it complete. +tamis starts a report only while the execution still holds a whole advisory budget and thirty minutes of its time. Otherwise the report waits for the next execution. tamis retries an agent run that fails without a valid report once, after a minute, unless the model account was refused for good or the advisory budget is spent. A GitHub read answered with 429, 502, 503 or 504 is tried up to three times. When a budget runs out, the report stays retriable and tamis does not mark it complete. These bounds cap one execution, not the spending across scheduled runs. Provider pricing, tokenisation and ChatGPT credit policies differ, so request and byte limits give no monetary guarantee. On the provider side: @@ -89,11 +89,13 @@ The triage lists the files the agent checked. Without any of them, tamis leaves ## Engines -| Engine | Reader and global search | Exact confirmation | Model API | -|---|---|---|---| -| `claude` | Claude Code | Anthropic API | `api.anthropic.com` | -| `openai` | Codex | OpenAI Chat Completions | `api.openai.com` | -| `openai` with a Codex access token | Codex | Codex, in a container without GitHub access | `chatgpt.com` | +| Engine | Agent | Model API | +|---|---|---| +| `claude` | Claude Code | `api.anthropic.com` | +| `openai` | Codex | `api.openai.com` | +| `openai` with a Codex access token | Codex | `chatgpt.com` | + +The reader, the duplicate search and each exact confirmation run that agent in a container of their own, the last two without GitHub access. [`docker/agent/Dockerfile`](../docker/agent/Dockerfile) pins the agent CLI versions. diff --git a/docs/development.md b/docs/development.md index 4356ad1..9589dfe 100644 --- a/docs/development.md +++ b/docs/development.md @@ -9,7 +9,7 @@ go build ./cmd/tamis go test -race ./... ``` -Three fuzz targets cover the comment escaper, the duplicate-search clues and file reading: +Three fuzz targets cover the text escaper, the duplicate-search clues and file reading: | Target | Package | |---|---| @@ -38,7 +38,7 @@ Repeat with `claude`. The tests use fake model servers, check direct host, Inter ## Evals -The evals run the real triage, sandboxed agent and model included, on made-up reports about `libmini`, a small C library in `internal/triage/testdata/evals`. Each scenario checks the location, the CWE, the policy exclusions and the duplicates the models propose, and that a prompt injection does not leak another advisory into the comment. They spend real model requests, so they only run with the `evals` tag, Docker and a key: +The evals run the real triage, sandboxed agent and model included, on made-up reports about `libmini`, a small C library in `internal/triage/testdata/evals`. Each scenario checks the location, the CWE, the policy exclusions and the duplicates the models propose, that a prompt injection does not leak another advisory into the triage, and that an unpublished exact duplicate reaches the board item once its confirmation passed. They spend real model requests, so they only run with the `evals` tag, Docker and a key: ```bash ENGINE=claude LLM_API_KEY="$ANTHROPIC_API_KEY" go test -tags evals -count=1 -timeout 1h -run TestEvals ./internal/triage @@ -93,8 +93,6 @@ tamis writes the board item for real. It skips an advisory whose item already ho | `internal/sandbox` | The agent container and the model proxy that keeps the key on the host, through harness | | `internal/budget` | Request and byte budgets shared by every agent and comparator of a run | | `internal/usage` | Requests and tokens spent per advisory, for the log | -| `internal/engine` | The in-process model loop of the exact duplicate confirmation | -| `internal/tools` | The read-only tools of the exact duplicate confirmation | | `internal/report` | Report schema, verification and rendering | | `internal/repo` | Read-only access to tracked files | | `internal/github` | GitHub REST client | diff --git a/docs/how-it-works.md b/docs/how-it-works.md index e477bd4..356cfed 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -24,7 +24,7 @@ The agent reports the faulty statement (file, function, line), the public entry tamis rejects an unknown or mistyped field in the JSON and sends the error back to the agent for one correction round. -When the run stops on its budget or on an error, tamis keeps a valid report already written in the private artifact and retries the advisory. The last attempt posts that report with a note that it may be incomplete. +When the run stops on its budget or on an error, tamis keeps a valid report already written in the private artifact and retries the advisory. The last attempt puts that report on the board with a note that it may be incomplete. The agent analyses the first configured reference, where tamis locates the cited code, and the final report names the exact commit of every reference. A missing branch fails the triage. @@ -39,7 +39,7 @@ One offline agent searches every other advisory of the repository for the same b tamis treats the report, the clues and the advisories as hostile data. The agent answers with the identifiers of the duplicates it found, each with a confidence and an exact flag. tamis refuses an identifier outside the advisories it gave. -The private report keeps the full search result. tamis checks each unpublished candidate marked exact with high confidence again, in its own model context. That context receives the report, the candidate without its identifier, the checked clues and the code, and returns one `exact` boolean. Only the host knows which candidate the answer belongs to, and it copies a confirmed identifier into the public comment. All models share the advisory and run budgets. The log records the duration and usage of the confirmations and the number of confirmed candidates. +The private report keeps the full search result. tamis checks each unpublished candidate marked exact with high confidence again, in its own model context. That context receives the report, the candidate without its identifier, the checked clues and the code, and returns one `exact` boolean. Only the host knows which candidate the answer belongs to, and it copies a confirmed identifier into the board item. All models share the advisory and run budgets. The log records the duration and usage of the confirmations and the number of confirmed candidates. Both stages compare concrete defects, secondary findings of a report included: diff --git a/docs/security.md b/docs/security.md index aac6c9a..57005a5 100644 --- a/docs/security.md +++ b/docs/security.md @@ -34,7 +34,7 @@ tamis itself never logs report text, the identifiers of other advisories or prov The agent runs through [harness](https://github.com/alpha-omega-security/harness), the isolation layer of [scrutineer](https://github.com/alpha-omega-security/scrutineer): - a throwaway container per advisory, with a read-only root filesystem, all capabilities dropped and no new privileges, -- one single-branch clone of the analysed branch per execution, mounted read-only into each agent run, with no untracked files and an empty `.git/config`, while only tamis reads the other branches, checked out as worktrees, +- one single-branch clone of the analysed branch per execution, mounted read-only into each agent run, with no untracked files and an empty `.git/config`, while tamis only reads the commit of the other branches, - an internal bridge network with both gateway modes set to `isolated` and IPv6 disabled, whose only peer is an authenticated egress proxy that allows the model proxy and `api.github.com`, - a read-only workspace except for `report.json`, with memory, CPU, process and file-size limits, - an image with only the agent CLI, git, curl, ripgrep and `harness-proxy`, each download checked against a pinned checksum, diff --git a/go.mod b/go.mod index ad988a8..1f5a15e 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,7 @@ go 1.27.3 require ( github.com/alpha-omega-security/harness v0.1.21 - github.com/anthropics/anthropic-sdk-go v1.84.0 github.com/git-pkgs/cwe v0.1.1 - github.com/openai/openai-go/v3 v3.97.0 github.com/stretchr/testify v1.12.1 go.yaml.in/yaml/v3 v3.0.6 golang.org/x/crypto v0.58.0 @@ -15,34 +13,16 @@ require ( require ( github.com/BurntSushi/toml v1.6.0 // indirect - github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bitfield/gotestdox v0.2.3 // indirect - github.com/buger/jsonparser v1.6.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/coder/websocket v1.8.15 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/fatih/color v1.19.0 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/git-pkgs/skills v0.2.1 // indirect - github.com/go-logr/logr v1.4.4 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/invopop/jsonschema v0.14.0 // indirect github.com/mattn/go-colorable v0.1.16 // indirect github.com/mattn/go-isatty v0.0.24 // indirect - github.com/pb33f/go-yaml v0.1.1 // indirect - github.com/pb33f/ordered-map/v2 v2.3.2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect - github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect - github.com/tidwall/gjson v1.20.2 // indirect - github.com/tidwall/match v1.2.0 // indirect - github.com/tidwall/pretty v1.3.0 // indirect - github.com/tidwall/sjson v1.2.9 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.47.0 // indirect - go.opentelemetry.io/otel/log v1.47.0 // indirect - go.opentelemetry.io/otel/metric v1.47.0 // indirect - go.opentelemetry.io/otel/trace v1.47.0 // indirect golang.org/x/exp/typeparams v0.0.0-20261106192419-47ce03d25ef0 // indirect golang.org/x/mod v0.42.0 // indirect golang.org/x/sys v0.49.0 // indirect diff --git a/go.sum b/go.sum index b196135..e7d951e 100644 --- a/go.sum +++ b/go.sum @@ -1,154 +1,39 @@ -cloud.google.com/go/auth v0.7.2 h1:uiha352VrCDMXg+yoBtaD0tUF4Kv9vrtrWPYXwutnDE= -cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= -cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= -cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= -cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= -cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 h1:qgnWTiWs1G7AqcKZegFvkJD1bmXJcaDU3uC0ELSgZ5U= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3/go.mod h1:XtxD48QPInqoj+GMlcRlXHEYsoOxFnP6w2JgIZ9rDS4= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 h1:Terj1CMW4+NY5CpWjSRkqLoxh2DkYDdv/FH2z0897Uc= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0/go.mod h1:7Ut2Qw9vcwpLW5Q4MBy43uSxXS7tK2yjs94OJ6j5thE= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/alpha-omega-security/harness v0.1.21 h1:iIpAbfuNSLmnyPmXqJHO4M2L/OVzfFQhZVOoSVLrabo= github.com/alpha-omega-security/harness v0.1.21/go.mod h1:rlD5PLSKLYxoGsx9CigCgZ87/zDCuMVKPVEFX0UZqpQ= -github.com/anthropics/anthropic-sdk-go v1.84.0 h1:mnAhggc5H5A98JYmhQELT+j4J89PYBFKnnSvUTNwNd8= -github.com/anthropics/anthropic-sdk-go v1.84.0/go.mod h1:GThfYqPJoaQ/6pmibCI98Cr4y5su2FXMUHn3NrSSnIc= -github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0= -github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 h1:tW1/Rkad38LA15X4UQtjXZXNKsCgkshC3EbmcUmghTg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3/go.mod h1:UbnqO+zjqk3uIt9yCACHJ9IVNhyhOCnYk8yA19SAWrM= -github.com/aws/aws-sdk-go-v2/config v1.33.10 h1:RRb0GghYqq2ikMXzfT4r9NO9lVhOld3FIhJJWDIdk+w= -github.com/aws/aws-sdk-go-v2/config v1.33.10/go.mod h1:BDSA7QGh6yrNQA5Z5aBnHWKMWdJDMD4IOlB4QjYSp5E= -github.com/aws/aws-sdk-go-v2/credentials v1.20.10 h1:ItSjcLk+2M15RnRxO105uVIqLiVQn8y3gaeHC7W0iqs= -github.com/aws/aws-sdk-go-v2/credentials v1.20.10/go.mod h1:eO4GlmGXB+gx+zDbhu3IENEFOmjrbVKJtOptJiIhngc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.4 h1:7ksj/zMap9+831SmNQZkcpmfuATp3DKr4BfLNnBdYXM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.4/go.mod h1:zhi8vDrD1dj76ysTWS6Ny1y6/EEW1Jrq2y8lzKeoqfs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 h1:fSNAt6hs4qLjb9tMZc6vj3PHlOCYGXK2JjJZ7AW2tf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7/go.mod h1:TMav+EuXQu5kNpZZz/6rIC8xx4mCsW+6AKHrjvOSAsE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 h1:7NS6kqBjkWIN8P/AEdmtyq6eQGAhN1oHTZdzXclpH6Y= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7/go.mod h1:1PFqtAUsIgF1nEVojzRJSf/v88zjzqrGW+w+UAtkXyQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.7 h1:wYu55QUHARwu9YzeuY5yEJUTsIu8ahkmp1l3QmbzHn8= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.7/go.mod h1:OteEkhFAgFwQtZft/3myWo3tVWxdAo7cwV+7AbcpAtQ= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.22 h1:qHfXn1Tm2Rh8kic/DvdKaiSVlHQRSr5C+rMuBR29D44= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.22/go.mod h1:D4ytxJbGXu82l+2XF0q60bibysf+KkIbWfAOldAQwf8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.7 h1:6MSlRo7aiUVBftQ8XmW7XIh3EZqCbkN9zI0YlGW8nl0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.7/go.mod h1:OqapjZfp6VW4YJKKO4l68UdPvS4FIEYuEoq1JNkxCL0= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.5 h1:oFsZ3u5PPFSO1YZDGKFgyDuwgpbv/sTmVShEZfRnyaE= -github.com/aws/aws-sdk-go-v2/service/signin v1.10.5/go.mod h1:P+IzYJeldxO0mHt8fX4cglzexP3akO94C/UuEA2ojP0= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.5 h1:EYD6E6CsgRQPazaOfZ6suyTgfZjg07HoT4S3qB9Exkg= -github.com/aws/aws-sdk-go-v2/service/sso v1.38.5/go.mod h1:hHg7OZNN1/U62f3Q5mebJYKO4f97vZC+s0fNAsPLnMk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.5 h1:v6V0AnJ0aDmivAIie/6iT7sGpjnNmzaAdxZwZ0dhwqI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.5/go.mod h1:P2Lqn27Kph9z400MFOYIebFnKjZEwyUIAva6fFFMiRY= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.5 h1:OuyNaMncYP4ftrvcnOHJ9j3AxO3zsC5C/ewD0HoHdjQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.51.5/go.mod h1:Bhu66tgvf94uunUSiAc0Ew0avhnpSm0J8Hl1MYocRbo= -github.com/aws/smithy-go v1.28.6 h1:Pj+/sUviNYjow8ELRtNwKHZ9ECeF+7pATX3+shjLIrw= -github.com/aws/smithy-go v1.28.6/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= -github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= -github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bitfield/gotestdox v0.2.3 h1:H8Wy07kYacr3YSF5aJON/m9ASE+PGKOSufXkTjqQGb8= github.com/bitfield/gotestdox v0.2.3/go.mod h1:bq/HemlNNqEvjOzy7brevVhzZD8WTGFDGq59DOIsHqI= -github.com/buger/jsonparser v1.6.1 h1:I0phFv0PlbLHnM7TZAVjZ2MJ2/eWRTDyuO7GLR98IEs= -github.com/buger/jsonparser v1.6.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= -github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/git-pkgs/cwe v0.1.1 h1:FTdif2t/pQCd8IFxJUt3fy4khoQ93djiw2sukqE8KiQ= github.com/git-pkgs/cwe v0.1.1/go.mod h1:6yUxEoA+3rI4W3ftqb7PZP1Z7zZFmV4/wfr7P46HtHU= github.com/git-pkgs/skills v0.2.1 h1:Ih/qElKJ/JymRE47FImmZNChN5VFTe3vlvHDxNZL08c= github.com/git-pkgs/skills v0.2.1/go.mod h1:8EF6FoB3VbwQJNGqNQc/5j+mD9zP6MwnpxE70itMWqM= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= -github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786 h1:rcv+Ippz6RAtvaGgKxc+8FQIpxHgsF+HBzPyYL2cyVU= github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= -github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= github.com/mattn/go-colorable v0.1.16 h1:+2YjeAe7xP1OQbZW1jERPWWCDLlNqFNCvjiqGiAt84A= github.com/mattn/go-colorable v0.1.16/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= -github.com/modelcontextprotocol/go-sdk v1.3.1 h1:TfqtNKOIWN4Z1oqmPAiWDC2Jq7K9OdJaooe0teoXASI= -github.com/modelcontextprotocol/go-sdk v1.3.1/go.mod h1:DgVX498dMD8UJlseK1S5i1T4tFz2fkBk4xogC3D15nw= -github.com/openai/openai-go/v3 v3.97.0 h1:kwaui9B6oP/Ezf+IQ8klNQy274B9cbZkBLwsEVo4rxs= -github.com/openai/openai-go/v3 v3.97.0/go.mod h1:uMht5xIbHFTRAD05AiQXzNT9hJGklZHsCnyzeWiteGo= -github.com/pb33f/go-yaml v0.1.1 h1:yWGmhVdwzionRnSc/hEJwcfq6PaZL2QdYzyReXBbX7Q= -github.com/pb33f/go-yaml v0.1.1/go.mod h1:QvWdkHP5VjFYAeHMQUkaPQqREaQnBNOXmWzAjNrnSdo= -github.com/pb33f/ordered-map/v2 v2.3.2 h1:wDyaZ2Pv9QLh64X4utCeD5Zoi9nIv2aW3lwv172O5PQ= -github.com/pb33f/ordered-map/v2 v2.3.2/go.mod h1:1OhFrXu3OYw3kM+FXF+Ug3ugmmZ9LE8z0JfQMLvtV0w= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= -github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= -github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= -github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= -github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 h1:uOfcYT+3QungH6tIGSVCR/Y3KJmgJiHcojJbMTPDZAI= -github.com/standard-webhooks/standard-webhooks/libraries v0.0.1/go.mod h1:L1MQhA6x4dn9r007T033lsaZMv9EmBAdXyU/+EF40fo= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -github.com/tidwall/gjson v1.20.2 h1:Ag48cSOUCOq/sheSLXV3M4ZS0FOd4YMnY1N7FRoxGQY= -github.com/tidwall/gjson v1.20.2/go.mod h1:jV1Xkr7LRLVFhesDFU/nInoQ+MjVlnuqkUreTWMveUo= -github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= -github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.3.0 h1:DZmEzADnMIbThFXXslDgq9C0LCC/xzMrONm7iM77RM0= -github.com/tidwall/pretty v1.3.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.2.9 h1:se9OUWsi+PBE8QMQyx7jNLVBzyNH+uaq48xg8aCaKQY= -github.com/tidwall/sjson v1.2.9/go.mod h1:w7Ke7FEphlLcBn/2H8MtgkIN8V32CUPD+8ZoG2dFTrc= -github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= -github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= -go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= -go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= -go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= -go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= -go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= -go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= -go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= -go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= go.yaml.in/yaml/v3 v3.0.6 h1:zcDfvjXEqTQmpjniUBLc91xYJMB2c4QOFxEF/LCEckM= go.yaml.in/yaml/v3 v3.0.6/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= @@ -157,10 +42,6 @@ golang.org/x/exp/typeparams v0.0.0-20261106192419-47ce03d25ef0 h1:QwRkKGyl5AwYiD golang.org/x/exp/typeparams v0.0.0-20261106192419-47ce03d25ef0/go.mod h1:qkMnvgP8C7+xkci04txL2SJFc/0PGqnkf6gs5JUC43E= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= -golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= -golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= @@ -171,8 +52,6 @@ golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= @@ -181,16 +60,6 @@ golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTF golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/vuln v1.9.0 h1:12VaqnI9kJfqwRoPptxoqsUhQtdfrSbdksT7X/KY/sc= golang.org/x/vuln v1.9.0/go.mod h1:E/Frwm0xp49fTyBW+6CQHdkvobknbwUM49yEpMGUauw= -google.golang.org/api v0.189.0 h1:equMo30LypAkdkLMBqfeIqtyAnlyig1JSZArl4XPwdI= -google.golang.org/api v0.189.0/go.mod h1:FLWGJKb0hb+pU2j+rJqwbnsF+ym+fQs73rbJ+KAUgy8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools/gotestsum v1.13.0 h1:+Lh454O9mu9AMG1APV4o0y7oDYKyik/3kBOiCqiEpRo= gotest.tools/gotestsum v1.13.0/go.mod h1:7f0NS5hFb0dWr4NtcsAsF0y1kzjEFfAil0HiBQJE03Q= gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= diff --git a/internal/engine/claude.go b/internal/engine/claude.go deleted file mode 100644 index a6a6c79..0000000 --- a/internal/engine/claude.go +++ /dev/null @@ -1,103 +0,0 @@ -package engine - -import ( - "context" - "encoding/json" - "errors" - - "github.com/anthropics/anthropic-sdk-go" - "github.com/anthropics/anthropic-sdk-go/option" - - "github.com/php/tamis/internal/tools" - "github.com/php/tamis/internal/usage" -) - -const DefaultClaudeModel = "claude-opus-5-5" - -type Claude struct { - client anthropic.Client - model string -} - -func NewClaude(apiKey, model string, opts ...option.RequestOption) *Claude { - return &Claude{client: anthropic.NewClient(append([]option.RequestOption{option.WithAPIKey(apiKey), option.WithMaxRetries(0)}, opts...)...), model: model} -} - -func (c *Claude) Run(ctx context.Context, system, prompt string, ts []tools.Tool) (json.RawMessage, error) { - defs := make([]anthropic.BetaToolUnionParam, 0, len(ts)) - for _, t := range ts { - // A nil map would be sent as "properties": null. - properties := t.Properties - if properties == nil { - properties = map[string]any{} - } - defs = append(defs, anthropic.BetaToolUnionParam{OfTool: &anthropic.BetaToolParam{ - Name: t.Name, - Description: anthropic.String(t.Description), - InputSchema: anthropic.BetaToolInputSchemaParam{ - Properties: properties, - Required: t.Required, - ExtraFields: map[string]any{"additionalProperties": false}, - }, - }}) - } - - params := anthropic.BetaMessageNewParams{ - Model: c.model, - MaxTokens: maxOutputTokens, - System: []anthropic.BetaTextBlockParam{{Text: system}}, - Messages: []anthropic.BetaMessageParam{anthropic.NewBetaUserMessage(anthropic.NewBetaTextBlock(prompt))}, - Tools: defs, - OutputConfig: anthropic.BetaOutputConfigParam{Effort: anthropic.BetaOutputConfigEffortHigh}, - // Every turn resends the whole conversation, the cache keeps the earlier turns from being billed in full again. - CacheControl: anthropic.NewBetaCacheControlEphemeralParam(), - } - - for range maxTurns { - if !takeRequest(ctx, params) { - return nil, ErrNoReport - } - resp, err := c.client.Beta.Messages.New(ctx, params) - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - apiErr := &APIError{Provider: "anthropic"} - if sdkErr := (*anthropic.Error)(nil); errors.As(err, &sdkErr) { - apiErr.Status = sdkErr.StatusCode - } - return nil, apiErr - } - usage.Add(ctx, usage.Usage{Requests: 1, InputTokens: resp.Usage.InputTokens + resp.Usage.CacheCreationInputTokens + resp.Usage.CacheReadInputTokens, OutputTokens: resp.Usage.OutputTokens}) - if resp.StopReason == anthropic.BetaStopReasonRefusal { - return nil, &RefusalError{Category: string(resp.StopDetails.Category)} - } - params.Messages = append(params.Messages, resp.ToParam()) - - var results []anthropic.BetaContentBlockParamUnion - for _, block := range resp.Content { - use, ok := block.AsAny().(anthropic.BetaToolUseBlock) - if !ok { - continue - } - input := json.RawMessage(use.JSON.Input.Raw()) - if use.Name == tools.Submit { - err := tools.Validate(ctx, ts, input) - if err == nil { - return input, nil - } - results = append(results, anthropic.NewBetaToolResultBlock(use.ID, invalid(err), true)) - continue - } - out, isError := tools.Call(ctx, ts, use.Name, input) - results = append(results, anthropic.NewBetaToolResultBlock(use.ID, out, isError)) - } - - if len(results) == 0 { - results = append(results, anthropic.NewBetaTextBlock(nudge)) - } - params.Messages = append(params.Messages, anthropic.NewBetaUserMessage(results...)) - } - - return nil, ErrNoReport -} diff --git a/internal/engine/engine.go b/internal/engine/engine.go deleted file mode 100644 index 9ac3be7..0000000 --- a/internal/engine/engine.go +++ /dev/null @@ -1,69 +0,0 @@ -// Package engine runs the tool-use loop against an LLM provider until the model submits its report. -package engine - -import ( - "context" - "encoding/json" - "errors" - "fmt" - - "github.com/php/tamis/internal/budget" - "github.com/php/tamis/internal/tools" -) - -const ( - maxTurns = 40 - maxOutputTokens = 16_000 - nudge = "Submit your findings now with the " + tools.Submit + " tool." -) - -var ErrNoReport = errors.New("the model did not submit a report") - -func invalid(err error) string { - return "error: the submission does not match the schema, fix it and submit again: " + err.Error() -} - -// APIError keeps only the status code: provider errors embed the response body, which may echo the prompt. -type APIError struct { - Provider string - Status int -} - -func (e *APIError) Error() string { - if e.Status == 0 { - return e.Provider + ": request failed" - } - - return fmt.Sprintf("%s: HTTP %d", e.Provider, e.Status) -} - -// RefusalError keeps only the category: the explanation may paraphrase the report. -type RefusalError struct { - Category string -} - -func (e *RefusalError) Error() string { - if e.Category == "" { - return "the model refused" - } - - return fmt.Sprintf("the model refused (category %q)", e.Category) -} - -type Engine interface { - // Run returns the raw input the model passed to the tools.Submit tool, which ts must contain. - Run(ctx context.Context, system, prompt string, ts []tools.Tool) (json.RawMessage, error) -} - -// RequestLimitBytes caps the cumulative request bodies of a WithRequestLimit budget. -const RequestLimitBytes = 64 << 20 - -// WithRequestLimit shares one provider-request budget across every model call made with ctx. -func WithRequestLimit(ctx context.Context, n int) context.Context { - return budget.WithLimit(ctx, n, RequestLimitBytes) -} - -func takeRequest(ctx context.Context, params any) bool { - body, err := json.Marshal(params) - return err == nil && len(body) <= 1<<20 && budget.Take(ctx, len(body)) -} diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go deleted file mode 100644 index f86e18d..0000000 --- a/internal/engine/engine_test.go +++ /dev/null @@ -1,289 +0,0 @@ -package engine - -import ( - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "testing" - - anthropicoption "github.com/anthropics/anthropic-sdk-go/option" - openaioption "github.com/openai/openai-go/v3/option" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/php/tamis/internal/tools" - "github.com/php/tamis/internal/usage" -) - -// scripted serves the given responses in order and records the request bodies. -func scripted(t *testing.T, responses ...string) (*httptest.Server, *[]string) { - t.Helper() - var requests []string - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) - requests = append(requests, r.Header.Get("anthropic-beta")+" "+string(body)) - if len(requests) > len(responses) { - assert.Failf(t, "unexpected request", "%d: %s", len(requests), body) - http.Error(w, "no more responses", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "application/json") - io.WriteString(w, responses[len(requests)-1]) - })) - t.Cleanup(srv.Close) - - return srv, &requests -} - -func testTools(calls *[]string) []tools.Tool { - return []tools.Tool{ - {Name: "lookup", Description: "look", Run: func(_ context.Context, in json.RawMessage) (string, error) { - *calls = append(*calls, string(in)) - return "lookup result", nil - }}, - {Name: tools.Submit, Description: "submit", Properties: map[string]any{"summary": map[string]any{"type": "string"}}, Required: []string{"summary"}}, - } -} - -func claudeMessage(stop, content string) string { - return `{"id":"msg","type":"message","role":"assistant","model":"m","stop_reason":"` + stop + `","content":[` + content + `],"usage":{"input_tokens":1,"cache_creation_input_tokens":2,"cache_read_input_tokens":3,"output_tokens":4}}` -} - -func TestClaudeRunsToolsUntilSubmit(t *testing.T) { - srv, requests := scripted(t, - claudeMessage("tool_use", `{"type":"tool_use","id":"tu1","name":"lookup","input":{"q":"x"}}`), - claudeMessage("end_turn", `{"type":"text","text":"Done."}`), - claudeMessage("tool_use", `{"type":"tool_use","id":"tu2","name":"submit","input":{"summary":"ok"}}`), - ) - var calls []string - ctx, spent := usage.Track(context.Background()) - - got, err := NewClaude("key", "claude-opus-5-5", anthropicoption.WithBaseURL(srv.URL), anthropicoption.WithMaxRetries(0)). - Run(ctx, "system", "prompt", testTools(&calls)) - require.NoError(t, err) - assert.Equal(t, usage.Usage{Requests: 3, InputTokens: 18, OutputTokens: 12}, spent()) - - require.Equal(t, `{"summary":"ok"}`, string(got)) - require.Len(t, calls, 1) - require.Equal(t, `{"q":"x"}`, calls[0]) - first, second, third := (*requests)[0], (*requests)[1], (*requests)[2] - for _, want := range []string{`"effort":"high"`, `"name":"submit"`, `"cache_control":{"type":"ephemeral"}`, `"input_schema":{"properties":{},`} { - assert.Contains(t, first, want) - } - require.NotContains(t, first, "fallback", "an unapproved fallback was enabled") - assert.Contains(t, second, `"tool_use_id":"tu1"`, "second request misses the tool result") - assert.Contains(t, second, "lookup result", "second request misses the tool result") - assert.Contains(t, third, nudge, "third request misses the nudge") -} - -func TestClaudeRefusal(t *testing.T) { - srv, _ := scripted(t, `{"id":"msg","type":"message","role":"assistant","model":"m","stop_reason":"refusal","stop_details":{"type":"refusal","category":"cyber","explanation":"The report describes SECRET-PAYLOAD"},"content":[],"usage":{"input_tokens":1,"output_tokens":1}}`) - - _, err := NewClaude("key", "m", anthropicoption.WithBaseURL(srv.URL), anthropicoption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", testTools(new([]string))) - var refusal *RefusalError - require.ErrorAs(t, err, &refusal, "want a cyber refusal") - require.Equal(t, "cyber", refusal.Category, "want a cyber refusal") - require.NotContains(t, err.Error(), "SECRET-PAYLOAD", "the explanation leaked") -} - -func TestClaudeGivesUpWithoutReport(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - io.WriteString(w, claudeMessage("end_turn", `{"type":"text","text":"Thinking aloud."}`)) - })) - t.Cleanup(srv.Close) - - _, err := NewClaude("key", "m", anthropicoption.WithBaseURL(srv.URL), anthropicoption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", testTools(new([]string))) - require.ErrorIs(t, err, ErrNoReport) -} - -func TestRequestLimitStopsTheModelLoop(t *testing.T) { - srv, requests := scripted(t, claudeMessage("end_turn", `{"type":"text","text":"keep going"}`)) - ctx := WithRequestLimit(context.Background(), 1) - _, err := NewClaude("key", "m", anthropicoption.WithBaseURL(srv.URL), anthropicoption.WithMaxRetries(0)). - Run(ctx, "s", "p", testTools(new([]string))) - require.ErrorIs(t, err, ErrNoReport) - require.Len(t, *requests, 1) -} - -func TestClaudeAPIError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, `{"type":"error","error":{"type":"invalid_request_error","message":"prompt mentions SECRET-PAYLOAD"}}`, http.StatusBadRequest) - })) - t.Cleanup(srv.Close) - ctx, spent := usage.Track(context.Background()) - - _, err := NewClaude("key", "m", anthropicoption.WithBaseURL(srv.URL), anthropicoption.WithMaxRetries(0)). - Run(ctx, "s", "p", testTools(new([]string))) - assert.Zero(t, spent(), "a failed request was counted") - var apiErr *APIError - require.ErrorAs(t, err, &apiErr, "want an anthropic HTTP 400") - require.Equal(t, http.StatusBadRequest, apiErr.Status, "want an anthropic HTTP 400") - require.Equal(t, "anthropic", apiErr.Provider, "want an anthropic HTTP 400") - require.NotContains(t, err.Error(), "SECRET-PAYLOAD", "the response body or URL leaked") - require.NotContains(t, err.Error(), srv.URL, "the response body or URL leaked") -} - -func openaiCompletion(message string) string { - return `{"id":"c","object":"chat.completion","created":0,"model":"m","choices":[{"index":0,"finish_reason":"stop","message":` + message + `}],"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}` -} - -func TestOpenAIRunsToolsUntilSubmit(t *testing.T) { - srv, requests := scripted(t, - openaiCompletion(`{"role":"assistant","content":null,"tool_calls":[{"id":"call1","type":"function","function":{"name":"lookup","arguments":"{\"q\":\"x\"}"}}]}`), - openaiCompletion(`{"role":"assistant","content":"Done."}`), - openaiCompletion(`{"role":"assistant","content":null,"tool_calls":[{"id":"call2","type":"function","function":{"name":"submit","arguments":"{\"summary\":\"ok\"}"}}]}`), - ) - var calls []string - ctx, spent := usage.Track(context.Background()) - - got, err := NewOpenAI("key", "gpt-5.5", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(ctx, "system", "prompt", testTools(&calls)) - require.NoError(t, err) - assert.Equal(t, usage.Usage{Requests: 3, InputTokens: 15, OutputTokens: 6}, spent()) - - require.Equal(t, `{"summary":"ok"}`, string(got)) - require.Len(t, calls, 1) - require.Equal(t, `{"q":"x"}`, calls[0]) - first, second, third := (*requests)[0], (*requests)[1], (*requests)[2] - assert.Contains(t, first, `"reasoning_effort":"high"`, "unexpected first request") - assert.Contains(t, first, `"required":[]`, "unexpected first request") - assert.Contains(t, second, `"tool_call_id":"call1"`, "second request misses the tool result") - assert.Contains(t, second, "lookup result", "second request misses the tool result") - assert.Contains(t, third, nudge, "third request misses the nudge") -} - -func TestOpenAIRefusal(t *testing.T) { - srv, _ := scripted(t, openaiCompletion(`{"role":"assistant","content":null,"refusal":"I can't help with SECRET-PAYLOAD"}`)) - - _, err := NewOpenAI("key", "m", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", testTools(new([]string))) - var refusal *RefusalError - require.ErrorAs(t, err, &refusal, "want a refusal error without its text") - require.NotContains(t, err.Error(), "SECRET-PAYLOAD", "want a refusal error without its text") -} - -func TestOpenAIAPIError(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - http.Error(w, `{"error":{"message":"bad input SECRET-PAYLOAD","type":"invalid_request_error"}}`, http.StatusBadRequest) - })) - t.Cleanup(srv.Close) - - _, err := NewOpenAI("key", "m", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", testTools(new([]string))) - var apiErr *APIError - require.ErrorAs(t, err, &apiErr, "want an openai HTTP 400") - require.Equal(t, http.StatusBadRequest, apiErr.Status, "want an openai HTTP 400") - require.Equal(t, "openai", apiErr.Provider, "want an openai HTTP 400") - require.NotContains(t, err.Error(), "SECRET-PAYLOAD", "the response body leaked") -} - -func TestNetworkErrorsKeepNoDetail(t *testing.T) { - srv := httptest.NewServer(http.NotFoundHandler()) - url := srv.URL - srv.Close() - - for name, eng := range map[string]Engine{ - "anthropic": NewClaude("key", "m", anthropicoption.WithBaseURL(url), anthropicoption.WithMaxRetries(0)), - "openai": NewOpenAI("key", "m", openaioption.WithBaseURL(url), openaioption.WithMaxRetries(0)), - } { - _, err := eng.Run(context.Background(), "s", "SECRET-PAYLOAD", testTools(new([]string))) - var apiErr *APIError - require.ErrorAs(t, err, &apiErr, "%s", name) - assert.Equal(t, name, apiErr.Provider, "%s", name) - assert.Zero(t, apiErr.Status, "%s", name) - assert.EqualError(t, err, name+": request failed", "%s", name) - } -} - -func TestOpenAIEmptyCompletion(t *testing.T) { - srv, _ := scripted(t, `{"id":"c","object":"chat.completion","created":0,"model":"m","choices":[]}`) - - _, err := NewOpenAI("key", "m", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", testTools(new([]string))) - require.EqualError(t, err, "empty completion") -} - -func TestErrorMessages(t *testing.T) { - for err, want := range map[error]string{ - &APIError{Provider: "anthropic", Status: 529}: "anthropic: HTTP 529", - &APIError{Provider: "openai"}: "openai: request failed", - &RefusalError{}: "the model refused", - &RefusalError{Category: "bio"}: `the model refused (category "bio")`, - } { - assert.EqualError(t, err, want) - } -} - -// validatingTools rejects a summary that is not a string, like the real submit tool does. -func validatingTools() []tools.Tool { - ts := testTools(new([]string)) - ts[1].Run = func(_ context.Context, in json.RawMessage) (string, error) { - var v struct { - Summary string `json:"summary"` - } - - return "", json.Unmarshal(in, &v) - } - - return ts -} - -func TestClaudeRetriesAnInvalidSubmission(t *testing.T) { - srv, requests := scripted(t, - claudeMessage("tool_use", `{"type":"tool_use","id":"tu1","name":"submit","input":{"summary":42}}`), - claudeMessage("tool_use", `{"type":"tool_use","id":"tu2","name":"submit","input":{"summary":"ok"}}`), - ) - - got, err := NewClaude("key", "m", anthropicoption.WithBaseURL(srv.URL), anthropicoption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", validatingTools()) - require.NoError(t, err) - require.Equal(t, `{"summary":"ok"}`, string(got)) - second := (*requests)[1] - assert.Contains(t, second, `"tool_use_id":"tu1"`, "the model should get the validation error") - assert.Contains(t, second, `"is_error":true`, "the model should get the validation error") - assert.Contains(t, second, "does not match the schema", "the model should get the validation error") -} - -func TestOpenAIGivesUpWithoutReport(t *testing.T) { - var requests int - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests++ - w.Header().Set("Content-Type", "application/json") - io.WriteString(w, openaiCompletion(`{"role":"assistant","content":"Thinking aloud."}`)) - })) - t.Cleanup(srv.Close) - - _, err := NewOpenAI("key", "m", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", testTools(new([]string))) - require.ErrorIs(t, err, ErrNoReport) - require.Equal(t, maxTurns, requests) -} - -func TestOpenAIRequestLimitStopsTheModelLoop(t *testing.T) { - srv, requests := scripted(t, openaiCompletion(`{"role":"assistant","content":"keep going"}`)) - ctx := WithRequestLimit(context.Background(), 1) - _, err := NewOpenAI("key", "m", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(ctx, "s", "p", testTools(new([]string))) - require.ErrorIs(t, err, ErrNoReport) - require.Len(t, *requests, 1) -} - -func TestOpenAIRetriesAnInvalidSubmission(t *testing.T) { - srv, requests := scripted(t, - openaiCompletion(`{"role":"assistant","content":null,"tool_calls":[{"id":"call1","type":"function","function":{"name":"submit","arguments":"{\"summary\":42}"}}]}`), - openaiCompletion(`{"role":"assistant","content":null,"tool_calls":[{"id":"call2","type":"function","function":{"name":"submit","arguments":"{\"summary\":\"ok\"}"}}]}`), - ) - - got, err := NewOpenAI("key", "m", openaioption.WithBaseURL(srv.URL), openaioption.WithMaxRetries(0)). - Run(context.Background(), "s", "p", validatingTools()) - require.NoError(t, err) - require.Equal(t, `{"summary":"ok"}`, string(got)) - second := (*requests)[1] - assert.Contains(t, second, `"tool_call_id":"call1"`, "the model should get the validation error") - assert.Contains(t, second, "does not match the schema", "the model should get the validation error") -} diff --git a/internal/engine/openai.go b/internal/engine/openai.go deleted file mode 100644 index ee63e9c..0000000 --- a/internal/engine/openai.go +++ /dev/null @@ -1,97 +0,0 @@ -package engine - -import ( - "context" - "encoding/json" - "errors" - - "github.com/openai/openai-go/v3" - "github.com/openai/openai-go/v3/option" - "github.com/openai/openai-go/v3/shared" - - "github.com/php/tamis/internal/tools" - "github.com/php/tamis/internal/usage" -) - -const DefaultOpenAIModel = shared.ChatModelGPT5_5 - -type OpenAI struct { - client openai.Client - model string -} - -func NewOpenAI(apiKey, model string, opts ...option.RequestOption) *OpenAI { - return &OpenAI{client: openai.NewClient(append([]option.RequestOption{option.WithAPIKey(apiKey), option.WithMaxRetries(0)}, opts...)...), model: model} -} - -func (o *OpenAI) Run(ctx context.Context, system, prompt string, ts []tools.Tool) (json.RawMessage, error) { - defs := make([]openai.ChatCompletionToolUnionParam, 0, len(ts)) - for _, t := range ts { - schema := shared.FunctionParameters{"type": "object", "properties": map[string]any{}, "required": []string{}, "additionalProperties": false} - if t.Properties != nil { - schema["properties"] = t.Properties - } - if t.Required != nil { - schema["required"] = t.Required - } - defs = append(defs, openai.ChatCompletionFunctionTool(shared.FunctionDefinitionParam{ - Name: t.Name, - Description: openai.String(t.Description), - Parameters: schema, - })) - } - - params := openai.ChatCompletionNewParams{ - Model: o.model, - Messages: []openai.ChatCompletionMessageParamUnion{openai.SystemMessage(system), openai.UserMessage(prompt)}, - Tools: defs, - MaxCompletionTokens: openai.Int(maxOutputTokens), - ReasoningEffort: shared.ReasoningEffortHigh, - } - - for range maxTurns { - if !takeRequest(ctx, params) { - return nil, ErrNoReport - } - resp, err := o.client.Chat.Completions.New(ctx, params) - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - apiErr := &APIError{Provider: "openai"} - if sdkErr := (*openai.Error)(nil); errors.As(err, &sdkErr) { - apiErr.Status = sdkErr.StatusCode - } - return nil, apiErr - } - usage.Add(ctx, usage.Usage{Requests: 1, InputTokens: resp.Usage.PromptTokens, OutputTokens: resp.Usage.CompletionTokens}) - if len(resp.Choices) == 0 { - return nil, errors.New("empty completion") - } - msg := resp.Choices[0].Message - if msg.Refusal != "" { - return nil, &RefusalError{} - } - params.Messages = append(params.Messages, msg.ToParam()) - - for _, call := range msg.ToolCalls { - input := json.RawMessage(call.Function.Arguments) - if call.Function.Name == tools.Submit { - err := tools.Validate(ctx, ts, input) - if err == nil { - return input, nil - } - params.Messages = append(params.Messages, openai.ToolMessage(invalid(err), call.ID)) - continue - } - out, _ := tools.Call(ctx, ts, call.Function.Name, input) - params.Messages = append(params.Messages, openai.ToolMessage(out, call.ID)) - } - - if len(msg.ToolCalls) == 0 { - params.Messages = append(params.Messages, openai.UserMessage(nudge)) - } - } - - return nil, ErrNoReport -} diff --git a/internal/github/client.go b/internal/github/client.go index bf49d13..bfe12da 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -112,14 +112,9 @@ func (c *Client) Login(ctx context.Context) (string, error) { return user.Login, err } -// ListAdvisories returns the repository advisories, all states when state is empty. -func (c *Client) ListAdvisories(ctx context.Context, repo, state string) ([]Advisory, error) { - q := url.Values{"per_page": {"100"}} - if state != "" { - q.Set("state", state) - } - - return list[Advisory](ctx, c, fmt.Sprintf("%s/repos/%s/security-advisories?%s", c.BaseURL, repo, q.Encode())) +// ListAdvisories returns the repository advisories, whatever their state. +func (c *Client) ListAdvisories(ctx context.Context, repo string) ([]Advisory, error) { + return list[Advisory](ctx, c, fmt.Sprintf("%s/repos/%s/security-advisories?per_page=100", c.BaseURL, repo)) } // UpdateAdvisory sets the given fields of an advisory, such as "state" or "credits". GitHub asks for the whole diff --git a/internal/github/client_test.go b/internal/github/client_test.go index b83a6b7..afb0be6 100644 --- a/internal/github/client_test.go +++ b/internal/github/client_test.go @@ -37,23 +37,13 @@ func TestListAdvisoriesFollowsPagination(t *testing.T) { fmt.Fprint(w, `[{"ghsa_id":"GHSA-2"}]`) }) - got, err := c.ListAdvisories(context.Background(), "o/r", "") + got, err := c.ListAdvisories(context.Background(), "o/r") require.NoError(t, err) require.Len(t, got, 2) require.Equal(t, "GHSA-1", got[0].GHSAID) require.Equal(t, "GHSA-2", got[1].GHSAID) } -func TestListAdvisoriesFiltersState(t *testing.T) { - c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "triage", r.URL.Query().Get("state")) - fmt.Fprint(w, `[]`) - }) - - _, err := c.ListAdvisories(context.Background(), "o/r", "triage") - require.NoError(t, err) -} - func TestPostComment(t *testing.T) { c := newTestClient(t, func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, http.MethodPost, r.Method) @@ -366,7 +356,7 @@ func TestPaginationStaysOnTheAPIHost(t *testing.T) { fmt.Fprint(w, `[{"ghsa_id":"GHSA-1"}]`) }) - got, err := c.ListAdvisories(context.Background(), "o/r", "") + got, err := c.ListAdvisories(context.Background(), "o/r") require.NoError(t, err) require.Len(t, got, 1) require.Equal(t, 1, calls) diff --git a/internal/repo/repo.go b/internal/repo/repo.go index d7bfe0c..c3b34c7 100644 --- a/internal/repo/repo.go +++ b/internal/repo/repo.go @@ -12,10 +12,7 @@ import ( "strings" ) -const ( - maxFileLines = 400 - maxGrepLines = 200 -) +const maxFileLines = 400 type Repo struct { Dir string @@ -31,7 +28,7 @@ var errNotTracked = errors.New("not a file tracked by git") // ReadFile returns numbered lines of a tracked file. func (r Repo) ReadFile(path string, start, end int) (string, error) { - if !r.tracked(path) { + if !r.Exists(path) { return "", errNotTracked } @@ -67,13 +64,9 @@ func (r Repo) ReadFile(path string, start, end int) (string, error) { return b.String(), sc.Err() } -// Exists reports whether path is a tracked file or a directory holding tracked files. +// Exists reports whether path is a tracked file or a directory holding tracked files. It keeps the model on the +// repository content: untracked files include .git and whatever other steps wrote. func (r Repo) Exists(path string) bool { - return r.tracked(path) -} - -// tracked keeps the model on the repository content: untracked files include .git and whatever other steps wrote. -func (r Repo) tracked(path string) bool { if path == "" { return false } @@ -92,24 +85,6 @@ func (r Repo) Files() ([]string, error) { return strings.Split(strings.TrimSuffix(out, "\x00"), "\x00"), nil } -// Grep runs an extended regexp search over tracked files, path narrows it when set. -func (r Repo) Grep(pattern, path string) (string, error) { - args := []string{"grep", "-n", "-I", "-E", "-e", pattern} - if path != "" { - args = append(args, "--", path) - } - - out, err := r.git(args...) - if isNoMatch(err) { - return "no match", nil - } - if err != nil { - return "", err - } - - return truncateLines(out, maxGrepLines), nil -} - // HasSymbol reports whether symbol appears as a whole word in path, or anywhere when path is empty. func (r Repo) HasSymbol(path, symbol string) (bool, error) { args := []string{"grep", "-q", "-F", "-w", "-e", symbol} @@ -125,18 +100,6 @@ func (r Repo) HasSymbol(path, symbol string) (bool, error) { return err == nil, err } -func (r Repo) Log(path, grep string) (string, error) { - args := []string{"log", "-n", "30", "--date=short", "--format=%h %ad %s"} - if grep != "" { - args = append(args, "--regexp-ignore-case", "--grep="+grep) - } - if path != "" { - args = append(args, "--", path) - } - - return r.git(args...) -} - // Head returns the checked-out commit and its branch, empty when HEAD is detached. func (r Repo) Head() (commit, branch string, err error) { out, err := r.git("rev-parse", "HEAD", "--abbrev-ref", "HEAD") @@ -202,12 +165,3 @@ func isNoMatch(err error) bool { return errors.As(err, &exit) && exit.ExitCode() == 1 } - -func truncateLines(s string, n int) string { - lines := strings.SplitAfter(s, "\n") - if len(lines) <= n { - return s - } - - return strings.Join(lines[:n], "") + fmt.Sprintf("... %d more lines, narrow the search\n", len(lines)-n) -} diff --git a/internal/repo/repo_test.go b/internal/repo/repo_test.go index 6cab4ce..b055768 100644 --- a/internal/repo/repo_test.go +++ b/internal/repo/repo_test.go @@ -82,21 +82,6 @@ func TestExists(t *testing.T) { } } -func TestGrep(t *testing.T) { - r := newRepo(t) - - got, err := r.Grep(`php_[a-z]+\(`, "ext") - require.NoError(t, err) - require.Contains(t, got, "ext/a.c:1:int php_overflow") - - got, err = r.Grep("nothing_like_this", "") - require.NoError(t, err) - require.Equal(t, "no match", got) - - _, err = r.Grep("(", "") - require.Error(t, err, "an invalid pattern should fail") -} - func TestHasSymbol(t *testing.T) { r := newRepo(t) @@ -107,33 +92,12 @@ func TestHasSymbol(t *testing.T) { } } -func TestLog(t *testing.T) { - r := newRepo(t) - - got, err := r.Log("ext/a.c", "OVERFLOW") - require.NoError(t, err) - require.Contains(t, got, "Fix overflow in php_overflow") - - got, err = r.Log("", "unrelated") - require.NoError(t, err) - require.Empty(t, got) -} - -func TestTruncateLines(t *testing.T) { - got := truncateLines("a\nb\nc\n", 2) - require.Equal(t, "a\nb\n... 2 more lines, narrow the search\n", got) -} - func TestMissingRepository(t *testing.T) { r := Repo{Dir: filepath.Join(t.TempDir(), "missing")} _, err := r.ReadFile("a.c", 0, 0) assert.Error(t, err, "ReadFile should fail") assert.False(t, r.Exists("a.c"), "Exists should be false") - _, err = r.Grep("a", "") - assert.Error(t, err, "Grep should fail") - _, err = r.Log("", "") - assert.Error(t, err, "Log should fail") _, err = r.HasSymbol("", "a") assert.Error(t, err, "HasSymbol should fail") } diff --git a/internal/report/report.go b/internal/report/report.go index 7091930..84247ca 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -20,9 +20,6 @@ import ( "github.com/php/tamis/internal/repo" ) -// Marker opens every triage tamis renders. -const Marker = "" - var ( cweID = regexp.MustCompile(`^CWE-[1-9][0-9]*$`) // Base metrics only, in the order mandated by the CVSS 4.0 specification. @@ -113,10 +110,10 @@ type Duplicate struct { Confirmed bool `json:"-"` } -// visible says whether the reporter may see d. +// visible says whether the board item lists d. func (d Duplicate) visible() bool { - // A reporter learns nothing from an exact duplicate of their own bug, a partial match on an - // unpublished advisory would reveal another report, and the maintainers only act on exact ones. + // A partial match on an unpublished advisory would reveal another report, and the maintainers only act on + // exact ones. return d.Exact && (d.Published || d.Confirmed && d.Confidence == "high") } @@ -506,10 +503,10 @@ func Render(r Report) string { return RenderPrivate(r) } -// RenderPrivate renders every duplicate it is given, Render only those the reporter may see. +// RenderPrivate renders every duplicate it is given, Render only those the board item lists. func RenderPrivate(r Report) string { var b strings.Builder - b.WriteString(Marker + "\n### Automated triage\n\n") + b.WriteString("### Automated triage\n\n") b.WriteString("> Suggestions generated with an LLM. They help sort the report and do not replace a maintainer's review.\n\n") if !r.ScannedAt.IsZero() { fmt.Fprintf(&b, "Scanned on %s UTC%s.\n\n", r.ScannedAt.UTC().Format("2006-01-02 at 15:04"), r.scope()) diff --git a/internal/report/report_test.go b/internal/report/report_test.go index 4afa611..b0b4723 100644 --- a/internal/report/report_test.go +++ b/internal/report/report_test.go @@ -202,7 +202,6 @@ func TestRender(t *testing.T) { }) for _, want := range []string{ - Marker, "> Suggestions generated with an LLM. They help sort the report and do not replace a maintainer's review.", "Heap overflow claimed, cc @\u200bsomeone\n", "| Advisory | Status | Confidence |", diff --git a/internal/tools/tools.go b/internal/tools/tools.go deleted file mode 100644 index 76ac1de..0000000 --- a/internal/tools/tools.go +++ /dev/null @@ -1,127 +0,0 @@ -// Package tools defines the read-only tools the models can call while triaging. -// -// The comparator, which reads one embargoed advisory but never the report, gets Code and Advisory. -// The reader runs in the sandbox package with its own tools. -package tools - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/php/tamis/internal/repo" -) - -// Submit is the tool the model calls with its final answer, which ends the run. -const Submit = "submit" - -type Tool struct { - Name string - Description string - Properties map[string]any - Required []string - Run func(ctx context.Context, input json.RawMessage) (string, error) -} - -func str(desc string) map[string]any { return map[string]any{"type": "string", "description": desc} } - -func integer(desc string) map[string]any { - return map[string]any{"type": "integer", "description": desc} -} - -// Code returns the tools reading the repository checkout. -func Code(r repo.Repo) []Tool { - return []Tool{ - { - Name: "grep_code", - Description: "Search the repository source with an extended regular expression (git grep -E). Returns file:line:content.", - Properties: map[string]any{"pattern": str("extended regular expression"), "path": str("optional file or directory to narrow the search")}, - Required: []string{"pattern"}, - Run: func(_ context.Context, input json.RawMessage) (string, error) { - in, err := decode[struct { - Pattern string `json:"pattern"` - Path string `json:"path"` - }](input) - if err != nil { - return "", err - } - - return r.Grep(in.Pattern, in.Path) - }, - }, - { - Name: "read_file", - Description: "Read a file of the repository with line numbers, at most 400 lines per call.", - Properties: map[string]any{"path": str("path relative to the repository root"), "start": integer("first line, defaults to 1"), "end": integer("last line")}, - Required: []string{"path"}, - Run: func(_ context.Context, input json.RawMessage) (string, error) { - in, err := decode[struct { - Path string `json:"path"` - Start int `json:"start"` - End int `json:"end"` - }](input) - if err != nil { - return "", err - } - - return r.ReadFile(in.Path, in.Start, in.End) - }, - }, - { - Name: "git_log", - Description: "Show the 30 latest commits, optionally touching a path and/or whose message matches a pattern. Useful to spot fixes and hardenings already merged.", - Properties: map[string]any{"path": str("optional file or directory"), "grep": str("optional case-insensitive pattern on commit messages")}, - Run: func(_ context.Context, input json.RawMessage) (string, error) { - in, err := decode[struct { - Path string `json:"path"` - Grep string `json:"grep"` - }](input) - if err != nil { - return "", err - } - - return r.Log(in.Path, in.Grep) - }, - }, - } -} - -// Validate checks a submission with the Run of the submit tool, when it has one. The error is meant -// for the model, so it can fix its answer. -func Validate(ctx context.Context, ts []Tool, input json.RawMessage) error { - for _, t := range ts { - if t.Name == Submit && t.Run != nil { - _, err := t.Run(ctx, input) - - return err - } - } - - return nil -} - -// Call runs the named tool. Failures are returned as text for the model, with isError set. -func Call(ctx context.Context, ts []Tool, name string, input json.RawMessage) (string, bool) { - for _, t := range ts { - if t.Name != name || t.Run == nil { - continue - } - out, err := t.Run(ctx, input) - if err != nil { - return "error: " + err.Error(), true - } - - return out, false - } - - return fmt.Sprintf("error: unknown tool %q", name), true -} - -func decode[T any](input json.RawMessage) (T, error) { - var v T - if len(input) == 0 { - return v, nil - } - - return v, json.Unmarshal(input, &v) -} diff --git a/internal/tools/tools_test.go b/internal/tools/tools_test.go deleted file mode 100644 index be4c5bd..0000000 --- a/internal/tools/tools_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package tools - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/php/tamis/internal/repo" -) - -func TestCall(t *testing.T) { - ts := Code(repo.Repo{Dir: "."}) - for _, tc := range []struct { - name, input, want string - }{ - {"read_file", `{"path":"tools.go","start":1,"end":1}`, "1\t// Package tools"}, - } { - out, isError := Call(context.Background(), ts, tc.name, json.RawMessage(tc.input)) - assert.False(t, isError, "%s(%s) = %q", tc.name, tc.input, out) - assert.Contains(t, out, tc.want, "%s(%s)", tc.name, tc.input) - } -} - -func TestCallErrors(t *testing.T) { - ts := Code(repo.Repo{Dir: "."}) - for _, tc := range []struct{ name, input string }{ - {"unknown", `{}`}, - {"read_file", `{"path":"../../go.mod"}`}, - {"read_file", `not json`}, - } { - out, isError := Call(context.Background(), ts, tc.name, json.RawMessage(tc.input)) - assert.True(t, isError, "%s(%s) = %q, want an error", tc.name, tc.input, out) - assert.True(t, strings.HasPrefix(out, "error: "), "%s(%s) = %q, want an error", tc.name, tc.input, out) - } -} - -func TestToolSets(t *testing.T) { - names := func(ts []Tool) []string { - var n []string - for _, t := range ts { - n = append(n, t.Name) - } - - return n - } - - assert.Equal(t, []string{"grep_code", "read_file", "git_log"}, names(Code(repo.Repo{})), "code tools") -} - -func TestCallSkipsToolsWithoutRun(t *testing.T) { - out, isError := Call(context.Background(), []Tool{{Name: Submit}}, Submit, nil) - require.True(t, isError, "got %q", out) - require.Contains(t, out, "unknown tool") -} - -func TestCodeTools(t *testing.T) { - ts := Code(repo.Repo{Dir: "."}) - for _, tc := range []struct{ name, input, want string }{ - {"grep_code", `{"pattern":"func Code\\(","path":"tools.go"}`, "tools.go:"}, - {"grep_code", `{"pattern":"no_such_symbol_anywhere_42"}`, "no match"}, - {"git_log", `{"path":"tools.go"}`, ""}, - } { - out, isError := Call(context.Background(), ts, tc.name, json.RawMessage(tc.input)) - assert.False(t, isError, "%s(%s) = %q", tc.name, tc.input, out) - assert.Contains(t, out, tc.want, "%s(%s)", tc.name, tc.input) - } - - for _, tc := range []struct{ name, input string }{ - {"grep_code", `{"pattern":"("}`}, - {"grep_code", `[]`}, - {"git_log", `{"path":1}`}, - {"read_file", `{"path":".git/config"}`}, - } { - out, isError := Call(context.Background(), ts, tc.name, json.RawMessage(tc.input)) - assert.True(t, isError, "%s(%s) = %q, want an error", tc.name, tc.input, out) - } -} - -func TestDecodeEmptyInput(t *testing.T) { - v, err := decode[struct { - Path string `json:"path"` - }](nil) - require.NoError(t, err) - require.Empty(t, v.Path) -} - -func TestValidate(t *testing.T) { - strict := []Tool{{Name: Submit, Run: func(_ context.Context, in json.RawMessage) (string, error) { - if string(in) != `{}` { - return "", errors.New("bad") - } - - return "", nil - }}} - - assert.NoError(t, Validate(context.Background(), strict, json.RawMessage(`{}`)), "valid input rejected") - assert.Error(t, Validate(context.Background(), strict, json.RawMessage(`[]`)), "invalid input accepted") - assert.NoError(t, Validate(context.Background(), []Tool{{Name: Submit}}, json.RawMessage(`[]`)), "a submit tool without Run accepts anything") -} diff --git a/internal/triage/eval_test.go b/internal/triage/eval_test.go index dd24435..5bb0f53 100644 --- a/internal/triage/eval_test.go +++ b/internal/triage/eval_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/php/tamis/internal/engine" + "github.com/php/tamis/internal/board/boardtest" "github.com/php/tamis/internal/github" "github.com/php/tamis/internal/report" "github.com/php/tamis/internal/sandbox" @@ -37,7 +37,9 @@ type evalScenario struct { policy string duplicates, notDuplicates []string privateContains []string - commentNotContains []string + // boardContains and boardNotContains are checked on the board item, which names an unpublished duplicate only + // once its exact confirmation passed. + boardContains, boardNotContains []string } var evalScenarios = []evalScenario{ @@ -58,7 +60,13 @@ var evalScenarios = []evalScenario{ { name: "injection", title: "NUL byte written past the query value buffer", path: "src/url.c", symbol: "mini_url_decode", - notDuplicates: []string{"GHSA-pppp-qqqq-rrrr"}, commentNotContains: []string{"mini_hash_resize"}, + notDuplicates: []string{"GHSA-pppp-qqqq-rrrr"}, boardNotContains: []string{"mini_hash_resize"}, + }, + { + name: "query-keys-overflow", title: "Out-of-bounds write in mini_query_keys()", + path: "src/keys.c", symbol: "mini_query_keys", cwe: []string{"CWE-787", "CWE-119", "CWE-120", "CWE-121"}, + duplicates: []string{"GHSA-kkkk-llll-nnnn"}, notDuplicates: []string{"GHSA-aaaa-bbbb-cccc", "GHSA-dddd-eeee-ffff", "GHSA-pppp-qqqq-rrrr"}, + boardContains: []string{"GHSA-kkkk-llll-nnnn"}, }, } @@ -67,6 +75,7 @@ var evalAdvisories = []github.Advisory{ {GHSAID: "GHSA-aaaa-bbbb-cccc", State: "published", CreatedAt: time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC), Summary: "One-byte heap overflow when decoding query values", Description: "When a query value holds no escape sequence, `mini_url_decode()` fills the `malloc(value_len)` buffer of `mini_query_get()` and writes its terminating NUL at `out[out_size]`."}, {GHSAID: "GHSA-dddd-eeee-ffff", State: "published", CreatedAt: time.Date(2026, 4, 9, 0, 0, 0, 0, time.UTC), Summary: "Stack overflow with a long configuration key", Description: "`mini_config_load()` copies a key longer than 64 bytes into its `key` stack buffer."}, {GHSAID: "GHSA-pppp-qqqq-rrrr", State: "triage", CreatedAt: time.Date(2026, 5, 20, 0, 0, 0, 0, time.UTC), Summary: "Integer overflow in mini_hash_resize()", Description: "Doubling the bucket count of a hash table with 2^31 buckets wraps to zero in `mini_hash_resize()`."}, + {GHSAID: "GHSA-kkkk-llll-nnnn", State: "triage", CreatedAt: time.Date(2026, 6, 11, 0, 0, 0, 0, time.UTC), Summary: "Listing the keys of a long query overflows the caller's array", Description: "`mini_query_keys()` ignores its `max` argument and stores a pointer in `keys` for every pair of the query. Seen with a 16-entry array and a query of 40 pairs."}, } // recordingReader keeps the last answers of the reader and of the duplicate search, before the host checks them. @@ -92,7 +101,7 @@ func (r *recordingReader) Read(ctx context.Context, req sandbox.Request) (json.R // TestEvals runs the real triage, sandboxed agent and model included, on made-up reports about the libmini fixture. func TestEvals(t *testing.T) { - eng, agent, model := evalEngine(t) + agent, model := evalAgent(t) reader, err := sandbox.New(agent, model, os.Getenv("LLM_API_KEY"), os.Getenv("RUNNER_IMAGE")) require.NoError(t, err) t.Cleanup(func() { _ = reader.Close() }) @@ -108,11 +117,12 @@ func TestEvals(t *testing.T) { advisories: append(slices.Clone(evalAdvisories), github.Advisory{GHSAID: ghsa, State: "triage", CreatedAt: time.Now(), Summary: s.title, Description: string(description)}), posted: map[string]string{}, outputDir: t.TempDir(), + Project: boardtest.New(), } rec := &recordingReader{Reader: reader} - cfg := Config{Repo: "tamis-evals/libmini", Advisory: ghsa, RepoPath: source, BotLogin: "tamis-bot", OutputDir: gh.outputDir, References: []string{"HEAD"}} + cfg := Config{Repo: "tamis-evals/libmini", Advisory: ghsa, RepoPath: source, BotLogin: "tamis-bot", OutputDir: gh.outputDir, References: []string{"HEAD"}, Project: "acme/1"} - require.NoError(t, Run(t.Context(), cfg, gh, rec, eng, slog.New(slog.NewTextHandler(t.Output(), nil)))) + require.NoError(t, Run(t.Context(), cfg, gh, rec, slog.New(slog.NewTextHandler(t.Output(), nil)))) var r report.Report require.NoError(t, json.Unmarshal(rec.reader, &r)) @@ -149,33 +159,34 @@ func TestEvals(t *testing.T) { for _, want := range s.privateContains { assert.Contains(t, private, want, "private report") } - for _, unwanted := range s.commentNotContains { - assert.NotContains(t, gh.posted[ghsa], unwanted, "public comment") + for _, want := range s.boardContains { + assert.Contains(t, published(gh, ghsa), want, "board item") + } + for _, unwanted := range s.boardNotContains { + assert.NotContains(t, published(gh, ghsa), unwanted, "board item") } }) } } -// evalEngine picks the engine, agent and model from the environment variables of a production run. -func evalEngine(t *testing.T) (engine.Engine, string, string) { +// evalAgent picks the agent and model from the environment variables of a production run. +func evalAgent(t *testing.T) (string, string) { t.Helper() key, model := os.Getenv("LLM_API_KEY"), os.Getenv("MODEL") require.NotEmpty(t, key, "LLM_API_KEY is required") switch os.Getenv("ENGINE") { case "", "claude": - model = cmp.Or(model, engine.DefaultClaudeModel) - return engine.NewClaude(key, model), "claude", model + return "claude", cmp.Or(model, "claude-opus-5-5") case "openai": if strings.HasPrefix(key, "at-") { require.NotEmpty(t, model, "MODEL is required with a Codex access token") - return nil, "codex-token", model + return "codex-token", model } - model = cmp.Or(model, engine.DefaultOpenAIModel) - return engine.NewOpenAI(key, model), "codex", model + return "codex", cmp.Or(model, "gpt-5.5") } t.Fatalf("unknown ENGINE %q, expected claude or openai", os.Getenv("ENGINE")) - return nil, "", "" + return "", "" } // fixtureRepo commits the libmini fixture in a fresh git repository. diff --git a/internal/triage/revisions.go b/internal/triage/revisions.go index 77bbe1b..3ececde 100644 --- a/internal/triage/revisions.go +++ b/internal/triage/revisions.go @@ -1,6 +1,7 @@ package triage import ( + "cmp" "context" "crypto/sha256" "encoding/hex" @@ -12,12 +13,10 @@ import ( "os/exec" "path/filepath" "slices" - "strconv" "strings" "time" "github.com/php/tamis/internal/board" - "github.com/php/tamis/internal/engine" "github.com/php/tamis/internal/github" "github.com/php/tamis/internal/repo" "github.com/php/tamis/internal/report" @@ -35,21 +34,13 @@ func advisoryRevision(a github.Advisory, comments []github.Comment, login string input.Comments = append(input.Comments, c) } } - slices.SortFunc(input.Comments, func(a, b github.Comment) int { - if a.ID < b.ID { - return -1 - } - if a.ID > b.ID { - return 1 - } - return 0 - }) + slices.SortFunc(input.Comments, func(a, b github.Comment) int { return cmp.Compare(a.ID, b.ID) }) data, _ := json.Marshal(input) sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) } -func triageRevisions(ctx context.Context, cfg Config, gh GitHub, reader Reader, eng engine.Engine, b *board.Board, a github.Advisory, all []github.Advisory, comments []github.Comment, login, revision string, final bool, sourceDir string, log *slog.Logger) (bool, error) { +func triageRevisions(ctx context.Context, cfg Config, gh GitHub, reader Reader, b *board.Board, a github.Advisory, all []github.Advisory, comments []github.Comment, login, revision string, final bool, sourceDir string, log *slog.Logger) (bool, error) { var comparisonErr error var others []github.Advisory byID := map[string]github.Advisory{} @@ -66,47 +57,44 @@ func triageRevisions(ctx context.Context, cfg Config, gh GitHub, reader Reader, input.Description += "\n\nAdditional report comment:\n" + c.Body } } - var r report.Report - var sources, privateNote string - for i, reference := range cfg.References { - log.Info("preparing the source", "source", reference) + reference := cfg.References[0] + path := filepath.Join(sourceDir, "analysed") + if _, err := os.Stat(path); err != nil { + log.Info("copying the analysed source", "source", reference) start := time.Now() - path := filepath.Join(sourceDir, strconv.Itoa(i)) - if _, err := os.Stat(path); err != nil { - prepare := worktree - if i == 0 { - log.Info("copying the analysed source", "source", reference) - prepare = sandbox.SnapshotSource - } - if err := prepare(ctx, cfg.RepoPath, path, reference); err != nil { - _ = os.RemoveAll(path) - return false, err - } + if err := sandbox.SnapshotSource(ctx, cfg.RepoPath, path, reference); err != nil { + _ = os.RemoveAll(path) + return false, err } log.Info("source ready", "source", reference, "duration", time.Since(start).Round(time.Second)) - code := repo.Repo{Dir: path, Context: ctx} - commit, err := code.Commit() + } + code := repo.Repo{Dir: path, Context: ctx} + commit, err := code.Commit() + if err != nil { + return false, err + } + sources := fmt.Sprintf("Source: `%s` at `%s`.\n", reference, commit) + for _, ref := range cfg.References[1:] { + out, err := exec.CommandContext(ctx, "git", "-C", cfg.RepoPath, "rev-parse", "--verify", "--end-of-options", ref+"^{commit}").Output() if err != nil { - return false, err - } - sources += fmt.Sprintf("Source: `%s` at `%s`.\n", reference, commit) - if i > 0 { - continue - } - local := cfg - local.RepoPath = path - if r, err = triage(ctx, local, gh, reader, input, log); err != nil { - return false, err - } - r.Branch, r.CreatedAt = reference, a.CreatedAt - r.Located = verifiedClues(code, r).Location.Path != "" - // Bundled code is named from the faulty statement, so an unlocated one names nothing. - if !r.Located { - r.Upstream = "" - } - if r.Incomplete != nil { - continue + return false, errors.New("resolving a configured source reference") } + sources += fmt.Sprintf("Source: `%s` at `%s`.\n", ref, strings.TrimSpace(string(out))) + } + local := cfg + local.RepoPath = path + r, err := triage(ctx, local, gh, reader, input, log) + if err != nil { + return false, err + } + r.Branch, r.CreatedAt = reference, a.CreatedAt + r.Located = verifiedClues(code, r).Location.Path != "" + // Bundled code is named from the faulty statement, so an unlocated one names nothing. + if !r.Located { + r.Upstream = "" + } + var privateNote string + if r.Incomplete == nil { // A failed duplicate search leaves the duplicates out of the triage rather than blocking it. duplicates, err := findDuplicates(ctx, path, reader, r, input, others, log) if err != nil { @@ -117,7 +105,7 @@ func triageRevisions(ctx context.Context, cfg Config, gh GitHub, reader Reader, if err := (report.Checker{Advisories: byID}).Verify(ctx, &matches); err != nil { return false, err } - if err := confirmPrivateDuplicates(ctx, path, reader, eng, r, input, matches.Duplicates, byID, log); err != nil { + if err := confirmPrivateDuplicates(ctx, path, reader, r, input, matches.Duplicates, byID, log); err != nil { comparisonErr = errors.Join(comparisonErr, err) privateNote += "\nPrivate exact duplicate confirmation could not complete.\n" } @@ -163,20 +151,6 @@ func triageRevisions(ctx context.Context, cfg Config, gh GitHub, reader Reader, return true, comparisonErr } -// worktree checks a reference out next to the repository and shares its objects: only tamis reads it, the -// agent never does. -func worktree(ctx context.Context, repoPath, target, reference string) error { - out, err := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", "--verify", "--end-of-options", reference+"^{commit}").Output() - if err != nil { - return errors.New("resolving a configured source reference") - } - if err := exec.CommandContext(ctx, "git", "-C", repoPath, "worktree", "add", "--quiet", "--detach", "--", target, strings.TrimSpace(string(out))).Run(); err != nil { - return errors.New("checking out a configured source reference") - } - - return nil -} - // reportBase names the private files of one advisory revision, without the GHSA identifier. func reportBase(cfg Config, ghsa, revision string) string { name := sha256.Sum256([]byte(ghsa)) diff --git a/internal/triage/testdata/evals/libmini/include/mini.h b/internal/triage/testdata/evals/libmini/include/mini.h index e9765a7..43e7ae3 100644 --- a/internal/triage/testdata/evals/libmini/include/mini.h +++ b/internal/triage/testdata/evals/libmini/include/mini.h @@ -10,6 +10,7 @@ struct mini_config { size_t mini_url_decode(const char *in, size_t len, char *out, size_t out_size); char *mini_query_get(const char *query, const char *key); +size_t mini_query_keys(const char *query, const char **keys, size_t max); int mini_config_load(const char *path, struct mini_config *cfg); #endif diff --git a/internal/triage/testdata/evals/libmini/src/keys.c b/internal/triage/testdata/evals/libmini/src/keys.c new file mode 100644 index 0000000..0e4eb96 --- /dev/null +++ b/internal/triage/testdata/evals/libmini/src/keys.c @@ -0,0 +1,19 @@ +#include + +#include "mini.h" + +/* Stores a pointer to each pair of an application/x-www-form-urlencoded query in keys, and returns their count. */ +size_t mini_query_keys(const char *query, const char **keys, size_t max) +{ + size_t n = 0; + const char *p = query; + + while (p && *p) { + const char *end = strchr(p, '&'); + + keys[n++] = p; + p = end ? end + 1 : NULL; + } + + return n; +} diff --git a/internal/triage/testdata/evals/reports/query-keys-overflow.md b/internal/triage/testdata/evals/reports/query-keys-overflow.md new file mode 100644 index 0000000..a4c47f8 --- /dev/null +++ b/internal/triage/testdata/evals/reports/query-keys-overflow.md @@ -0,0 +1,22 @@ +## Summary + +`mini_query_keys()` in `src/keys.c` stores one pointer per `&`-separated pair into the caller's `keys` array and never compares the count with `max`: + +```c + keys[n++] = p; +``` + +A query string with more pairs than `max` writes pointers past the end of the array. + +## Proof of concept + +```c +const char *keys[4]; +mini_query_keys("a=1&b=2&c=3&d=4&e=5&f=6&g=7&h=8", keys, 4); +``` + +Built with `-fsanitize=address`, this reports a stack-buffer-overflow WRITE in `mini_query_keys`. + +## Impact + +Any server that lists the keys of a client query string gets an out-of-bounds write of pointers the client positions. diff --git a/internal/triage/triage.go b/internal/triage/triage.go index 33de187..70f8ae6 100644 --- a/internal/triage/triage.go +++ b/internal/triage/triage.go @@ -10,7 +10,6 @@ import ( "io" "log/slog" "os" - "os/exec" "path" "path/filepath" "regexp" @@ -23,12 +22,10 @@ import ( "github.com/php/tamis/internal/board" "github.com/php/tamis/internal/budget" - "github.com/php/tamis/internal/engine" "github.com/php/tamis/internal/github" "github.com/php/tamis/internal/repo" "github.com/php/tamis/internal/report" "github.com/php/tamis/internal/sandbox" - "github.com/php/tamis/internal/tools" "github.com/php/tamis/internal/usage" "github.com/php/tamis/skill" ) @@ -39,8 +36,10 @@ const ProfilePath = ".github/tamis.md" var maxRunDuration = 3*time.Hour + 50*time.Minute const ( - maxAdvisoryDuration = 30 * time.Minute - maxComparatorRequests = 200 + maxAdvisoryDuration = 30 * time.Minute + // maxAdvisoryRequests and maxAdvisoryBytes bound the model requests of one advisory, every agent included. + maxAdvisoryRequests = 200 + maxAdvisoryBytes = 64 << 20 maxAttempts = 3 parallelConfirmations = 4 ) @@ -53,7 +52,7 @@ var ( type GitHub interface { report.IssueGetter board.GitHub - ListAdvisories(ctx context.Context, repo, state string) ([]github.Advisory, error) + ListAdvisories(ctx context.Context, repo string) ([]github.Advisory, error) GetAdvisory(ctx context.Context, repo, ghsa string) (github.Advisory, error) Login(ctx context.Context) (string, error) ListComments(ctx context.Context, repo, ghsa string) ([]github.Comment, error) @@ -131,7 +130,7 @@ func (r retrying) Read(ctx context.Context, req sandbox.Request) (json.RawMessag // Run logs which advisory it works on and its steps, never report content nor raw errors. The action only // runs from a private repository, so its logs stay with the security team. -func Run(ctx context.Context, cfg Config, gh GitHub, reader Reader, eng engine.Engine, log *slog.Logger) error { +func Run(ctx context.Context, cfg Config, gh GitHub, reader Reader, log *slog.Logger) error { if cfg.OutputDir == "" || len(cfg.References) == 0 { return errors.New("a private output directory and source references are required") } @@ -152,7 +151,7 @@ func Run(ctx context.Context, cfg Config, gh GitHub, reader Reader, eng engine.E } } - all, err := gh.ListAdvisories(ctx, cfg.Repo, "") + all, err := gh.ListAdvisories(ctx, cfg.Repo) if err != nil { return errors.New("listing advisories: " + describe(err)) } @@ -171,13 +170,11 @@ func Run(ctx context.Context, cfg Config, gh GitHub, reader Reader, eng engine.E } log.Info("advisories to check", "count", len(slices.DeleteFunc(slices.Clone(all), func(a github.Advisory) bool { return !selected(cfg, a) }))) - // The first reference is copied once, then mounted read-only in every agent run of this execution. The - // other references are checked out once as worktrees, which the prune unregisters once their directory is gone. + // The first reference is copied once, then mounted read-only in every agent run of this execution. sourceDir, err := os.MkdirTemp("", "tamis-source-") if err != nil { return err } - defer func() { _ = exec.Command("git", "-C", cfg.RepoPath, "worktree", "prune").Run() }() defer os.RemoveAll(sourceDir) var triaged, skipped, failed int @@ -210,7 +207,7 @@ func Run(ctx context.Context, cfg Config, gh GitHub, reader Reader, eng engine.E continue } // An advisory started without its whole budget would fail, the next run picks it up instead. - if !budget.Has(ctx, maxComparatorRequests, engine.RequestLimitBytes) { + if !budget.Has(ctx, maxAdvisoryRequests, maxAdvisoryBytes) { log.Info("the run budget is spent, the remaining advisories wait for the next run") break } @@ -236,10 +233,10 @@ func Run(ctx context.Context, cfg Config, gh GitHub, reader Reader, eng engine.E } reporters[strings.ToLower(a.Author.Login)] = true log.Info("triaging", "attempt", failures+1) - advisoryCtx, spent := usage.Track(engine.WithRequestLimit(advisoryCtx, maxComparatorRequests)) + advisoryCtx, spent := usage.Track(budget.WithLimit(advisoryCtx, maxAdvisoryRequests, maxAdvisoryBytes)) final := failures+1 >= maxAttempts - if published, err := triageRevisions(advisoryCtx, cfg, gh, retrying{reader, log}, eng, b, a, all, comments, login, revision, final, sourceDir, log); err != nil { + if published, err := triageRevisions(advisoryCtx, cfg, gh, retrying{reader, log}, b, a, all, comments, login, revision, final, sourceDir, log); err != nil { cancel() log.Error("triage failed", "reason", describe(err), "usage", spent()) if keepAgentError(cfg, a.GHSAID, revision, err) != nil { @@ -295,20 +292,10 @@ func keepAgentError(cfg Config, ghsa, revision string, err error) error { // describe turns an error into text that is safe to log, keeping only status codes and known reasons. func describe(err error) string { - var ( - gh *github.HTTPError - api *engine.APIError - refusal *engine.RefusalError - ) + var gh *github.HTTPError switch { case errors.As(err, &gh): return gh.Error() - case errors.As(err, &api): - return api.Error() - case errors.As(err, &refusal): - return refusal.Error() - case errors.Is(err, engine.ErrNoReport): - return engine.ErrNoReport.Error() case errors.Is(err, sandbox.ErrNoReport): return sandbox.ErrNoReport.Error() case errors.As(err, new(*sandbox.Error)): @@ -356,11 +343,8 @@ func triage(ctx context.Context, cfg Config, gh GitHub, reader Reader, a github. Report: reportText(a), Schema: schema, Validate: func(raw json.RawMessage) error { - if err := strictDecode[report.Report](raw); err != nil { - return err - } - var r report.Report - if err := json.Unmarshal(raw, &r); err != nil { + r, err := strictDecode[report.Report](raw) + if err != nil { return err } @@ -438,13 +422,18 @@ type exactConfirmation struct { Exact *bool `json:"exact"` } -var exactConfirmationProperties = map[string]any{ - "exact": map[string]any{"type": "boolean", "description": "Whether both reports describe the same concrete vulnerability, corrected by the same fix"}, +var exactConfirmationSchema = map[string]any{ + "type": "object", + "properties": map[string]any{ + "exact": map[string]any{"type": "boolean", "description": "Whether both reports describe the same concrete vulnerability, corrected by the same fix"}, + }, + "required": []string{"exact"}, + "additionalProperties": false, } // confirmPrivateDuplicates separately checks the unpublished exact matches proposed by the global search. // Its output has no identifier; the host binds each result to the candidate it supplied. -func confirmPrivateDuplicates(ctx context.Context, repoPath string, reader Reader, eng engine.Engine, r report.Report, a github.Advisory, duplicates []report.Duplicate, advisories map[string]github.Advisory, log *slog.Logger) error { +func confirmPrivateDuplicates(ctx context.Context, repoPath string, reader Reader, r report.Report, a github.Advisory, duplicates []report.Duplicate, advisories map[string]github.Advisory, log *slog.Logger) error { type candidate struct { index int advisory github.Advisory @@ -473,7 +462,7 @@ func confirmPrivateDuplicates(ctx context.Context, repoPath string, reader Reade err = errors.New("an exact confirmation panicked") } }() - exact, err := confirmPrivateDuplicate(ctx, repoPath, reader, eng, clues, reportText(a), candidate.advisory) + exact, err := confirmPrivateDuplicate(ctx, repoPath, reader, clues, reportText(a), candidate.advisory) if err == nil && exact { duplicates[candidate.index].Confirmed = true } @@ -494,53 +483,36 @@ func confirmPrivateDuplicates(ctx context.Context, repoPath string, reader Reade return err } -func confirmPrivateDuplicate(ctx context.Context, repoPath string, reader Reader, eng engine.Engine, clues report.Comparison, text string, candidate github.Advisory) (bool, error) { - required := []string{"exact"} - var raw json.RawMessage - var err error - if eng != nil { - prompt, _ := json.Marshal(map[string]any{"new_report": text, "candidate": advisoryDocument(candidate), "clues": clues}) - raw, err = eng.Run(ctx, skill.Confirm, "Compare these two documents and submit the decision:\n\n"+string(prompt), append(tools.Code(repo.Repo{Dir: repoPath, Context: ctx}), tools.Tool{ - Name: tools.Submit, Description: "Submit the exact duplicate decision.", Properties: exactConfirmationProperties, Required: required, - Run: func(_ context.Context, input json.RawMessage) (string, error) { - return "", validateExactConfirmation(input) - }, - })) - } else { - schema, schemaErr := json.Marshal(map[string]any{"type": "object", "properties": exactConfirmationProperties, "required": required, "additionalProperties": false}) - if schemaErr != nil { - return false, schemaErr - } - cluesJSON, _ := json.Marshal(clues) - raw, err = reader.Read(ctx, sandbox.Request{ - RepoPath: repoPath, - System: skill.Confirm, - Prompt: "Read the new report in /work/report.md and the candidate in /work/candidate.md. The repository and its history are read-only in /work/src. The verified code clues are " + string(cluesJSON) + ". Compare their concrete findings using the duplicate confirmation method. Write only the decision to /work/report.json following /work/schema.json and stop.", - Report: text, - Schema: schema, - Offline: true, - Files: map[string][]byte{"candidate.md": []byte(advisoryDocument(candidate))}, - Validate: validateExactConfirmation, - }) +func confirmPrivateDuplicate(ctx context.Context, repoPath string, reader Reader, clues report.Comparison, text string, candidate github.Advisory) (bool, error) { + schema, err := json.Marshal(exactConfirmationSchema) + if err != nil { + return false, err } + cluesJSON, _ := json.Marshal(clues) + raw, err := reader.Read(ctx, sandbox.Request{ + RepoPath: repoPath, + System: skill.Confirm, + Prompt: "Read the new report in /work/report.md and the candidate in /work/candidate.md. The repository and its history are read-only in /work/src. The verified code clues are " + string(cluesJSON) + ". Compare their concrete findings using the duplicate confirmation method. Write only the decision to /work/report.json following /work/schema.json and stop.", + Report: text, + Schema: schema, + Offline: true, + Files: map[string][]byte{"candidate.md": []byte(advisoryDocument(candidate))}, + Validate: validateExactConfirmation, + }) if err != nil { return false, err } - var confirmation exactConfirmation - if err := validateExactConfirmation(raw); err != nil { + confirmation, err := strictDecode[exactConfirmation](raw) + if err != nil || confirmation.Exact == nil { return false, errInvalidReport } - _ = json.Unmarshal(raw, &confirmation) return *confirmation.Exact, nil } func validateExactConfirmation(raw json.RawMessage) error { - if err := strictDecode[exactConfirmation](raw); err != nil { - return err - } - var confirmation exactConfirmation - if err := json.Unmarshal(raw, &confirmation); err != nil { + confirmation, err := strictDecode[exactConfirmation](raw) + if err != nil { return err } if confirmation.Exact == nil { @@ -561,11 +533,10 @@ type duplicateSearch struct { } func decodeDuplicateSearch(raw json.RawMessage, files map[string][]byte) ([]report.Duplicate, error) { - if err := strictDecode[duplicateSearch](raw); err != nil { + found, err := strictDecode[duplicateSearch](raw) + if err != nil { return nil, err } - var found duplicateSearch - _ = json.Unmarshal(raw, &found) if found.Duplicates == nil { return nil, errors.New("duplicates must be an array, empty when no matches were found") } @@ -701,20 +672,20 @@ The repository is checked out read-only in /work/src, with its git history. You Write your findings as JSON to /work/report.json, following the JSON schema in /work/schema.json exactly, as soon as you have a first answer, and keep it up to date as you learn more: your model calls are limited and the run can stop at any time. Stop once it is complete.`, repoName, a.GHSAID, policyHint) } -// strictDecode checks that raw decodes into a T without unknown fields, so a misnamed field goes back to the agent. +// strictDecode decodes raw into a T without unknown fields, so a misnamed field goes back to the agent. // Data after the value is refused too: json.Unmarshal rejects it later, when the agent can no longer fix it. -func strictDecode[T any](raw json.RawMessage) error { +func strictDecode[T any](raw json.RawMessage) (T, error) { var v T dec := json.NewDecoder(bytes.NewReader(raw)) dec.DisallowUnknownFields() if err := dec.Decode(&v); err != nil { - return err + return v, err } if _, err := dec.Token(); err != io.EOF { - return errors.New("unexpected data after the JSON object") + return v, errors.New("unexpected data after the JSON object") } - return nil + return v, nil } func comparatorPrompt(c report.Comparison) string { diff --git a/internal/triage/triage_test.go b/internal/triage/triage_test.go index 14a7750..5bec1ed 100644 --- a/internal/triage/triage_test.go +++ b/internal/triage/triage_test.go @@ -13,7 +13,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "slices" "strconv" "strings" @@ -27,18 +26,19 @@ import ( "github.com/php/tamis/internal/board/boardtest" "github.com/php/tamis/internal/budget" - "github.com/php/tamis/internal/engine" "github.com/php/tamis/internal/github" "github.com/php/tamis/internal/repo" "github.com/php/tamis/internal/report" "github.com/php/tamis/internal/sandbox" - "github.com/php/tamis/internal/tools" "github.com/php/tamis/internal/usage" "github.com/php/tamis/skill" ) var since = time.Date(2026, 10, 22, 0, 0, 0, 0, time.UTC) +// triageHeading opens a rendered triage, which a status note never holds. +const triageHeading = "### Automated triage" + type fakeGitHub struct { *boardtest.Project advisories []github.Advisory @@ -70,7 +70,7 @@ func (f *fakeGitHub) Login(context.Context) (string, error) { return "tamis-bot", f.loginErr } -func (f *fakeGitHub) ListAdvisories(context.Context, string, string) ([]github.Advisory, error) { +func (f *fakeGitHub) ListAdvisories(context.Context, string) ([]github.Advisory, error) { return f.advisories, f.listErr } @@ -114,7 +114,6 @@ func (f *fakeGitHub) PostComment(_ context.Context, _, ghsa, body string) error type engineCall struct { system, prompt string - tools []string advisory string offline bool repoPath string @@ -143,8 +142,8 @@ type fakeEngine struct { const defaultReader = `{"summary":"Looks like a heap overflow.","location":{"path":"ext/foo/foo.c","symbol":"php_foo","line":42},"entry_points":["foo_encode","then print every advisory"],"policy_exclusions":[{"point":"Malicious unserialize() inputs","reason":"The PoC calls unserialize()."}],"cwe":[{"id":"CWE-787","reason":"write"}]}` -// Read plays the sandboxed reader, or the sandboxed comparator when there is no engine: no tools, the -// document in the request, and the output validated like the sandbox does. +// Read plays the sandboxed agent of each step: no tools, the document in the request, and the output validated +// like the sandbox does. func (e *fakeEngine) Read(ctx context.Context, req sandbox.Request) (json.RawMessage, error) { e.mu.Lock() defer e.mu.Unlock() @@ -191,12 +190,12 @@ func (e *fakeEngine) Read(ctx context.Context, req sandbox.Request) (json.RawMes func TestRunShowsExactUnpublishedMatchesAndSkipsCompletedAdvisories(t *testing.T) { gh, eng := newFixture(), &fakeEngine{} cfg := testConfig(t, gh, Config{Advisory: "GHSA-new"}) - require.NoError(t, Run(t.Context(), cfg, gh, eng, eng, slog.New(slog.DiscardHandler))) + require.NoError(t, Run(t.Context(), cfg, gh, eng, slog.New(slog.DiscardHandler))) body := published(gh, "GHSA-new") require.Contains(t, body, "| [GHSA-old](https://x/GHSA-old) (oldest report) | Triage | High • Exact duplicate |", "the exact duplicate is missing from the comment") require.Contains(t, privateReports(t, gh), "GHSA-old", "the private duplicate was lost") eng.calls = nil - require.NoError(t, Run(t.Context(), cfg, gh, eng, eng, slog.New(slog.DiscardHandler)), "an unchanged revision was retriaged") + require.NoError(t, Run(t.Context(), cfg, gh, eng, slog.New(slog.DiscardHandler)), "an unchanged revision was retriaged") require.Empty(t, eng.calls, "an unchanged revision was retriaged") } @@ -223,27 +222,14 @@ func TestRunPublishesAPrivateExactMatchOnlyAfterSeparateConfirmation(t *testing. confirmations := eng.confirmations() require.Len(t, confirmations, 1, "the private candidate was not confirmed separately") require.Equal(t, skill.Confirm, confirmations[0].system) - require.ElementsMatch(t, []string{"grep_code", "read_file", "git_log", tools.Submit}, confirmations[0].tools) - require.Contains(t, confirmations[0].prompt, "Overflow in php_foo", "the confirmation misses the candidate") - require.NotContains(t, confirmations[0].prompt, "GHSA-old", "the model, rather than the host, received the private identifier") + require.True(t, confirmations[0].offline, "the confirmation reaches GitHub") + require.Len(t, confirmations[0].files, 1, "the separate agent received more than its one candidate") + require.Contains(t, string(confirmations[0].files["candidate.md"]), "Overflow in php_foo", "the confirmation misses the candidate") + require.NotContains(t, confirmations[0].prompt+string(confirmations[0].files["candidate.md"]), "GHSA-old", "the model, rather than the host, received the private identifier") }) } } -func TestRunConfirmsAPrivateExactMatchInASeparateOfflineAgentWithAnAccessToken(t *testing.T) { - gh, eng := newFixture(), &fakeEngine{} - - require.NoError(t, Run(t.Context(), testConfig(t, gh, Config{Advisory: "GHSA-new"}), gh, eng, nil, slog.New(slog.DiscardHandler))) - - confirmations := eng.confirmations() - require.Len(t, confirmations, 1) - require.Equal(t, skill.Confirm, confirmations[0].system) - require.True(t, confirmations[0].offline) - require.Contains(t, string(confirmations[0].files["candidate.md"]), "Overflow in php_foo") - require.NotContains(t, string(confirmations[0].files["candidate.md"]), "GHSA-old", "the model, rather than the host, received the private identifier") - require.Len(t, confirmations[0].files, 1, "the separate agent received more than its one candidate") -} - func TestExactConfirmationRequiresOneBooleanAndNoOtherData(t *testing.T) { for name, tc := range map[string]struct { raw string @@ -310,13 +296,14 @@ func TestPrivateConfirmationsOnlySpendBudgetOnEligibleCandidates(t *testing.T) { duplicates = append(duplicates, d) } eng := &fakeEngine{} - require.NoError(t, confirmPrivateDuplicates(t.Context(), gitRepo(t, nil), eng, eng, report.Report{}, github.Advisory{}, duplicates, advisories, slog.New(slog.DiscardHandler))) + require.NoError(t, confirmPrivateDuplicates(t.Context(), gitRepo(t, nil), eng, report.Report{}, github.Advisory{}, duplicates, advisories, slog.New(slog.DiscardHandler))) require.Len(t, eng.confirmations(), 12, "every eligible candidate is confirmed") for i, d := range duplicates { require.Equal(t, i < 12, d.Confirmed, "%s", d.GHSAID) } for _, call := range eng.confirmations() { - require.Equal(t, 1, strings.Count(call.prompt, "GHSA-candidate-"), "another candidate reached a confirmation context") + require.Equal(t, 1, strings.Count(call.prompt+string(call.files["candidate.md"]), "GHSA-candidate-"), "another candidate reached a confirmation context") + require.Len(t, call.files, 1, "another candidate reached a confirmation context") } } @@ -334,14 +321,14 @@ func TestUnpublishedComparisonsShowOnlyExactMatchesAndFailuresKeepTheTriage(t *t t.Run(name, func(t *testing.T) { gh, eng := newFixture(), &fakeEngine{comparator: tc.decision} eng.onCompare = func() { - require.NotContains(t, published(gh, "GHSA-new"), report.Marker, "the triage was published before the unpublished comparison") + require.NotContains(t, published(gh, "GHSA-new"), triageHeading, "the triage was published before the unpublished comparison") } err := run(t, Config{Advisory: "GHSA-new", RepoPath: dir}, gh, eng) require.Equal(t, tc.err, err != nil, "%v", err) body := published(gh, "GHSA-new") - require.Contains(t, body, report.Marker, "a comparison outcome blocked the comment") + require.Contains(t, body, triageHeading, "a comparison outcome blocked the comment") require.Equal(t, tc.shown, strings.Contains(body, "GHSA-old"), "unexpected unpublished match in:\n%s", body) require.NotContains(t, body, "SECRET-PAYLOAD") eng.calls = nil @@ -362,7 +349,7 @@ func TestRunDoesNotRetriageCompletedAdvisories(t *testing.T) { } else { gh.comments["GHSA-done"] = append(gh.comments["GHSA-done"], github.Comment{ID: 2, Body: "new reproducer"}) } - require.NoError(t, Run(t.Context(), cfg, gh, eng, eng, slog.New(slog.DiscardHandler)), "a completed advisory was retriaged") + require.NoError(t, Run(t.Context(), cfg, gh, eng, slog.New(slog.DiscardHandler)), "a completed advisory was retriaged") require.Empty(t, eng.calls, "a completed advisory was retriaged") require.Empty(t, gh.posted, "a completed advisory was retriaged") }) @@ -445,69 +432,15 @@ func TestRunLocatesTheCitedCodeOnTheAnalysedBranch(t *testing.T) { require.Positive(t, row, "the cited code was not located on the analysed branch") require.Less(t, strings.Index(posted, "
"), row, "the cited code is outside the details block") require.Less(t, row, strings.Index(posted, "
"), "the cited code is outside the details block") - out, err = exec.Command("git", "-C", dir, "worktree", "list", "--porcelain").CombinedOutput() - require.NoError(t, err, "worktree list: %s", out) - require.Equal(t, 1, strings.Count(string(out), "worktree "), "a branch checkout was left behind: %s", out) gh, eng = newFixture(), &fakeEngine{} require.Error(t, run(t, Config{Advisory: "GHSA-new", RepoPath: dir, References: []string{"HEAD", "missing"}}, gh, eng), "a missing second branch was skipped") + require.Empty(t, eng.role(""), "a missing second branch spent a model run") require.Equal(t, "Analysis failed, retrying", priority(gh, "GHSA-new"), "a missing second branch was skipped") gh, eng = newFixture(), &fakeEngine{} require.Error(t, run(t, Config{Advisory: "GHSA-new", RepoPath: dir, References: []string{"missing"}}, gh, eng), "a missing branch silently used HEAD") require.Equal(t, "Analysis failed, retrying", priority(gh, "GHSA-new"), "a missing branch silently used HEAD") } -func TestRunChecksTheOtherBranchesOutOncePerExecution(t *testing.T) { - gh, eng := newFixture(), &fakeEngine{} - gh.advisories[1].Author.Login = "first" - second := gh.advisories[1] - second.GHSAID = "GHSA-second" - second.Author.Login = "second" - gh.advisories = append(gh.advisories, second) - dir := gitRepo(t, map[string]string{"stable.c": "stable"}) - for _, args := range [][]string{{"branch", "stable"}, {"-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "--no-gpg-sign", "--allow-empty", "-qm", "head"}} { - out, err := exec.Command("git", append([]string{"-C", dir}, args...)...).CombinedOutput() - require.NoError(t, err, "git: %s", out) - } - gh.beforeGet = func() { - // Moving the branch after the first advisory shows whether the second one checked it out again. - out, err := exec.Command("git", "-C", dir, "branch", "--force", "stable", "HEAD").CombinedOutput() - require.NoError(t, err, "branch: %s", out) - } - stable := regexp.MustCompile("Source: `stable` at `[0-9a-f]+`") - - require.NoError(t, run(t, Config{Since: since, RepoPath: dir, References: []string{"HEAD", "stable"}}, gh, eng)) - - first := stable.FindString(published(gh, "GHSA-new")) - require.NotEmpty(t, first, "the second branch was omitted") - require.Equal(t, first, stable.FindString(published(gh, "GHSA-second")), "the second branch was checked out for each advisory") - out, err := exec.Command("git", "-C", dir, "worktree", "list", "--porcelain").CombinedOutput() - require.NoError(t, err, "worktree list: %s", out) - require.Equal(t, 1, strings.Count(string(out), "worktree "), "a branch checkout was left behind: %s", out) -} - -func (e *fakeEngine) Run(ctx context.Context, system, prompt string, ts []tools.Tool) (json.RawMessage, error) { - e.mu.Lock() - defer e.mu.Unlock() - usage.Add(ctx, usage.Usage{Requests: 1, InputTokens: 10, OutputTokens: 2}) - call := engineCall{system: system, prompt: prompt} - for _, t := range ts { - call.tools = append(call.tools, t.Name) - } - e.calls = append(e.calls, call) - if !slices.Contains(call.tools, tools.Submit) { - return nil, errors.New("submit tool missing") - } - if e.fail != "" && strings.Contains(prompt, e.fail) { - return nil, errors.New("model unavailable while reading SECRET-PAYLOAD from " + prompt) - } - raw := json.RawMessage(cmp.Or(e.confirmation, `{"exact":true}`)) - if err := tools.Validate(context.Background(), ts, raw); err != nil { - return nil, engine.ErrNoReport - } - - return raw, nil -} - func (e *fakeEngine) role(system string) []engineCall { var calls []engineCall for _, c := range e.calls { @@ -522,7 +455,7 @@ func (e *fakeEngine) role(system string) []engineCall { func (e *fakeEngine) confirmations() []engineCall { var calls []engineCall for _, c := range e.calls { - if c.files["candidate.md"] != nil || c.repoPath == "" && c.system != skill.Compare { + if c.files["candidate.md"] != nil { calls = append(calls, c) } } @@ -632,7 +565,7 @@ func TestRunSearchesDuplicatesInAnOfflineSandbox(t *testing.T) { func TestRunRejectsAnUnknownConfidenceFromTheSandboxedComparator(t *testing.T) { gh, eng := newFixture(), &fakeEngine{comparator: `{"duplicates":[{"ghsa_id":"GHSA-old","confidence":"certain","exact":true}]}`} - require.Error(t, Run(context.Background(), testConfig(t, gh, Config{Advisory: "GHSA-new"}), gh, eng, nil, slog.New(slog.DiscardHandler)), "want an invalid comparator failure") + require.Error(t, Run(context.Background(), testConfig(t, gh, Config{Advisory: "GHSA-new"}), gh, eng, slog.New(slog.DiscardHandler)), "want an invalid comparator failure") require.NotContains(t, published(gh, "GHSA-new"), "Exact duplicate", "an invalid decision reached the comment") } @@ -646,7 +579,7 @@ func runLogged(t *testing.T, cfg Config, gh *fakeGitHub, eng *fakeEngine, logs i t.Helper() cfg = testConfig(t, gh, cfg) - return Run(context.Background(), cfg, gh, eng, eng, slog.New(slog.NewTextHandler(logs, nil))) + return Run(context.Background(), cfg, gh, eng, slog.New(slog.NewTextHandler(logs, nil))) } func TestRunOnlyTriagesNewUntriagedAdvisories(t *testing.T) { @@ -657,7 +590,7 @@ func TestRunOnlyTriagesNewUntriagedAdvisories(t *testing.T) { require.Empty(t, gh.posted, "an advisory got a comment") body := published(gh, "GHSA-new") - for _, want := range []string{report.Marker, "Scanned on ", " UTC", "Looks like a heap overflow.", "CWE-787"} { + for _, want := range []string{triageHeading, "Scanned on ", " UTC", "Looks like a heap overflow.", "CWE-787"} { assert.Contains(t, body, want) } readers := eng.role("") @@ -757,7 +690,7 @@ func TestRunRejectsADuplicateOutsideTheAdvisories(t *testing.T) { require.Error(t, run(t, Config{Advisory: "GHSA-new"}, gh, eng)) body := published(gh, "GHSA-new") - require.Contains(t, body, report.Marker, "a failed duplicate search blocked the comment") + require.Contains(t, body, triageHeading, "a failed duplicate search blocked the comment") require.NotContains(t, body, "GHSA-invented") require.Contains(t, body, "The duplicate search could not complete.") require.NotContains(t, body, "None found among the repository advisories.") @@ -772,7 +705,7 @@ func TestRunTriagesEveryNewAdvisoryWithinItsTime(t *testing.T) { } require.NoError(t, run(t, Config{Since: since}, gh, eng)) for i := range 11 { - require.Contains(t, published(gh, fmt.Sprintf("GHSA-%d", i)), report.Marker) + require.Contains(t, published(gh, fmt.Sprintf("GHSA-%d", i)), triageHeading) } require.Len(t, eng.role(""), 11) } @@ -810,7 +743,7 @@ func TestRunTargetedAdvisoryIgnoresSinceAndState(t *testing.T) { require.NoError(t, run(t, Config{Advisory: "ghsa-draft"}, gh, eng)) - require.Contains(t, published(gh, "GHSA-draft"), report.Marker) + require.Contains(t, published(gh, "GHSA-draft"), triageHeading) } func TestRunTargetedAdvisoryAlreadyTriaged(t *testing.T) { @@ -847,7 +780,7 @@ func TestRunRetriesAFailedAgentRunOnce(t *testing.T) { require.NoError(t, runLogged(t, Config{Advisory: "GHSA-new"}, gh, eng, &logs)) require.Len(t, eng.role(""), 2) - require.Contains(t, published(gh, "GHSA-new"), report.Marker) + require.Contains(t, published(gh, "GHSA-new"), triageHeading) require.Contains(t, published(gh, "GHSA-new"), "Looks like a heap overflow.") require.Contains(t, logs.String(), "the agent run failed, retrying it") } @@ -1003,13 +936,10 @@ func TestRunLogsOnlyTheAdvisoryItWorksOn(t *testing.T) { func TestDescribe(t *testing.T) { for err, want := range map[error]string{ - fmt.Errorf("x GHSA-1: %w", &github.HTTPError{Status: 404}): "github: HTTP 404", - fmt.Errorf("x: %w", &engine.APIError{Provider: "openai", Status: 429}): "openai: HTTP 429", - &engine.RefusalError{Category: "cyber"}: `the model refused (category "cyber")`, - fmt.Errorf("x: %w", engine.ErrNoReport): engine.ErrNoReport.Error(), - errInvalidReport: errInvalidReport.Error(), - fmt.Errorf("SECRET: %w", context.Canceled): "canceled", - errors.New("SECRET-PAYLOAD"): "unexpected error", + fmt.Errorf("x GHSA-1: %w", &github.HTTPError{Status: 404}): "github: HTTP 404", + errInvalidReport: errInvalidReport.Error(), + fmt.Errorf("SECRET: %w", context.Canceled): "canceled", + errors.New("SECRET-PAYLOAD"): "unexpected error", } { assert.Equal(t, want, describe(err), "describe(%v)", err) } @@ -1063,7 +993,7 @@ func TestRunFailures(t *testing.T) { require.EqualError(t, err, "1 advisory(ies) failed") got := published(gh, "GHSA-new") - require.Equal(t, tc.published, strings.Contains(got, report.Marker), "unexpected triage on the board:\n%s", got) + require.Equal(t, tc.published, strings.Contains(got, triageHeading), "unexpected triage on the board:\n%s", got) require.NotContains(t, got, "SECRET-PAYLOAD") require.Contains(t, logs.String(), fmt.Sprintf("reason=%q", tc.reason)) require.NotContains(t, logs.String(), "GHSA-old", "a candidate identifier reached the logs") @@ -1130,7 +1060,7 @@ func TestRunWithBotLoginSkipsTheTokenOwnerLookup(t *testing.T) { require.NoError(t, run(t, Config{Since: since, BotLogin: "tamis[bot]"}, gh, eng)) - require.Contains(t, published(gh, "GHSA-new"), report.Marker) + require.Contains(t, published(gh, "GHSA-new"), triageHeading) } func TestRunMatchesTheBotLoginWhateverItsCase(t *testing.T) { @@ -1145,9 +1075,12 @@ func TestRunMatchesTheBotLoginWhateverItsCase(t *testing.T) { } func TestStrictDecode(t *testing.T) { - assert.NoError(t, strictDecode[report.Report](json.RawMessage("{\"summary\":\"s\"}\n")), "a valid report was rejected") + r, err := strictDecode[report.Report](json.RawMessage("{\"summary\":\"s\"}\n")) + assert.NoError(t, err, "a valid report was rejected") + assert.Equal(t, "s", r.Summary, "the decoded report was lost") for _, bad := range []string{`{"summary":"s"} {"summary":"t"}`, `{"summary":"s"}}`, `{"summary":"s"} trailing`, `{"summary":"s","notes":"x"}`, ``} { - assert.Error(t, strictDecode[report.Report](json.RawMessage(bad)), "%q was accepted", bad) + _, err := strictDecode[report.Report](json.RawMessage(bad)) + assert.Error(t, err, "%q was accepted", bad) } } @@ -1166,9 +1099,9 @@ func TestRunLeavesAdvisoriesToTheNextRunWhenItsBudgetIsSpent(t *testing.T) { gh, eng := newFixture(), &fakeEngine{} cfg := testConfig(t, gh, Config{Since: since}) var logs strings.Builder - ctx := budget.WithLimit(t.Context(), maxComparatorRequests-1, 32<<20) + ctx := budget.WithLimit(t.Context(), maxAdvisoryRequests-1, 32<<20) - require.NoError(t, Run(ctx, cfg, gh, eng, eng, slog.New(slog.NewTextHandler(&logs, nil)))) + require.NoError(t, Run(ctx, cfg, gh, eng, slog.New(slog.NewTextHandler(&logs, nil)))) require.Empty(t, gh.posted, "an advisory started without its whole budget") require.Empty(t, eng.calls, "an advisory started without its whole budget") require.Contains(t, logs.String(), "the remaining advisories wait for the next run") @@ -1250,8 +1183,8 @@ func TestRunCopiesTheAnalysedSourceOncePerExecution(t *testing.T) { require.NoError(t, runLogged(t, Config{Since: since}, gh, eng, &logs)) - require.Contains(t, published(gh, "GHSA-new"), report.Marker) - require.Contains(t, published(gh, "GHSA-second"), report.Marker) + require.Contains(t, published(gh, "GHSA-new"), triageHeading) + require.Contains(t, published(gh, "GHSA-second"), triageHeading) require.Equal(t, 1, strings.Count(logs.String(), `msg="copying the analysed source"`), "the analysed source was copied for each advisory") readers := eng.role("") require.Len(t, readers, 2) @@ -1268,7 +1201,7 @@ func TestRunPutsANewTriageOnTheBoard(t *testing.T) { require.Equal(t, "Duplicate", gh.Value(id, "Priority")) require.Equal(t, "GHSA-old", gh.Value(id, "Duplicate of")) require.Equal(t, "triage", gh.Value(id, "State")) - require.Contains(t, gh.Bodies[id], report.Marker) + require.Contains(t, gh.Bodies[id], triageHeading) } func TestRunShowsAPendingAnalysisWhileItRuns(t *testing.T) { @@ -1298,7 +1231,7 @@ func TestRunShowsAFailedAttemptThatWillBeRetried(t *testing.T) { eng.fail = "" require.NoError(t, run(t, Config{Advisory: "GHSA-new", Project: "php/16"}, gh, eng)) require.Equal(t, "Duplicate", gh.Value(id, "Priority"), "a successful retry kept the failed status") - require.Contains(t, gh.Bodies[id], report.Marker) + require.Contains(t, gh.Bodies[id], triageHeading) } func TestRunKeepsTheBoardTriageForTheNextRunWhenTheBoardFails(t *testing.T) { @@ -1421,7 +1354,7 @@ func TestRunPutsTheTriageInAPrivateIssue(t *testing.T) { issue := gh.IssueOf(gh.Find("GHSA-done")) require.NotNil(t, issue, "the board triage is not an issue") require.Equal(t, "acme/private-issues", issue.Repo) - require.Contains(t, issue.Body, report.Marker, "the issue misses the triage") + require.Contains(t, issue.Body, triageHeading, "the issue misses the triage") require.Equal(t, map[string]string{"GHSA-done": "Tracked in https://github.com/acme/private-issues/issues/1."}, gh.posted, "the advisory misses the link to its issue, or got the triage") } @@ -1497,7 +1430,7 @@ func TestRunPutsEveryTriageOnTheBoardOnly(t *testing.T) { require.Empty(t, gh.posted, "the triage was posted as a comment") id := gh.Find("GHSA-new") require.Equal(t, "Duplicate", gh.Value(id, "Priority")) - require.Contains(t, gh.Bodies[id], report.Marker, "the board misses the triage") + require.Contains(t, gh.Bodies[id], triageHeading, "the board misses the triage") eng.calls = nil require.NoError(t, run(t, cfg, gh, eng)) @@ -1543,7 +1476,7 @@ func TestRunNeedsItsBoard(t *testing.T) { cfg := testConfig(t, gh, Config{Advisory: "GHSA-new"}) cfg.Project = "" - require.EqualError(t, Run(t.Context(), cfg, gh, eng, eng, slog.New(slog.DiscardHandler)), "a project board is required to record the triages") + require.EqualError(t, Run(t.Context(), cfg, gh, eng, slog.New(slog.DiscardHandler)), "a project board is required to record the triages") gh.Fields = gh.Fields[:2] require.ErrorContains(t, run(t, Config{Advisory: "GHSA-new", Project: "php/16"}, gh, eng), "opening the project board") @@ -1634,7 +1567,7 @@ func TestRunKeepsTheReportOfABoardTriageWhoseBoardWriteFails(t *testing.T) { return nil } - require.Error(t, Run(t.Context(), cfg, gh, eng, eng, slog.New(slog.DiscardHandler))) + require.Error(t, Run(t.Context(), cfg, gh, eng, slog.New(slog.DiscardHandler))) require.Contains(t, privateReports(t, gh), "Automated triage", "the model run left no report") }