Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 16 additions & 19 deletions cmd/tamis/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,19 @@ import (
"time"

"github.com/php/tamis/internal/board"
"github.com/php/tamis/internal/engine"
"github.com/php/tamis/internal/github"
"github.com/php/tamis/internal/sandbox"
"github.com/php/tamis/internal/setup"
"github.com/php/tamis/internal/triage"
"github.com/php/tamis/skill"
)

// defaultCodexTokenModel is Codex's default with a ChatGPT account, which does not serve the API models.
const defaultCodexTokenModel = "gpt-6.1-sol"
const (
defaultClaudeModel = "claude-opus-5-5"
defaultOpenAIModel = "gpt-5.5"
// defaultCodexTokenModel is Codex's default with a ChatGPT account, which does not serve the API models.
defaultCodexTokenModel = "gpt-6.1-sol"
)

func main() {
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
Expand Down Expand Up @@ -211,7 +214,7 @@ func run(ctx context.Context, log *slog.Logger) error {
if cfg.CloseDuplicates && cfg.IssuesRepository == "" {
return errors.New("CLOSE_DUPLICATES needs ISSUES_REPOSITORY")
}
eng, agent, model, err := newEngine(os.Getenv("ENGINE"), os.Getenv("MODEL"), os.Getenv("LLM_API_KEY"))
agent, model, err := selectAgent(os.Getenv("ENGINE"), os.Getenv("MODEL"), os.Getenv("LLM_API_KEY"))
if err != nil {
return err
}
Expand Down Expand Up @@ -239,7 +242,7 @@ func run(ctx context.Context, log *slog.Logger) error {
defer reader.Close()
reader.Daybreak = daybreak

return triage.Run(ctx, cfg, github.New(token), reader, eng, log)
return triage.Run(ctx, cfg, github.New(token), reader, log)
}

func parseSince(v string) (time.Time, error) {
Expand All @@ -257,28 +260,22 @@ func parseSince(v string) (time.Time, error) {
return t, nil
}

// newEngine returns the comparator engine, and the sandboxed agent and model the reader runs with. A Codex
// access token cannot call the OpenAI API, so the comparator runs as a sandboxed agent too: no engine.
func newEngine(name, model, apiKey string) (engine.Engine, string, string, error) {
// selectAgent returns the sandboxed agent every model step runs as, and its model. A Codex access token cannot
// call the OpenAI API, so it gets an agent of its own.
func selectAgent(name, model, apiKey string) (string, string, error) {
if apiKey == "" {
return nil, "", "", errors.New("LLM_API_KEY is required")
return "", "", errors.New("LLM_API_KEY is required")
}

switch name {
case "", "claude":
if model == "" {
model = engine.DefaultClaudeModel
}
return engine.NewClaude(apiKey, model), "claude", model, nil
return "claude", cmp.Or(model, defaultClaudeModel), nil
case "openai":
if strings.HasPrefix(apiKey, "at-") {
return nil, "codex-token", cmp.Or(model, defaultCodexTokenModel), nil
}
if model == "" {
model = engine.DefaultOpenAIModel
return "codex-token", cmp.Or(model, defaultCodexTokenModel), nil
}
return engine.NewOpenAI(apiKey, model), "codex", model, nil
return "codex", cmp.Or(model, defaultOpenAIModel), nil
default:
return nil, "", "", fmt.Errorf("unknown ENGINE %q, expected claude or openai", name)
return "", "", fmt.Errorf("unknown ENGINE %q, expected claude or openai", name)
}
}
22 changes: 10 additions & 12 deletions cmd/tamis/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,30 +28,28 @@ func TestParseSince(t *testing.T) {
assert.Error(t, err, "an invalid date should fail")
}

func TestNewEngine(t *testing.T) {
for name, want := range map[string]string{"": "claude", "claude": "claude", "openai": "codex"} {
_, agent, model, err := newEngine(name, "", "key")
func TestSelectAgent(t *testing.T) {
for name, want := range map[string][2]string{"": {"claude", defaultClaudeModel}, "claude": {"claude", defaultClaudeModel}, "openai": {"codex", defaultOpenAIModel}} {
agent, model, err := selectAgent(name, "", "key")
assert.NoError(t, err, "%q", name)
assert.Equal(t, want, agent, "%q", name)
assert.NotEmpty(t, model, "%q", name)
assert.Equal(t, want[0], agent, "%q", name)
assert.Equal(t, want[1], model, "%q", name)
}

eng, agent, model, err := newEngine("openai", "", "at-token")
agent, model, err := selectAgent("openai", "", "at-token")
assert.NoError(t, err, "a Codex access token")
assert.Nil(t, eng, "a Codex access token")
assert.Equal(t, "codex-token", agent, "a Codex access token")
assert.Equal(t, defaultCodexTokenModel, model, "a Codex access token")

eng, agent, _, _ = newEngine("claude", "", "at-key")
assert.NotNil(t, eng, "only the openai engine reads a Codex access token")
agent, _, _ = selectAgent("claude", "", "at-key")
assert.Equal(t, "claude", agent, "only the openai engine reads a Codex access token")

_, _, model, _ = newEngine("claude", "claude-sonnet-5-5", "key")
_, model, _ = selectAgent("claude", "claude-sonnet-5-5", "key")
assert.Equal(t, "claude-sonnet-5-5", model, "an explicit model should be kept")

_, _, _, err = newEngine("gemini", "", "key")
_, _, err = selectAgent("gemini", "", "key")
assert.Error(t, err, "an unknown engine should fail")
_, _, _, err = newEngine("claude", "", "")
_, _, err = selectAgent("claude", "", "")
assert.Error(t, err, "a missing API key should fail")
}

Expand Down
14 changes: 8 additions & 6 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ Each report names the source commit and the input revision it covers.
| Request | 1 MiB of body. API responses ask for at most 16,000 output tokens, and the ChatGPT backend applies its own output limits. |
| Reports | One per known reporter and per execution |

tamis starts a report only while the execution still holds a whole advisory budget and thirty minutes of its time. Otherwise the report waits for the next execution. SDK retries are off, but tamis retries an agent run that fails without a valid report once, after a minute, unless the model account was refused for good or the advisory budget is spent. A GitHub read answered with 429, 502, 503 or 504 is tried up to three times. When a budget runs out, the report stays retriable and tamis does not mark it complete.
tamis starts a report only while the execution still holds a whole advisory budget and thirty minutes of its time. Otherwise the report waits for the next execution. tamis retries an agent run that fails without a valid report once, after a minute, unless the model account was refused for good or the advisory budget is spent. A GitHub read answered with 429, 502, 503 or 504 is tried up to three times. When a budget runs out, the report stays retriable and tamis does not mark it complete.

These bounds cap one execution, not the spending across scheduled runs. Provider pricing, tokenisation and ChatGPT credit policies differ, so request and byte limits give no monetary guarantee. On the provider side:

Expand Down Expand Up @@ -89,11 +89,13 @@ The triage lists the files the agent checked. Without any of them, tamis leaves

## Engines

| Engine | Reader and global search | Exact confirmation | Model API |
|---|---|---|---|
| `claude` | Claude Code | Anthropic API | `api.anthropic.com` |
| `openai` | Codex | OpenAI Chat Completions | `api.openai.com` |
| `openai` with a Codex access token | Codex | Codex, in a container without GitHub access | `chatgpt.com` |
| Engine | Agent | Model API |
|---|---|---|
| `claude` | Claude Code | `api.anthropic.com` |
| `openai` | Codex | `api.openai.com` |
| `openai` with a Codex access token | Codex | `chatgpt.com` |

The reader, the duplicate search and each exact confirmation run that agent in a container of their own, the last two without GitHub access.

[`docker/agent/Dockerfile`](../docker/agent/Dockerfile) pins the agent CLI versions.

Expand Down
6 changes: 2 additions & 4 deletions docs/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ go build ./cmd/tamis
go test -race ./...
```

Three fuzz targets cover the comment escaper, the duplicate-search clues and file reading:
Three fuzz targets cover the text escaper, the duplicate-search clues and file reading:

| Target | Package |
|---|---|
Expand Down Expand Up @@ -38,7 +38,7 @@ Repeat with `claude`. The tests use fake model servers, check direct host, Inter

## Evals

The evals run the real triage, sandboxed agent and model included, on made-up reports about `libmini`, a small C library in `internal/triage/testdata/evals`. Each scenario checks the location, the CWE, the policy exclusions and the duplicates the models propose, and that a prompt injection does not leak another advisory into the comment. They spend real model requests, so they only run with the `evals` tag, Docker and a key:
The evals run the real triage, sandboxed agent and model included, on made-up reports about `libmini`, a small C library in `internal/triage/testdata/evals`. Each scenario checks the location, the CWE, the policy exclusions and the duplicates the models propose, that a prompt injection does not leak another advisory into the triage, and that an unpublished exact duplicate reaches the board item once its confirmation passed. They spend real model requests, so they only run with the `evals` tag, Docker and a key:

```bash
ENGINE=claude LLM_API_KEY="$ANTHROPIC_API_KEY" go test -tags evals -count=1 -timeout 1h -run TestEvals ./internal/triage
Expand Down Expand Up @@ -93,8 +93,6 @@ tamis writes the board item for real. It skips an advisory whose item already ho
| `internal/sandbox` | The agent container and the model proxy that keeps the key on the host, through harness |
| `internal/budget` | Request and byte budgets shared by every agent and comparator of a run |
| `internal/usage` | Requests and tokens spent per advisory, for the log |
| `internal/engine` | The in-process model loop of the exact duplicate confirmation |
| `internal/tools` | The read-only tools of the exact duplicate confirmation |
| `internal/report` | Report schema, verification and rendering |
| `internal/repo` | Read-only access to tracked files |
| `internal/github` | GitHub REST client |
Expand Down
4 changes: 2 additions & 2 deletions docs/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ The agent reports the faulty statement (file, function, line), the public entry

tamis rejects an unknown or mistyped field in the JSON and sends the error back to the agent for one correction round.

When the run stops on its budget or on an error, tamis keeps a valid report already written in the private artifact and retries the advisory. The last attempt posts that report with a note that it may be incomplete.
When the run stops on its budget or on an error, tamis keeps a valid report already written in the private artifact and retries the advisory. The last attempt puts that report on the board with a note that it may be incomplete.

The agent analyses the first configured reference, where tamis locates the cited code, and the final report names the exact commit of every reference. A missing branch fails the triage.

Expand All @@ -39,7 +39,7 @@ One offline agent searches every other advisory of the repository for the same b

tamis treats the report, the clues and the advisories as hostile data. The agent answers with the identifiers of the duplicates it found, each with a confidence and an exact flag. tamis refuses an identifier outside the advisories it gave.

The private report keeps the full search result. tamis checks each unpublished candidate marked exact with high confidence again, in its own model context. That context receives the report, the candidate without its identifier, the checked clues and the code, and returns one `exact` boolean. Only the host knows which candidate the answer belongs to, and it copies a confirmed identifier into the public comment. All models share the advisory and run budgets. The log records the duration and usage of the confirmations and the number of confirmed candidates.
The private report keeps the full search result. tamis checks each unpublished candidate marked exact with high confidence again, in its own model context. That context receives the report, the candidate without its identifier, the checked clues and the code, and returns one `exact` boolean. Only the host knows which candidate the answer belongs to, and it copies a confirmed identifier into the board item. All models share the advisory and run budgets. The log records the duration and usage of the confirmations and the number of confirmed candidates.

Both stages compare concrete defects, secondary findings of a report included:

Expand Down
2 changes: 1 addition & 1 deletion docs/security.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ tamis itself never logs report text, the identifiers of other advisories or prov
The agent runs through [harness](https://github.com/alpha-omega-security/harness), the isolation layer of [scrutineer](https://github.com/alpha-omega-security/scrutineer):

- a throwaway container per advisory, with a read-only root filesystem, all capabilities dropped and no new privileges,
- one single-branch clone of the analysed branch per execution, mounted read-only into each agent run, with no untracked files and an empty `.git/config`, while only tamis reads the other branches, checked out as worktrees,
- one single-branch clone of the analysed branch per execution, mounted read-only into each agent run, with no untracked files and an empty `.git/config`, while tamis only reads the commit of the other branches,
- an internal bridge network with both gateway modes set to `isolated` and IPv6 disabled, whose only peer is an authenticated egress proxy that allows the model proxy and `api.github.com`,
- a read-only workspace except for `report.json`, with memory, CPU, process and file-size limits,
- an image with only the agent CLI, git, curl, ripgrep and `harness-proxy`, each download checked against a pinned checksum,
Expand Down
22 changes: 1 addition & 21 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@ go 1.27.3

require (
github.com/alpha-omega-security/harness v0.1.21
github.com/anthropics/anthropic-sdk-go v1.84.0
github.com/git-pkgs/cwe v0.1.1
github.com/openai/openai-go/v3 v3.97.0
github.com/stretchr/testify v1.12.1
go.yaml.in/yaml/v3 v3.0.6
golang.org/x/crypto v0.58.0
Expand All @@ -15,34 +13,16 @@ require (

require (
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/bitfield/gotestdox v0.2.3 // indirect
github.com/buger/jsonparser v1.6.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/coder/websocket v1.8.15 // indirect
github.com/dnephin/pflag v1.0.7 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/git-pkgs/skills v0.2.1 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/invopop/jsonschema v0.14.0 // indirect
github.com/mattn/go-colorable v0.1.16 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/pb33f/go-yaml v0.1.1 // indirect
github.com/pb33f/ordered-map/v2 v2.3.2 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 // indirect
github.com/tidwall/gjson v1.20.2 // indirect
github.com/tidwall/match v1.2.0 // indirect
github.com/tidwall/pretty v1.3.0 // indirect
github.com/tidwall/sjson v1.2.9 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/otel/trace v1.47.0 // indirect
golang.org/x/exp/typeparams v0.0.0-20261106192419-47ce03d25ef0 // indirect
golang.org/x/mod v0.42.0 // indirect
golang.org/x/sys v0.49.0 // indirect
Expand Down
Loading
Loading