From 9b4e1bb8ce9a2ae13773702678968f644a5fba5b Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Wed, 18 Nov 2026 13:52:39 +0200 Subject: [PATCH 1/3] feat(openrca): export the difficulty of each task index The shell's benchmark picker groups tasks by difficulty the way the scorer does, so difficultyOf and Difficulty move from the scoring summary to tasks.ts and the package exports them. --- .changeset/openrca-difficulty.md | 5 +++++ packages/openrca/README.md | 2 +- packages/openrca/src/index.ts | 8 ++++---- packages/openrca/src/scoring/index.ts | 3 ++- packages/openrca/src/scoring/summary.ts | 20 ++------------------ packages/openrca/src/tasks.ts | 17 +++++++++++++++++ 6 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 .changeset/openrca-difficulty.md diff --git a/.changeset/openrca-difficulty.md b/.changeset/openrca-difficulty.md new file mode 100644 index 0000000..59dc54c --- /dev/null +++ b/.changeset/openrca-difficulty.md @@ -0,0 +1,5 @@ +--- +"@evorca/openrca": minor +--- + +Exports `difficultyOf`, the difficulty of each task index, and its `Difficulty` type, so a caller can group tasks the way the scorer does. diff --git a/packages/openrca/README.md b/packages/openrca/README.md index 26b42e9..f137842 100644 --- a/packages/openrca/README.md +++ b/packages/openrca/README.md @@ -25,7 +25,7 @@ OpenRCA benchmark support for evorca. It reads [OpenRCA](https://github.com/micr `src/index.ts` is the entry for an agent runner. It exposes no scoring points. - `openRcaScenarios` maps each scenario slug, such as `market-cloudbed-1`, to its OpenRCA system, such as `Market/cloudbed-1`. `OpenRcaScenario` is one slug. Each scenario's files live in `//`, where `dataDir` is the benchmark's data directory, `data/benchmarks/openrca` in this repository. -- `readTasks(scenario, dataDir)` returns the tasks in `//query.csv` as `OpenRcaTask` values, in file order. A task's `TaskIndex`, `task_1` to `task_7`, decides which root cause fields the answer needs. +- `readTasks(scenario, dataDir)` returns the tasks in `//query.csv` as `OpenRcaTask` values, in file order. A task's `TaskIndex`, `task_1` to `task_7`, decides which root cause fields the answer needs, and `difficultyOf` gives its `Difficulty`: `task_1` to `task_3` are easy, `task_4` to `task_6` are middle, and `task_7` is hard, as in OpenRCA's `main/evaluate.py`. - `openRcaCandidates` maps each scenario slug to its `Candidates`, the `components` and `reasons` that an answer can name. See [Show an agent the candidates](#show-an-agent-the-candidates). - `formatPrediction(diagnosis)` and `parsePrediction(raw)` convert between a `Diagnosis` and OpenRCA's prediction JSON. `parsePrediction` returns `undefined` for text that is not a valid prediction, so only `{}` parses as no causes. - `writePredictions(tasks, diagnoses)` returns the prediction CSV for one scenario, with one row per task in the order of `tasks`. diff --git a/packages/openrca/src/index.ts b/packages/openrca/src/index.ts index a9ffb47..2f59dfc 100644 --- a/packages/openrca/src/index.ts +++ b/packages/openrca/src/index.ts @@ -1,7 +1,7 @@ /** * The `@evorca/openrca` entry, for an agent runner. It names the OpenRCA - * scenarios, lists the candidate root causes of each, reads their tasks, and - * writes predictions in OpenRCA's format. + * scenarios, lists the candidate root causes of each, reads their tasks and + * gives the difficulty of each, and writes predictions in OpenRCA's format. * * It exposes no scoring points, so an agent that imports it cannot read the * answers through it. Scoring has its own entry, `@evorca/openrca/scoring`. @@ -13,5 +13,5 @@ export type { Candidates } from "./candidates.js"; export { formatPrediction, parsePrediction, writePredictions } from "./prediction.js"; export { openRcaScenarios } from "./scenarios.js"; export type { OpenRcaScenario } from "./scenarios.js"; -export { readTasks } from "./tasks.js"; -export type { OpenRcaTask, TaskIndex } from "./tasks.js"; +export { difficultyOf, readTasks } from "./tasks.js"; +export type { Difficulty, OpenRcaTask, TaskIndex } from "./tasks.js"; diff --git a/packages/openrca/src/scoring/index.ts b/packages/openrca/src/scoring/index.ts index d6cc1f8..b3b622b 100644 --- a/packages/openrca/src/scoring/index.ts +++ b/packages/openrca/src/scoring/index.ts @@ -14,4 +14,5 @@ export type { Mismatch, Stage } from "./archive.js"; export { evaluatorVersions, scoreWithOfficialEvaluator } from "./evaluator.js"; export type { ScoredRow } from "./evaluator.js"; export { summarize } from "./summary.js"; -export type { Accuracy, Difficulty, Summary } from "./summary.js"; +export type { Difficulty } from "../tasks.js"; +export type { Accuracy, Summary } from "./summary.js"; diff --git a/packages/openrca/src/scoring/summary.ts b/packages/openrca/src/scoring/summary.ts index 4f0c296..3c21db5 100644 --- a/packages/openrca/src/scoring/summary.ts +++ b/packages/openrca/src/scoring/summary.ts @@ -4,15 +4,9 @@ * * @module */ -import type { TaskIndex } from "../tasks.js"; +import { difficultyOf, type Difficulty } from "../tasks.js"; import type { ScoredRow } from "./evaluator.js"; -/** - * An OpenRCA difficulty. As in `main/evaluate.py`, `task_1` to `task_3` are easy, - * `task_4` to `task_6` are middle, and `task_7` is hard. - */ -export type Difficulty = "easy" | "middle" | "hard"; - /** * The accuracy over a set of tasks. `strict` is the share of tasks with a score * of 1, and `partial` is the mean score. Both are 0 for an empty set. @@ -26,19 +20,9 @@ export type Accuracy = { /** The accuracy per difficulty and over all tasks, in the key order easy, middle, hard, total. */ export type Summary = Readonly>; -const difficulty: Record = { - task_1: "easy", - task_2: "easy", - task_3: "easy", - task_4: "middle", - task_5: "middle", - task_6: "middle", - task_7: "hard", -}; - /** Computes the accuracy per difficulty and over all rows, which may come in any order. */ export function summarize(rows: readonly ScoredRow[]): Summary { - const byLevel = Object.groupBy(rows, (row) => difficulty[row.taskIndex]); + const byLevel = Object.groupBy(rows, (row) => difficultyOf[row.taskIndex]); return { easy: accuracy(byLevel.easy ?? []), middle: accuracy(byLevel.middle ?? []), diff --git a/packages/openrca/src/tasks.ts b/packages/openrca/src/tasks.ts index 94e1d76..57199e5 100644 --- a/packages/openrca/src/tasks.ts +++ b/packages/openrca/src/tasks.ts @@ -21,6 +21,23 @@ const taskIndices = ["task_1", "task_2", "task_3", "task_4", "task_5", "task_6", */ export type TaskIndex = (typeof taskIndices)[number]; +/** + * An OpenRCA difficulty. As in `main/evaluate.py`, `task_1` to `task_3` are easy, + * `task_4` to `task_6` are middle, and `task_7` is hard. + */ +export type Difficulty = "easy" | "middle" | "hard"; + +/** The difficulty of each task index. */ +export const difficultyOf: Readonly> = { + task_1: "easy", + task_2: "easy", + task_3: "easy", + task_4: "middle", + task_5: "middle", + task_6: "middle", + task_7: "hard", +}; + /** * One OpenRCA task, without its scoring points. `row` is the zero-based row of * the task in `query.csv`, and a prediction file answers the task on the same From e002fdca85fe63ad51bb17eaa0ae219a0972c132 Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Wed, 18 Nov 2026 13:53:31 +0200 Subject: [PATCH 2/3] feat(cli): give --rows per scenario --rows can repeat, and a value that starts with a scenario, such as --rows=bank:3,4, gives the rows of that scenario only. A bare --rows=0,7,21 still gives the rows of every scenario that no other value names. run.json keeps its format, so a resume checks the rows of each scenario against its own record, and earlier runs resume and report as before. The scenario list moves to runs.ts so the parser and report share it. --- .changeset/cli-evaluate-scenario-rows.md | 5 +++ packages/cli/README.md | 10 ++--- packages/cli/src/main.ts | 23 +++++------ packages/cli/src/openrca.ts | 34 ++++++++++------ packages/cli/src/report.ts | 9 ++--- packages/cli/src/runs.test.ts | 41 +++++++++++++++++++ packages/cli/src/runs.ts | 50 +++++++++++++++++++++++- 7 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 .changeset/cli-evaluate-scenario-rows.md diff --git a/.changeset/cli-evaluate-scenario-rows.md b/.changeset/cli-evaluate-scenario-rows.md new file mode 100644 index 0000000..ff33ea5 --- /dev/null +++ b/.changeset/cli-evaluate-scenario-rows.md @@ -0,0 +1,5 @@ +--- +"@evorca/cli": minor +--- + +`--rows` can repeat, and a value that starts with a scenario, such as `--rows=bank:3,4`, gives the rows of that scenario only. `--rows=0,7,21` still gives the rows of every scenario, and `run.json` keeps its format, so earlier runs resume and report as before. diff --git a/packages/cli/README.md b/packages/cli/README.md index 9f9a2fd..cbfbff2 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -29,9 +29,9 @@ evorca evorca fetch --benchmark= [--scenario=...] [--archive=...] evorca reproduce --benchmark= [--scenario=...] [--archive=...] evorca evaluate --benchmark= --agent= [--scenario=...] - [--runs=] [--rows=,...] [--jobs=] [--model=] - [--effort=] [--budget-usd=] [--task-budget-usd=] - [--plain] + [--runs=] [--rows=[:],...] [--jobs=] + [--model=] [--effort=] [--budget-usd=] + [--task-budget-usd=] [--plain] evorca evaluate --benchmark= --resume= [--scenario=...] [--jobs=] [--budget-usd=] [--plain] evorca report --benchmark= [--scenario=...] ... @@ -44,7 +44,7 @@ evorca watch --benchmark= [--scenario=...] [] - `--agent` is required by `evorca evaluate`, unless it resumes a run. The agents are `random` and [`react`](../../agents/react/README.md). - `--archive` names a dataset archive on disk and can repeat. `evorca fetch` and `evorca reproduce` read the files of that archive from disk instead of downloading them, and check them in the same way. See [Fetch a scenario](#fetch-a-scenario). - `--runs` sets how many complete runs `evorca evaluate` makes, one after another, with the same settings. Without it, the command makes one run. See [Repeat a run](#repeat-a-run). -- `--rows` limits `evorca evaluate` to some rows of each scenario's `query.csv`, counted from 0, such as `--rows=0,7,21` for a pilot. Without it, the command runs every task. +- `--rows` limits `evorca evaluate` to some rows of each scenario's `query.csv`, counted from 0, such as `--rows=0,7,21` for a pilot. Without it, the command runs every task. It can repeat, and a value that starts with a scenario, such as `--rows=bank:3,4`, gives the rows of that scenario only. So `--rows=0,7 --rows=bank:3,4` runs rows 0 and 7 of each scenario but `bank`, and rows 3 and 4 of `bank`. - `--jobs` sets how many tasks `evorca evaluate` runs at once. Without it, the command uses the `jobs` of the run it resumes, or else the agent's `jobs`. - `--model` and `--effort` set the model that the agent calls and its effort. The only model is `claude-sonnet-5`, and its default effort is `high`. The efforts are `low`, `medium`, `high`, `xhigh`, and `max`. - `--budget-usd` caps what the run spends on the model, in US dollars, and defaults to 50. `--task-budget-usd` caps what each task spends and defaults to 2. @@ -68,7 +68,7 @@ fetched data/benchmarks/openrca/telecom/telemetry/2020_04_11/trace/trace_span.c Both commands exit with code 1 and list the valid choices if the command, the benchmark, or a scenario is unknown. They also exit with code 1, and print the reason, if an `--archive` matches no archive of the benchmark, if a file on disk does not match its pinned hash, if the missing files need more than the free disk space, or if a download fails. The free-space message gives both sizes in GB. When Google Drive refuses a download because the file's quota is used up, the message names the archive, its link for a browser, and the command to run with `--archive` once you have downloaded it. -`evorca evaluate` prints one line per task, then the outcomes and the accuracy of each scenario. It exits with code 1 and lists the agents if `--agent` is missing or unknown. Before it runs any task, it exits with code 1 if a model flag is given for an agent that calls no model, if an agent that calls a model has no API key, if a scenario's data is missing, with the `evorca fetch` command that downloads it, if a row in `--rows` does not exist, if Docker does not answer, if a container cannot reach the model proxy, or if the run to resume is missing or has other settings. It names each setting that differs. It exits with code 1 for `--runs` together with `--resume`, because a resumed series keeps its number of runs. +`evorca evaluate` prints one line per task, then the outcomes and the accuracy of each scenario. It exits with code 1 and lists the agents if `--agent` is missing or unknown. Before it runs any task, it exits with code 1 if a model flag is given for an agent that calls no model, if an agent that calls a model has no API key, if a scenario's data is missing, with the `evorca fetch` command that downloads it, if a row in `--rows` does not exist, if a `--rows` value names a scenario that the run does not cover or gives rows that another value gives, if Docker does not answer, if a container cannot reach the model proxy, or if the run to resume is missing or has other settings. It names each setting that differs. It exits with code 1 for `--runs` together with `--resume`, because a resumed series keeps its number of runs. `evorca report` exits with code 1 if a run is missing, if two runs have other settings, or if no run of a scenario is finished. It names each setting that differs. diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 550b221..2a74f1b 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -38,7 +38,7 @@ import { agents } from "./agents.js"; import { efforts, models } from "./models.js"; import { evaluateOpenRca, reproduceOpenRca } from "./openrca.js"; import { reportRuns } from "./report.js"; -import { isRunId } from "./runs.js"; +import { isRunId, parseRows } from "./runs.js"; const apiKey = process.env.ANTHROPIC_API_KEY; delete process.env.ANTHROPIC_API_KEY; @@ -87,6 +87,8 @@ const commands = { flags.agent === undefined && resume !== undefined ? undefined : choose(agents, flags.agent, "agent"); + const rows = flags.rows === undefined ? undefined : parseRows(flags.rows, scenarios); + if (typeof rows === "string") fail(rows); const options = { ...dirs, scenarios, @@ -94,7 +96,7 @@ const commands = { runs: flags.runs === undefined ? undefined : parseCount(flags.runs, "--runs"), resume, seed: readSeed(), - rows: flags.rows === undefined ? undefined : parseRows(flags.rows), + rows, jobs: flags.jobs === undefined ? undefined : parseCount(flags.jobs, "--jobs"), model: flags.model === undefined ? undefined : choose(models, flags.model, "model"), effort: flags.effort === undefined ? undefined : chooseEffort(flags.effort), @@ -154,9 +156,9 @@ const usage = [ " evorca --benchmark= [--scenario=...]", " [--archive=...]", " evorca evaluate --benchmark= --agent= [--scenario=...]", - " [--runs=] [--rows=,...] [--jobs=] [--model=]", - " [--effort=] [--budget-usd=] [--task-budget-usd=]", - " [--plain]", + " [--runs=] [--rows=[:],...] [--jobs=]", + " [--model=] [--effort=] [--budget-usd=]", + " [--task-budget-usd=] [--plain]", " evorca evaluate --benchmark= --resume= [--scenario=...]", " [--jobs=] [--budget-usd=] [--plain]", " evorca report --benchmark= [--scenario=...] ...", @@ -213,7 +215,7 @@ type Flags = { readonly archive?: readonly string[] | undefined; readonly agent?: string | undefined; readonly runs?: string | undefined; - readonly rows?: string | undefined; + readonly rows?: readonly string[] | undefined; readonly jobs?: string | undefined; readonly resume?: string | undefined; readonly model?: string | undefined; @@ -233,7 +235,7 @@ function parseCommandLine() { agent: { type: "string" }, archive: { type: "string", multiple: true }, runs: { type: "string" }, - rows: { type: "string" }, + rows: { type: "string", multiple: true }, jobs: { type: "string" }, resume: { type: "string" }, model: { type: "string" }, @@ -257,13 +259,6 @@ function readSeed(): number | undefined { return Number(seed); } -function parseRows(text: string): number[] { - if (!/^\d+(,\d+)*$/.test(text)) { - fail(`--rows must list row numbers, such as --rows=0,7,21, not ${JSON.stringify(text)}.`); - } - return [...new Set(text.split(",").map(Number))].sort((a, b) => a - b); -} - function parseCount(text: string, flag: string): number { if (!/^[1-9]\d*$/.test(text)) fail(`${flag} must be a positive integer, not ${JSON.stringify(text)}.`); diff --git a/packages/cli/src/openrca.ts b/packages/cli/src/openrca.ts index 965c526..4e7cf87 100644 --- a/packages/cli/src/openrca.ts +++ b/packages/cli/src/openrca.ts @@ -15,7 +15,6 @@ import { promisify } from "node:util"; import type { Diagnosis } from "@evorca/core"; import { openRcaCandidates, - openRcaScenarios, parsePrediction, readTasks, writePredictions, @@ -58,6 +57,7 @@ import { startModelProxy, type ModelProxy } from "./proxy.js"; import { proxyUrl, removeRunNetworks, startRelay, type Relay } from "./relay.js"; import { reportRuns } from "./report.js"; import { + allScenarios, conflicts, isFinished, readRuns, @@ -69,6 +69,7 @@ import { type GitState, type ModelSettings, type RunRecord, + type Rows, type Series, type TaskRecord, } from "./runs.js"; @@ -140,7 +141,7 @@ export type EvaluateRequest = { readonly agent: AgentName | undefined; readonly runs: number | undefined; readonly seed: number | undefined; - readonly rows: readonly number[] | undefined; + readonly rows: Rows | undefined; readonly jobs: number | undefined; readonly resume: string | undefined; readonly model: ModelId | undefined; @@ -263,7 +264,7 @@ export async function evaluateOpenRca(request: EvaluateRequest & Dirs): Promise< const finish = `pnpm evorca evaluate --benchmark=openrca --resume=${runIds[0]}`; console.log(`If they stop, finish them with: ${finish}`); - let sameAs = recorded; + let sameAs = recorded === undefined ? undefined : resumed; for (const [index, runId] of runIds.entries()) { const existing = await readRuns(runsDir, runId, scenarios); if (existing.size > 0 && [...existing.values()].every(isFinished)) { @@ -275,7 +276,7 @@ export async function evaluateOpenRca(request: EvaluateRequest & Dirs): Promise< const run = { ...request, scenarios, seed: firstSeed + index, resume }; if (!(await evaluateRun(run, member)) || request.signal.aborted) return false; } - sameAs ??= (await readRuns(runsDir, runId, scenarios)).values().next().value; + sameAs ??= await readRuns(runsDir, runId, scenarios); } if (!(await reportRuns({ runsDir, scenarios, runIds }))) return false; const flags = scenarios.map((scenario) => `--scenario=${scenario}`).join(" "); @@ -287,7 +288,7 @@ export async function evaluateOpenRca(request: EvaluateRequest & Dirs): Promise< type Member = { readonly runId: string; readonly series: Series; - readonly sameAs: RunRecord | undefined; + readonly sameAs: ReadonlyMap | undefined; }; async function evaluateRun( @@ -312,7 +313,8 @@ async function evaluateRun( return false; } const recorded = resumed?.values().next().value; - const sameAs = recorded ?? member?.sameAs; + const earlier = resumed ?? member?.sameAs; + const sameAs = earlier?.values().next().value; const name = sameAs?.agent.name ?? request.agent; if (name === undefined) throw new Error("evaluateOpenRca needs an agent or a run to resume."); const runId = recorded?.runId ?? member?.runId ?? newRunId(name); @@ -346,9 +348,10 @@ async function evaluateRun( } const tasks = new Map(); for (const scenario of scenarios) tasks.set(scenario, await readTasks(scenario, dataDir)); - const rows = request.rows ?? sameAs?.rows ?? null; + const rowsOf = (scenario: OpenRcaScenario) => + request.rows?.get(scenario) ?? earlier?.get(scenario)?.rows ?? null; for (const [scenario, scenarioTasks] of tasks) { - const unknown = rows?.filter((row) => row >= scenarioTasks.length) ?? []; + const unknown = rowsOf(scenario)?.filter((row) => row >= scenarioTasks.length) ?? []; if (unknown.length > 0) { const range = `rows 0 to ${scenarioTasks.length - 1}`; console.error(`${scenario} has no row ${unknown.join(", ")}. Its tasks are ${range}.`); @@ -381,7 +384,15 @@ async function evaluateRun( : null; if (sameAs !== undefined) { const seed = recorded === undefined ? undefined : request.seed; - const problems = conflicts(sameAs, { agent: agentRecord, seed, rows, model }); + const problems = [ + ...conflicts(sameAs, { agent: agentRecord, seed, model }), + ...scenarios.flatMap((scenario) => { + const record = earlier?.get(scenario); + if (record === undefined) return []; + const rows = rowsOf(scenario); + return conflicts(record, { rows }).map((problem) => `${scenario} ${problem}`); + }), + ]; if (problems.length > 0) { console.error( recorded === undefined @@ -427,6 +438,7 @@ async function evaluateRun( for (const [scenario, scenarioTasks] of tasks) { const runDir = join(runsDir, scenario, runId); const previous = resumed?.get(scenario); + const rows = rowsOf(scenario); const selected = rows === null ? scenarioTasks : scenarioTasks.filter((task) => rows.includes(task.row)); const pending = tasksToRun(selected, previous?.tasks ?? []); @@ -731,10 +743,6 @@ function totalUsage( return usages.reduce((sum, usage) => addUsage(sum, usage, prices.usdPerMTok), noUsage); } -const allScenarios = Object.keys(openRcaScenarios).filter((scenario): scenario is OpenRcaScenario => - Object.hasOwn(openRcaScenarios, scenario), -); - function newRunId(agent: AgentName): string { return `${new Date().toISOString().replace(/[-:]|\.\d+/g, "")}-${agent}`; } diff --git a/packages/cli/src/report.ts b/packages/cli/src/report.ts index 2a68a5e..98dc3bd 100644 --- a/packages/cli/src/report.ts +++ b/packages/cli/src/report.ts @@ -9,9 +9,9 @@ */ import { readFile } from "node:fs/promises"; import { join, relative } from "node:path"; -import { openRcaScenarios, type OpenRcaScenario } from "@evorca/openrca"; +import type { OpenRcaScenario } from "@evorca/openrca"; import type { Summary } from "@evorca/openrca/scoring"; -import { conflicts, isFinished, readRuns, type RunRecord } from "./runs.js"; +import { allScenarios, conflicts, isFinished, readRuns, type RunRecord } from "./runs.js"; type FinishedRun = { readonly record: RunRecord; @@ -21,9 +21,6 @@ type FinishedRun = { const levels = ["easy", "middle", "hard", "total"] as const; const measures = ["strict", "partial"] as const; -const everyScenario = Object.keys(openRcaScenarios).filter( - (key): key is OpenRcaScenario => key in openRcaScenarios, -); /** * Prints the summary of the runs `runIds` for each of `scenarios` that one of @@ -47,7 +44,7 @@ export async function reportRuns({ readonly runIds: readonly string[]; }): Promise { const runs = new Map>(); - for (const runId of runIds) runs.set(runId, await readRuns(runsDir, runId, everyScenario)); + for (const runId of runIds) runs.set(runId, await readRuns(runsDir, runId, allScenarios)); const missing = [...runs].filter(([, run]) => run.size === 0).map(([runId]) => runId); if (missing.length > 0) { console.error(`No run ${missing.join(", ")} in ${relative(process.cwd(), runsDir)}.`); diff --git a/packages/cli/src/runs.test.ts b/packages/cli/src/runs.test.ts index cae3490..f8ca7a9 100644 --- a/packages/cli/src/runs.test.ts +++ b/packages/cli/src/runs.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from "vitest"; import { agents } from "./agents.js"; import { conflicts, + parseRows, runPool, tasksToRun, writeJson, @@ -84,6 +85,46 @@ describe("conflicts", () => { }); }); +describe("parseRows", () => { + const scenarios = ["bank", "telecom", "market-cloudbed-1"] as const; + + it("gives every scenario the rows of a value without a scenario, in order and without repeats", () => { + expect(parseRows(["21,0,7,0"], scenarios)).toEqual( + new Map([ + ["bank", [0, 7, 21]], + ["telecom", [0, 7, 21]], + ["market-cloudbed-1", [0, 7, 21]], + ]), + ); + }); + + it("gives a scenario that a value names only those rows, and leaves the others whole", () => { + expect(parseRows(["telecom:3,1", "bank:0"], scenarios)).toEqual( + new Map([ + ["telecom", [1, 3]], + ["bank", [0]], + ]), + ); + }); + + it("gives the rows of a value without a scenario to each scenario that no value names", () => { + expect(parseRows(["bank:4", "0,7"], scenarios)).toEqual( + new Map([ + ["bank", [4]], + ["telecom", [0, 7]], + ["market-cloudbed-1", [0, 7]], + ]), + ); + }); + + it("refuses a value of another form, a scenario outside the run, and rows given twice", () => { + for (const values of [["0,,7"], ["telecom:"], ["x"], ["telecom:1", "telecom:2"], ["1", "2"]]) { + expect(parseRows(values, scenarios)).toEqual(expect.any(String)); + } + expect(parseRows(["market-cloudbed-2:0"], scenarios)).toContain("market-cloudbed-2"); + }); +}); + describe("tasksToRun", () => { const record = (row: number, outcome: TaskRecord["outcome"] = "answered"): TaskRecord => ({ row, diff --git a/packages/cli/src/runs.ts b/packages/cli/src/runs.ts index 1eff630..c24b4bc 100644 --- a/packages/cli/src/runs.ts +++ b/packages/cli/src/runs.ts @@ -8,7 +8,7 @@ import { readFile, rename, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { isDeepStrictEqual } from "node:util"; -import type { OpenRcaScenario, TaskIndex } from "@evorca/openrca"; +import { openRcaScenarios, type OpenRcaScenario, type TaskIndex } from "@evorca/openrca"; import type { pinnedInputs } from "@evorca/openrca/scoring"; import type { Agent, AgentName, Outcome } from "./agents.js"; import { isRecord, type Effort, type ModelId, type ModelUsage, type Prices } from "./models.js"; @@ -103,6 +103,11 @@ export type RunRecord = { readonly usage: ModelUsage | null; }; +/** Every OpenRCA scenario, in the order of `openRcaScenarios`. */ +export const allScenarios = Object.keys(openRcaScenarios).filter( + (scenario): scenario is OpenRcaScenario => Object.hasOwn(openRcaScenarios, scenario), +); + /** Whether `text` has the form of a run ID, such as `20261117T121537Z-random`. */ export function isRunId(text: string): boolean { return /^\d{8}T\d{6}Z-[a-z0-9-]+$/.test(text); @@ -113,6 +118,49 @@ export function taskDirOf(runDir: string, row: number): string { return join(runDir, "tasks", String(row).padStart(3, "0")); } +/** + * The rows that a run covers in each scenario that it does not cover whole, + * counted from 0 in the scenario's `query.csv`. + */ +export type Rows = ReadonlyMap; + +/** + * Parses the values of `--rows` for a run of `scenarios`. A value such as + * `0,7,21` gives the rows of each scenario that no other value names, and a + * value such as `telecom:0,7,21` gives the rows of one scenario. It returns + * each scenario's rows in order without repeats, or the problem if a value has + * another form, names a scenario outside `scenarios`, or gives rows that + * another value already gives. + */ +export function parseRows( + values: readonly string[], + scenarios: readonly OpenRcaScenario[], +): Rows | string { + const rows = new Map(); + let others: number[] | undefined; + for (const value of values) { + const [, name, list] = /^(?:([a-z0-9-]+):)?(\d+(?:,\d+)*)$/.exec(value) ?? []; + if (list === undefined) { + const forms = "--rows=0,7,21 or --rows=telecom:0,7,21"; + return `--rows must list row numbers, such as ${forms}, not ${JSON.stringify(value)}.`; + } + const numbers = [...new Set(list.split(",").map(Number))].sort((a, b) => a - b); + const scenario = scenarios.find((candidate) => candidate === name); + if (name !== undefined && scenario === undefined) { + return `--rows=${value} names ${name}, but the run covers ${scenarios.join(", ")}.`; + } + if (scenario === undefined ? others !== undefined : rows.has(scenario)) { + return `Two --rows values give the rows of ${scenario ?? "every scenario"}.`; + } + if (scenario === undefined) others = numbers; + else rows.set(scenario, numbers); + } + for (const scenario of scenarios) { + if (others !== undefined && !rows.has(scenario)) rows.set(scenario, others); + } + return rows; +} + /** The settings of a run that its results depend on. A resumed run keeps each of them. */ export type Conditions = Pick; From 5f28ec8998b2fb81f999ba788c314dbd14d5c04a Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Wed, 18 Nov 2026 13:54:09 +0200 Subject: [PATCH 3/3] feat(cli): pick and run a benchmark with /benchmark in the shell /benchmark opens a picker of four steps: the scenarios, the tasks of each with their difficulty, the agent and its settings, and a confirmation with the evorca evaluate command that repeats the run and, for an agent that calls a model, a cost estimate from finished runs. The picker is a pure state machine in benchmark.ts, and benchmark-view.tsx only draws it. It never shows the scoring points. The run goes in the shell's process, in the view of evorca watch, with the input line under the feed. ctrl+c stops it and returns to the transcript with the run's lines and the command that finishes it. A second ctrl+c stops at once. /exit asks first, then stops the run and leaves. The shell's palette and chrome move to parts.tsx because the shell now imports the watch view, which imported them from the shell. --- .changeset/cli-benchmark.md | 5 + packages/cli/README.md | 14 +- packages/cli/src/benchmark-view.tsx | 204 ++++++++++++++ packages/cli/src/benchmark.test.ts | 286 +++++++++++++++++++ packages/cli/src/benchmark.ts | 420 ++++++++++++++++++++++++++++ packages/cli/src/commands.ts | 25 +- packages/cli/src/evaluate-view.tsx | 2 +- packages/cli/src/main.ts | 34 ++- packages/cli/src/openrca.ts | 18 +- packages/cli/src/parts.tsx | 96 +++++++ packages/cli/src/shell.test.tsx | 253 ++++++++++++++++- packages/cli/src/shell.tsx | 314 ++++++++++++--------- packages/cli/src/watch-view.tsx | 25 +- 13 files changed, 1527 insertions(+), 169 deletions(-) create mode 100644 .changeset/cli-benchmark.md create mode 100644 packages/cli/src/benchmark-view.tsx create mode 100644 packages/cli/src/benchmark.test.ts create mode 100644 packages/cli/src/benchmark.ts create mode 100644 packages/cli/src/parts.tsx diff --git a/.changeset/cli-benchmark.md b/.changeset/cli-benchmark.md new file mode 100644 index 0000000..6cb6daf --- /dev/null +++ b/.changeset/cli-benchmark.md @@ -0,0 +1,5 @@ +--- +"@evorca/cli": minor +--- + +`/benchmark` in the shell picks scenarios, tasks, and an agent, shows the `evorca evaluate` command and, for an agent that calls a model, an estimate of the cost from earlier runs, then runs the evaluation in the shell in the view of `evorca watch`. ctrl+c stops the run and returns to the shell, and the run's lines and summary stay in the transcript. diff --git a/packages/cli/README.md b/packages/cli/README.md index cbfbff2..0edd7d6 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -5,7 +5,7 @@ The `evorca` command. From the repository root, `pnpm evorca` runs it without a ## Scope - An interactive shell in the terminal, built with [OpenTUI](https://github.com/anomalyco/opentui), which `evorca` opens when it runs without arguments. It opens on a home screen with the evorca wordmark and an input line, then shows a transcript of the session above the input line. A footer shows the working directory and the version. -- The slash commands `/help` and `/exit`. +- The slash commands `/help`, `/benchmark`, and `/exit`. `/benchmark` picks scenarios, tasks, and an agent, then runs `evorca evaluate` in the shell's process, in the view of `evorca watch`. - `evorca fetch`, which downloads the answer keys and the telemetry of a benchmark's scenarios into `data/benchmarks///` at the repository root, or reads them from dataset archives on disk. - `evorca reproduce`, which downloads only the answer keys and checks that evorca reproduces the scores that a benchmark publishes. It keeps the scorer's checkout and environment in `.cache/` at the repository root. - `evorca evaluate`, which runs an agent from `agents/` on a benchmark's scenarios, one Docker container per task and several tasks at a time, scores the answers, and keeps each run in `data/runs///` at the repository root. A run that stops can resume from its last finished task. `--runs` repeats a run with the same settings. An agent that calls a model reaches the Anthropic API only through a proxy that holds the API key and the budgets. @@ -83,10 +83,22 @@ pnpm evorca The shell opens on the home screen. The wordmark is a panel of dots, like an LED display. Its dots switch on one at a time, and every few seconds a few of them dim for a moment. While the input line is empty, its placeholder types out example questions. Type a message and press Enter. After the first message, the transcript replaces the home screen. The transcript is a log. Each line you send appears in it, followed by the reply, and each entry starts with the local time and `you` or `evorca`. These commands are available: - `/help` lists the commands. +- `/benchmark` picks an evaluation and runs it in the shell, as [Run a benchmark from the shell](#run-a-benchmark-from-the-shell) explains. - `/exit` closes evorca. ctrl+c does the same. The shell paints the whole screen in grays on a near-black background, so it looks the same in light and dark terminals. When the terminal is narrower than 73 columns, the plain name replaces the wordmark. In the transcript view, nothing moves while the input line holds text, and the shell does no work until you type or resize the terminal. The shell needs an interactive terminal. When stdin or stdout is not one, as in `echo hi | pnpm evorca`, it prints `evorca needs an interactive terminal.` and exits with code 1. +### Run a benchmark from the shell + +`/benchmark` opens a picker of four steps in place of the transcript. ↑ and ↓ move, Enter goes to the next step, and Esc goes back. On the first step, Esc closes the picker. + +1. Scenarios. Space chooses a scenario, and several can be chosen. Each line shows the number of tasks and whether the scenario's telemetry is on disk. A scenario without answer keys cannot be chosen, and its line gives the `evorca fetch` command. +2. Tasks. The tasks of each chosen scenario, each with its row, its task index, its difficulty, and the start of its instruction. Every task starts chosen. Space toggles one task, or every task of a scenario on its scenario's line. `e`, `m`, and `h` toggle the easy, middle, and hard tasks. The picker never shows the scoring points. +3. Agent and runs. ← and → change the agent and the number of runs. For an agent that calls a model, they also change the model, the effort, the run budget, and the task budget, which start at the defaults of `evorca evaluate`. If the agent reads telemetry that a chosen scenario lacks, the step gives the `evorca fetch` command and goes no further. +4. Run. The tasks, the agent, the seed, and the `evorca evaluate` command that repeats the run. A scenario with tasks left out gets its rows in the command as `--rows=:`. For an agent that calls a model, the step also shows an estimate of the cost: the mean cost per task of the finished runs in `data/runs/` with the same agent, model, and effort, times the tasks to run. Without such a run, it shows no estimate. + +Enter on the last step writes the command to the transcript and runs it in the shell's process. The shell shows the run in the view of [`evorca watch`](#watch-a-run), with the lines that the run prints under the feed and the input line below them. When the run ends, the transcript returns with every line that the run printed, the accuracy and the summary among them. ctrl+c stops the run as it stops `evorca evaluate`. Once the run has stopped, the transcript returns, and its last line is the command that resumes the run or finishes the series. A second ctrl+c stops the run at once, prints its lines, and closes evorca. During a run, `/exit` asks first. A second `/exit` stops the run, and evorca prints the run's lines and closes once the run has stopped. + ## Fetch a scenario You need network access to drive.usercontent.google.com. From the repository root, run: diff --git a/packages/cli/src/benchmark-view.tsx b/packages/cli/src/benchmark-view.tsx new file mode 100644 index 0000000..aa8b8c8 --- /dev/null +++ b/packages/cli/src/benchmark-view.tsx @@ -0,0 +1,204 @@ +/** + * The `/benchmark` picker in the shell. It draws the step that `benchmark.ts` + * models, with a dot for each line like the board of `evorca watch`: lit when + * chosen, dark when not, and `×` when it cannot be chosen. It turns keys into + * the picker's actions, and shows a window of the lines around the cursor. + * + * @module + */ +// @jsxRuntime automatic +// @jsxImportSource @opentui/react +import { useKeyboard, useTerminalDimensions } from "@opentui/react"; +import { useState } from "react"; +import { + blockerOf, + commandOf, + estimateOf, + itemsOf, + openPicker, + planOf, + tasksOf, + update, + type Action, + type Catalog, + type Item, + type Plan, + type Step, +} from "./benchmark.js"; +import { gutter, Hints, palette } from "./parts.js"; + +/** + * The props of {@link BenchmarkPicker}. `seed` is the seed of the run that it + * picks. It calls `onCancel` when the user leaves its first step, and + * `onStart` when the user confirms the run. + */ +export type BenchmarkPickerProps = { + readonly catalog: Catalog; + readonly seed: number; + readonly onCancel: () => void; + readonly onStart: (plan: Plan) => void; +}; + +const actions: Readonly> = { + up: "up", + down: "down", + space: "toggle", + left: "left", + right: "right", + return: "next", + escape: "back", + e: "easy", + m: "middle", + h: "hard", +}; +const titles: Readonly> = { + scenarios: "Scenarios", + tasks: "Tasks", + agent: "Agent and runs", + confirm: "Run", +}; +const hints: Readonly> = { + scenarios: [ + ["↑↓", "move"], + ["space", "choose"], + ["enter", "next"], + ["esc", "close"], + ], + tasks: [ + ["↑↓", "move"], + ["space", "choose"], + ["e m h", "easy, middle, hard"], + ["enter", "next"], + ["esc", "back"], + ], + agent: [ + ["↑↓", "move"], + ["←→", "change"], + ["enter", "next"], + ["esc", "back"], + ], + confirm: [ + ["enter", "run"], + ["esc", "back"], + ], +}; +const chrome = 7; + +/** The picker, from its first step to the confirmed run. */ +export function BenchmarkPicker({ catalog, seed, onCancel, onStart }: BenchmarkPickerProps) { + const { width, height } = useTerminalDimensions(); + const [picker, setPicker] = useState(() => openPicker(catalog, seed)); + + useKeyboard((key) => { + const action = actions[key.name]; + if (key.ctrl || key.meta || action === undefined) return; + const next = update(picker, action, catalog); + if (next === "cancel") onCancel(); + else if (next === "start") onStart(planOf(picker, catalog)); + else setPicker(next); + }); + + const items = itemsOf(picker, catalog); + const size = Math.max(1, height - chrome); + const start = Math.max(0, Math.min(items.length - size, picker.cursor - Math.floor(size / 2))); + const step = Object.keys(titles).indexOf(picker.step) + 1; + return ( + + + {titles[picker.step]} + {` openrca, step ${step} of 4`} + + + {picker.step === "confirm" ? ( + + ) : ( + items + .slice(start, start + size) + .map((item, index) => ( + + )) + )} + + + {blockerOf(picker, catalog) ?? ""} + + + + + + ); +} + +function Line({ item, cursor }: { readonly item: Item; readonly cursor: boolean }) { + const [dot, text, note] = partsOf(item); + return ( + + + {cursor ? "› " : " "} + {item.kind === "scenario" && item.problem !== undefined ? "×" : "•"} + {` ${text}`} + + + {note === "" ? "" : ` ${note}`} + + + ); +} + +function partsOf(item: Item): readonly [dot: string, text: string, note: string] { + const ink = (chosen: boolean) => (chosen ? palette.bright : palette.border); + switch (item.kind) { + case "scenario": { + const { scenario, tasks, telemetry } = item.data; + const facts = `${tasks.length} tasks, ${telemetry ? "with" : "no"} telemetry`; + return [ink(item.chosen), scenario.padEnd(17), item.problem ?? facts]; + } + case "group": + return [ink(item.chosen > 0), item.scenario, `${item.chosen} of ${item.tasks} tasks`]; + case "task": { + const { row, taskIndex, instruction } = item.task; + const label = ` ${String(row).padStart(3, "0")} ${taskIndex} ${item.difficulty.padEnd(6)}`; + return [ink(item.chosen), label, instruction.replaceAll(/\s+/g, " ")]; + } + case "setting": + return [palette.muted, `${item.label.padEnd(12)} ${item.value}`, ""]; + } +} + +function Confirm({ plan, catalog }: { readonly plan: Plan; readonly catalog: Catalog }) { + const estimate = estimateOf(plan, catalog); + const scenarios = plan.scenarios.map((scenario) => { + const whole = catalog.scenarios.find((data) => data.scenario === scenario)?.tasks.length; + const rows = plan.rows.get(scenario)?.length ?? whole; + return `${scenario} ${rows} of ${whole}`; + }); + const facts: (readonly [string, string])[] = [ + ["tasks", `${tasksOf(plan, catalog)} per run: ${scenarios.join(", ")}`], + ["agent", `${plan.agent}, ${plan.runs} ${plan.runs === 1 ? "run" : "runs"}, seed ${plan.seed}`], + ...(plan.model === undefined + ? [] + : ([ + ["model", `${plan.model} ${plan.effort}`], + ["budget", `$${plan.budgetUsd} per run, $${plan.taskBudgetUsd} per task`], + [ + "estimate", + estimate === undefined + ? "none, no finished run of this agent and model" + : `about $${estimate.usd.toFixed(2)}, at $${estimate.perTaskUsd.toFixed(4)} per task over ${estimate.from} tasks of finished runs`, + ], + ] as const)), + ["command", commandOf(plan)], + ]; + return ( + + {facts.map(([label, value]) => ( + + + {label.padEnd(8)} + + {value} + + ))} + + ); +} diff --git a/packages/cli/src/benchmark.test.ts b/packages/cli/src/benchmark.test.ts new file mode 100644 index 0000000..3efd63c --- /dev/null +++ b/packages/cli/src/benchmark.test.ts @@ -0,0 +1,286 @@ +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OpenRcaScenario, OpenRcaTask, TaskIndex } from "@evorca/openrca"; +import { describe, expect, it } from "vitest"; +import { + blockerOf, + commandOf, + estimateOf, + itemsOf, + openPicker, + planOf, + readCatalog, + update, + type Action, + type Catalog, + type Picker, +} from "./benchmark.js"; +import { noUsage } from "./models.js"; +import { parseRows, type RunRecord } from "./runs.js"; +import { runRecord, taskRecord } from "./watch-fixtures.js"; + +function tasks(scenario: OpenRcaScenario, indices: readonly TaskIndex[]): OpenRcaTask[] { + return indices.map((taskIndex, row) => ({ + scenario, + row, + taskIndex, + instruction: `Find the cause of ${scenario} task ${row}.`, + })); +} + +const catalog: Catalog = { + scenarios: [ + { scenario: "bank", tasks: tasks("bank", ["task_1", "task_5"]), telemetry: false }, + { + scenario: "telecom", + tasks: tasks("telecom", ["task_1", "task_4", "task_7", "task_2", "task_6"]), + telemetry: true, + }, + { scenario: "market-cloudbed-1", tasks: [], telemetry: false }, + ], + runs: [], +}; + +function press(picker: Picker, ...actions: Action[]): Picker { + return actions.reduce((state, action) => { + const next = update(state, action, catalog); + if (typeof next === "string") throw new Error(`The picker answered ${next} to ${action}.`); + return next; + }, picker); +} + +const start = openPicker(catalog, 7); +const bothChosen = press(start, "toggle", "down", "toggle", "next"); + +describe("the picker", () => { + it("chooses scenarios with tasks, and not one without tasks, which names the fetch command", () => { + const picker = press(start, "toggle", "down", "down", "toggle"); + const items = itemsOf(picker, catalog); + + expect(items.map((item) => item.kind === "scenario" && item.chosen)).toEqual([ + true, + false, + false, + ]); + expect(items[2]).toMatchObject({ + problem: expect.stringContaining( + "pnpm evorca fetch --benchmark=openrca --scenario=market-cloudbed-1", + ), + }); + }); + + it("goes on from the scenarios only once one is chosen, and closes on back", () => { + expect(update(start, "next", catalog)).toBe(start); + expect(update(start, "back", catalog)).toBe("cancel"); + expect(bothChosen.step).toBe("tasks"); + }); + + it("lists the tasks of each chosen scenario under its name, every task chosen", () => { + const lines = itemsOf(bothChosen, catalog).map((item) => + item.kind === "group" + ? `${item.scenario} ${item.chosen}/${item.tasks}` + : item.kind === "task" + ? `${item.chosen ? "x" : "-"} ${item.task.row} ${item.task.taskIndex} ${item.difficulty}` + : "", + ); + + expect(lines).toEqual([ + "bank 2/2", + "x 0 task_1 easy", + "x 1 task_5 middle", + "telecom 5/5", + "x 0 task_1 easy", + "x 1 task_4 middle", + "x 2 task_7 hard", + "x 3 task_2 easy", + "x 4 task_6 middle", + ]); + }); + + it("drops one task, the tasks of a difficulty in the scenario under the cursor, or a whole scenario", () => { + const onTelecom = press(bothChosen, "down", "down", "down", "down"); + const oneDropped = press(onTelecom, "toggle"); + const middleDropped = press(onTelecom, "middle"); + const middleBack = press(middleDropped, "middle"); + const bankDropped = press(bothChosen, "toggle"); + + expect(planOf(oneDropped, catalog).rows).toEqual(new Map([["telecom", [1, 2, 3, 4]]])); + expect(planOf(middleDropped, catalog).rows).toEqual(new Map([["telecom", [0, 2, 3]]])); + expect(planOf(middleBack, catalog).rows).toEqual(new Map()); + expect(planOf(bankDropped, catalog).scenarios).toEqual(["telecom"]); + }); + + it("chooses every task of a difficulty when some of them are dropped", () => { + const onTelecom = press(bothChosen, "down", "down", "down", "down"); + const picker = press(onTelecom, "toggle", "easy"); + + expect(planOf(picker, catalog).rows).toEqual(new Map()); + }); + + it("goes on from the tasks only while one is chosen", () => { + const none = press(bothChosen, "toggle", "down", "down", "down", "toggle"); + + expect(update(none, "next", catalog)).toBe(none); + expect(press(bothChosen, "next").step).toBe("agent"); + }); + + it("shows the model settings only for an agent that calls a model, and changes the one under the cursor", () => { + const agentStep = press(bothChosen, "next"); + const random = itemsOf(agentStep, catalog).map((item) => item.kind === "setting" && item.label); + const react = press(agentStep, "right", "down", "right", "down", "down", "right"); + const settings = itemsOf(react, catalog).map( + (item) => item.kind === "setting" && `${item.label} ${item.value}`, + ); + + expect(random).toEqual(["agent", "runs"]); + expect(settings).toEqual([ + "agent react", + "runs 2", + "model claude-sonnet-5", + "effort xhigh", + "run budget $50", + "task budget $2", + ]); + }); + + it("refuses an agent that reads telemetry that a chosen scenario lacks, and names the fetch command", () => { + const react = press(bothChosen, "next", "right"); + const telecomOnly = press(start, "down", "toggle", "next", "next", "right"); + + expect(update(react, "next", catalog)).toBe(react); + expect(blockerOf(react, catalog)).toBe( + "Fetch the telemetry that react reads with pnpm evorca fetch --benchmark=openrca --scenario=bank", + ); + expect(press(telecomOnly, "next").step).toBe("confirm"); + }); + + it("starts the run once confirmed, and steps back on back", () => { + const confirm = press(bothChosen, "next", "next"); + + expect(update(confirm, "next", catalog)).toBe("start"); + expect(press(confirm, "back").step).toBe("agent"); + }); +}); + +describe("commandOf", () => { + const oneLine = (command: string) => command.replaceAll(" \\\n ", " "); + + it("repeats the pick with the seed, each scenario, the rows of each scenario it does not run whole, and the runs", () => { + const picker = press(bothChosen, "down", "toggle", "next", "down", "right"); + const plan = planOf(picker, catalog); + + expect(oneLine(commandOf(plan))).toBe( + "EVORCA_SEED=7 pnpm evorca evaluate --benchmark=openrca --agent=random --scenario=bank" + + " --scenario=telecom --rows=bank:1 --runs=2", + ); + }); + + it("gives the rows that --rows parses back to the plan's", () => { + const picker = press(bothChosen, "down", "toggle", "down", "down", "down", "hard"); + const plan = planOf(picker, catalog); + const values = oneLine(commandOf(plan)) + .split(" ") + .filter((flag) => flag.startsWith("--rows=")) + .map((flag) => flag.slice("--rows=".length)); + + expect(parseRows(values, plan.scenarios)).toEqual(plan.rows); + expect(plan.rows).toEqual( + new Map([ + ["bank", [1]], + ["telecom", [0, 1, 3, 4]], + ]), + ); + }); + + it("names every model setting for an agent that calls a model", () => { + const telecomOnly = press(start, "down", "toggle", "next", "next", "right"); + + expect(oneLine(commandOf(planOf(telecomOnly, catalog)))).toBe( + "EVORCA_SEED=7 pnpm evorca evaluate --benchmark=openrca --agent=react --scenario=telecom" + + " --model=claude-sonnet-5 --effort=high --budget-usd=50 --task-budget-usd=2", + ); + }); + + it("breaks into lines of at most 72 columns that continue with a backslash", () => { + const telecomOnly = press(start, "down", "toggle", "next", "next", "right"); + + expect(commandOf(planOf(telecomOnly, catalog)).split("\n")).toEqual([ + "EVORCA_SEED=7 pnpm evorca evaluate --benchmark=openrca --agent=react \\", + " --scenario=telecom --model=claude-sonnet-5 --effort=high \\", + " --budget-usd=50 --task-budget-usd=2", + ]); + }); +}); + +describe("estimateOf", () => { + const react = (changes: Partial) => + runRecord({ + endedAt: "2026-11-18T10:00:00.000Z", + tasks: [taskRecord(0, "answered", 0.3), taskRecord(1, "answered", 0.5)], + usage: { ...noUsage, costUsd: 1.5 }, + ...changes, + }); + const { model } = runRecord(); + const telecomOnly = press(start, "down", "toggle", "next", "next", "right", "down", "right"); + + it("multiplies the tasks of every run by the mean cost per task of finished runs of the same agent and model", () => { + const runs = [react({}), react({ usage: { ...noUsage, costUsd: 0.5 } })]; + const estimate = estimateOf(planOf(telecomOnly, catalog), { ...catalog, runs }); + + expect(estimate).toEqual({ usd: 5, perTaskUsd: 0.5, from: 4 }); + }); + + it("gives none without a finished run of the same agent, model, and effort, or for an agent that calls no model", () => { + const xhigh = react({ model: model && { ...model, effort: "xhigh" } }); + const unfinished = react({ endedAt: null }); + const random = press(start, "down", "toggle", "next", "next"); + + expect( + estimateOf(planOf(telecomOnly, catalog), { ...catalog, runs: [xhigh, unfinished] }), + ).toBe(undefined); + expect(estimateOf(planOf(random, catalog), { ...catalog, runs: [react({})] })).toBe(undefined); + }); +}); + +describe("readCatalog", () => { + it("reads each scenario's tasks without their scoring points, its telemetry, and the runs on disk", async () => { + const root = await mkdtemp(join(tmpdir(), "evorca-catalog-")); + const dataDir = join(root, "data"); + const runsDir = join(root, "runs"); + await mkdir(join(dataDir, "telecom", "telemetry"), { recursive: true }); + await mkdir(join(dataDir, "bank"), { recursive: true }); + await writeFile( + join(dataDir, "telecom", "query.csv"), + "task_index,instruction,scoring_points\ntask_2,Find the reason.,SECRET\n", + ); + await writeFile( + join(dataDir, "bank", "query.csv"), + "task_index,instruction,scoring_points\ntask_7,Find it all.,SECRET\n", + ); + const record = runRecord(); + await mkdir(join(runsDir, "telecom", record.runId), { recursive: true }); + await writeFile(join(runsDir, "telecom", record.runId, "run.json"), JSON.stringify(record)); + + const read = await readCatalog(dataDir, runsDir); + + expect(read.scenarios).toEqual([ + { + scenario: "bank", + tasks: [{ scenario: "bank", row: 0, taskIndex: "task_7", instruction: "Find it all." }], + telemetry: false, + }, + { + scenario: "telecom", + tasks: [ + { scenario: "telecom", row: 0, taskIndex: "task_2", instruction: "Find the reason." }, + ], + telemetry: true, + }, + { scenario: "market-cloudbed-1", tasks: [], telemetry: false }, + { scenario: "market-cloudbed-2", tasks: [], telemetry: false }, + ]); + expect(JSON.stringify(read.scenarios)).not.toContain("SECRET"); + expect(read.runs).toEqual([record]); + }); +}); diff --git a/packages/cli/src/benchmark.ts b/packages/cli/src/benchmark.ts new file mode 100644 index 0000000..67f11f6 --- /dev/null +++ b/packages/cli/src/benchmark.ts @@ -0,0 +1,420 @@ +/** + * What `/benchmark` in the shell picks. A picker is a state machine over four + * steps: the scenarios, the tasks in each, the agent and its settings, and a + * confirmation. `update` takes it to its next state on each key, `itemsOf` + * lists the lines of a step, and `planOf` turns the pick into the request for + * `evorca evaluate` and the command that repeats it. + * + * A task shows only its row, task index, difficulty, and instruction. The + * picker never reads the scoring points or `record.csv`. + * + * @module + */ +import { existsSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { + difficultyOf, + readTasks, + type Difficulty, + type OpenRcaScenario, + type OpenRcaTask, +} from "@evorca/openrca"; +import { agents, type AgentName } from "./agents.js"; +import { efforts, models, type Effort, type ModelId } from "./models.js"; +import { runDefaults, type EvaluateRequest } from "./openrca.js"; +import { allScenarios, isFinished, isRunId, readRuns, type RunRecord, type Rows } from "./runs.js"; + +/** + * A scenario of the benchmark: its tasks, none without `query.csv`, and + * whether its telemetry is on disk. + */ +export type ScenarioData = { + readonly scenario: OpenRcaScenario; + readonly tasks: readonly OpenRcaTask[]; + readonly telemetry: boolean; +}; + +/** What the picker offers: the scenarios, and the runs on disk that the cost estimate learns from. */ +export type Catalog = { + readonly scenarios: readonly ScenarioData[]; + readonly runs: readonly RunRecord[]; +}; + +/** + * The agent, how many runs to make, and the model settings, which apply only + * to an agent that calls a model. + */ +export type Settings = { + readonly agent: AgentName; + readonly runs: number; + readonly model: ModelId; + readonly effort: Effort; + readonly budgetUsd: number; + readonly taskBudgetUsd: number; +}; + +const steps = ["scenarios", "tasks", "agent", "confirm"] as const; + +/** A step of the picker. */ +export type Step = (typeof steps)[number]; + +/** + * The state of the picker. `cursor` indexes the items of the step. + * `scenarios` holds the chosen scenarios, and `rows` the chosen rows of each + * scenario with tasks, at first all of them. `seed` is the seed of the run. + */ +export type Picker = { + readonly step: Step; + readonly cursor: number; + readonly scenarios: ReadonlySet; + readonly rows: ReadonlyMap>; + readonly settings: Settings; + readonly seed: number; +}; + +/** + * A key of the picker. A difficulty chooses the tasks of that difficulty in + * the scenario under the cursor, or drops them if all are chosen. + */ +export type Action = "up" | "down" | "toggle" | "left" | "right" | "next" | "back" | Difficulty; + +/** + * One line of a step. A `problem` says why the line cannot be chosen, and + * how to fix that. + */ +export type Item = + | { + readonly kind: "scenario"; + readonly data: ScenarioData; + readonly chosen: boolean; + readonly problem: string | undefined; + } + | { + readonly kind: "group"; + readonly scenario: OpenRcaScenario; + readonly chosen: number; + readonly tasks: number; + } + | { + readonly kind: "task"; + readonly task: OpenRcaTask; + readonly difficulty: Difficulty; + readonly chosen: boolean; + } + | { readonly kind: "setting"; readonly label: string; readonly value: string }; + +/** + * What a pick runs, as a request for `evorca evaluate`. `rows` holds only the + * scenarios that the run does not cover whole. + */ +export type Plan = Pick< + EvaluateRequest, + "scenarios" | "model" | "effort" | "budgetUsd" | "taskBudgetUsd" +> & { + readonly agent: AgentName; + readonly runs: number; + readonly seed: number; + readonly rows: Rows; +}; + +type Setting = { + readonly label: string; + readonly show: (value: T) => string; + readonly change: (value: T, by: 1 | -1) => T; +}; + +const settings: { readonly [K in keyof Settings]: Setting } = { + agent: { label: "agent", show: String, change: (agent, by) => cycle(agentNames, agent, by) }, + runs: { label: "runs", show: String, change: (runs, by) => Math.max(1, runs + by) }, + model: { label: "model", show: String, change: (model, by) => cycle(modelIds, model, by) }, + effort: { label: "effort", show: String, change: (effort, by) => cycle(efforts, effort, by) }, + budgetUsd: { + label: "run budget", + show: (usd) => `$${usd}`, + change: (usd, by) => Math.max(5, usd + 5 * by), + }, + taskBudgetUsd: { + label: "task budget", + show: (usd) => `$${usd}`, + change: (usd, by) => Math.max(0.5, usd + 0.5 * by), + }, +}; +const commandWidth = 72; +const agentNames = Object.keys(agents).filter((name): name is AgentName => name in agents); +const modelIds = Object.keys(models).filter((id): id is ModelId => id in models); + +/** + * Reads what the picker offers: the tasks and the telemetry of each scenario + * in `dataDir`, and the `run.json` of each run in `runsDir`. + * + * @throws Error if a `query.csv` or a `run.json` exists but cannot be read. + */ +export async function readCatalog(dataDir: string, runsDir: string): Promise { + const scenarios = await Array.fromAsync(allScenarios, async (scenario) => ({ + scenario, + tasks: existsSync(join(dataDir, scenario, "query.csv")) + ? await readTasks(scenario, dataDir) + : [], + telemetry: existsSync(join(dataDir, scenario, "telemetry")), + })); + const runs: RunRecord[] = []; + for (const scenario of allScenarios) { + const ids = await readdir(join(runsDir, scenario)).catch(() => []); + for (const id of ids.filter(isRunId)) { + runs.push(...(await readRuns(runsDir, id, [scenario])).values()); + } + } + return { scenarios, runs }; +} + +/** The picker at its first step, with no scenario chosen, every task chosen, and the default settings. */ +export function openPicker(catalog: Catalog, seed: number): Picker { + return { + step: "scenarios", + cursor: 0, + scenarios: new Set(), + rows: new Map( + catalog.scenarios.map(({ scenario, tasks }) => [ + scenario, + new Set(tasks.map((task) => task.row)), + ]), + ), + settings: { + agent: "random", + runs: 1, + model: runDefaults.model, + effort: models[runDefaults.model].defaultEffort, + budgetUsd: runDefaults.budgetUsd, + taskBudgetUsd: runDefaults.taskBudgetUsd, + }, + seed, + }; +} + +/** The lines of the picker's step, which the cursor moves over. The confirmation has none. */ +export function itemsOf(picker: Picker, catalog: Catalog): Item[] { + if (picker.step === "scenarios") { + return catalog.scenarios.map((data) => ({ + kind: "scenario", + data, + chosen: picker.scenarios.has(data.scenario), + problem: data.tasks.length > 0 ? undefined : `Fetch its tasks with ${fetch([data.scenario])}`, + })); + } + if (picker.step === "tasks") { + return chosen(picker, catalog).flatMap(({ scenario, tasks }): Item[] => { + const rows = picker.rows.get(scenario) ?? new Set(); + return [ + { kind: "group", scenario, chosen: rows.size, tasks: tasks.length }, + ...tasks.map((task): Item => ({ + kind: "task", + task, + difficulty: difficultyOf[task.taskIndex], + chosen: rows.has(task.row), + })), + ]; + }); + } + if (picker.step === "agent") { + return shownSettings(picker.settings).map((name) => ({ + kind: "setting", + label: settings[name].label, + value: shown(picker.settings, name), + })); + } + return []; +} + +/** Why the picker cannot go on from its step, or undefined if it can. */ +export function blockerOf(picker: Picker, catalog: Catalog): string | undefined { + if (picker.step === "scenarios" && picker.scenarios.size === 0) return "Choose a scenario."; + if (picker.step === "tasks" && planOf(picker, catalog).scenarios.length === 0) { + return "Choose a task."; + } + if (picker.step === "agent") return agentProblem(picker, catalog); + return undefined; +} + +/** + * The picker's next state after `action`, or `cancel` if it closes, or + * `start` if the user confirms the run that `planOf` gives. + */ +export function update( + picker: Picker, + action: Action, + catalog: Catalog, +): Picker | "cancel" | "start" { + const items = itemsOf(picker, catalog); + const item = items[picker.cursor]; + const step = steps.indexOf(picker.step); + if (action === "up" || action === "down") { + const cursor = picker.cursor + (action === "up" ? -1 : 1); + return { ...picker, cursor: Math.max(0, Math.min(items.length - 1, cursor)) }; + } + if (action === "back") { + const previous = steps[step - 1]; + return previous === undefined ? "cancel" : { ...picker, step: previous, cursor: 0 }; + } + if (action === "next") { + if (blockerOf(picker, catalog) !== undefined) return picker; + const next = steps[step + 1]; + return next === undefined ? "start" : { ...picker, step: next, cursor: 0 }; + } + if (action === "left" || action === "right") { + if (picker.step !== "agent") return picker; + const name = shownSettings(picker.settings)[picker.cursor]; + return name === undefined + ? picker + : { ...picker, settings: changed(picker.settings, name, action === "left" ? -1 : 1) }; + } + if (item?.kind === "scenario") { + if (action !== "toggle" || item.problem !== undefined) return picker; + const scenarios = new Set(picker.scenarios); + if (!scenarios.delete(item.data.scenario)) scenarios.add(item.data.scenario); + return { ...picker, scenarios }; + } + if (item?.kind !== "group" && item?.kind !== "task") return picker; + const scenario = item.kind === "group" ? item.scenario : item.task.scenario; + const tasks = catalog.scenarios.find((data) => data.scenario === scenario)?.tasks ?? []; + if (action === "toggle") + return flip(picker, scenario, item.kind === "task" ? [item.task] : tasks); + return flip( + picker, + scenario, + tasks.filter((task) => difficultyOf[task.taskIndex] === action), + ); +} + +/** The run that the picker's choices describe. */ +export function planOf(picker: Picker, catalog: Catalog): Plan { + const covered = chosen(picker, catalog).flatMap(({ scenario, tasks }) => { + const rows = picker.rows.get(scenario) ?? new Set(); + return rows.size === 0 ? [] : [{ scenario, rows: [...rows].sort((a, b) => a - b), tasks }]; + }); + const { agent, runs, model, effort, budgetUsd, taskBudgetUsd } = picker.settings; + const modelSettings = agents[agent].needsModel + ? { model, effort, budgetUsd, taskBudgetUsd } + : { model: undefined, effort: undefined, budgetUsd: undefined, taskBudgetUsd: undefined }; + return { + scenarios: covered.map(({ scenario }) => scenario), + rows: new Map( + covered.flatMap(({ scenario, rows, tasks }) => + rows.length === tasks.length ? [] : [[scenario, rows] as const], + ), + ), + agent, + runs, + seed: picker.seed, + ...modelSettings, + }; +} + +/** + * The command that runs `plan` from a terminal at the repository root. It + * breaks into lines of at most 72 columns that end in a backslash, so a copy + * of it runs as it is. + */ +export function commandOf(plan: Plan): string { + const words = [ + `EVORCA_SEED=${plan.seed}`, + "pnpm", + "evorca", + "evaluate", + "--benchmark=openrca", + `--agent=${plan.agent}`, + ...plan.scenarios.map((scenario) => `--scenario=${scenario}`), + ...[...plan.rows].map(([scenario, rows]) => `--rows=${scenario}:${rows.join(",")}`), + ...(plan.runs === 1 ? [] : [`--runs=${plan.runs}`]), + ...(plan.model === undefined ? [] : [`--model=${plan.model}`]), + ...(plan.effort === undefined ? [] : [`--effort=${plan.effort}`]), + ...(plan.budgetUsd === undefined ? [] : [`--budget-usd=${plan.budgetUsd}`]), + ...(plan.taskBudgetUsd === undefined ? [] : [`--task-budget-usd=${plan.taskBudgetUsd}`]), + ]; + const lines: string[] = []; + for (const word of words) { + const last = lines.at(-1); + if (last !== undefined && `${last} ${word} \\`.length <= commandWidth) { + lines[lines.length - 1] = `${last} ${word}`; + } else lines.push(last === undefined ? word : ` ${word}`); + } + return lines.join(" \\\n"); +} + +/** How many tasks `plan` runs in each of its runs. */ +export function tasksOf(plan: Plan, catalog: Catalog): number { + return plan.scenarios.reduce((sum, scenario) => { + const whole = catalog.scenarios.find((data) => data.scenario === scenario)?.tasks.length ?? 0; + return sum + (plan.rows.get(scenario)?.length ?? whole); + }, 0); +} + +/** + * Estimates what `plan` spends on the model: its tasks over all its runs + * times the mean cost per task of the finished runs in the catalog with the + * same agent, model, and effort. It is undefined for an agent that calls no + * model, and when no such run is finished. + */ +export function estimateOf( + plan: Plan, + catalog: Catalog, +): { readonly usd: number; readonly perTaskUsd: number; readonly from: number } | undefined { + const same = catalog.runs.filter( + (record) => + isFinished(record) && + record.agent.name === plan.agent && + record.model?.id === plan.model && + record.model?.effort === plan.effort, + ); + const from = same.reduce((sum, record) => sum + record.tasks.length, 0); + if (plan.model === undefined || from === 0) return undefined; + const perTaskUsd = same.reduce((sum, record) => sum + (record.usage?.costUsd ?? 0), 0) / from; + return { usd: perTaskUsd * tasksOf(plan, catalog) * plan.runs, perTaskUsd, from }; +} + +function chosen(picker: Picker, catalog: Catalog): readonly ScenarioData[] { + return catalog.scenarios.filter(({ scenario }) => picker.scenarios.has(scenario)); +} + +function agentProblem(picker: Picker, catalog: Catalog): string | undefined { + const { agent } = picker.settings; + const missing = chosen(picker, catalog) + .filter((data) => !data.telemetry) + .map((data) => data.scenario); + if (!agents[agent].needsTelemetry || missing.length === 0) return undefined; + return `Fetch the telemetry that ${agent} reads with ${fetch(missing)}`; +} + +function fetch(scenarios: readonly OpenRcaScenario[]): string { + const flags = scenarios.map((scenario) => `--scenario=${scenario}`).join(" "); + return `pnpm evorca fetch --benchmark=openrca ${flags}`; +} + +function shownSettings(current: Settings): (keyof Settings)[] { + const always = ["agent", "runs"] as const; + const model = ["model", "effort", "budgetUsd", "taskBudgetUsd"] as const; + return agents[current.agent].needsModel ? [...always, ...model] : [...always]; +} + +function shown(current: Settings, name: K): string { + return settings[name].show(current[name]); +} + +function changed(current: Settings, name: K, by: 1 | -1): Settings { + return { ...current, [name]: settings[name].change(current[name], by) }; +} + +function flip(picker: Picker, scenario: OpenRcaScenario, tasks: readonly OpenRcaTask[]): Picker { + const rows = new Set(picker.rows.get(scenario)); + const choose = !tasks.every((task) => rows.has(task.row)); + for (const { row } of tasks) { + if (choose) rows.add(row); + else rows.delete(row); + } + return { ...picker, rows: new Map(picker.rows).set(scenario, rows) }; +} + +function cycle(values: readonly T[], value: T, by: 1 | -1): T { + const index = values.indexOf(value); + return values[(index + by + values.length) % values.length] ?? value; +} diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts index 5644dd5..0207038 100644 --- a/packages/cli/src/commands.ts +++ b/packages/cli/src/commands.ts @@ -1,6 +1,7 @@ /** * What happens when the user submits a line: the transcript entries it adds, - * or an exit. The module owns the table of slash commands. + * the benchmark picker, or an exit. The module owns the table of slash + * commands. * * @module */ @@ -8,16 +9,27 @@ /** One message in the transcript, with the local time when it was added. */ export type Entry = { readonly role: "you" | "evorca"; readonly text: string; readonly at: Date }; -/** The result of submitting a line: entries to append to the transcript, or an exit. */ +/** + * The result of submitting a line: entries to append to the transcript, those + * entries and a request to open the benchmark picker, or an exit. + */ export type Outcome = - { readonly kind: "append"; readonly entries: readonly Entry[] } | { readonly kind: "exit" }; + | { readonly kind: "append" | "benchmark"; readonly entries: readonly Entry[] } + | { readonly kind: "exit" }; -type Response = { readonly kind: "reply"; readonly text: string } | { readonly kind: "exit" }; +type Response = + | { readonly kind: "reply"; readonly text: string } + | { readonly kind: "benchmark" } + | { readonly kind: "exit" }; type Command = { readonly description: string; readonly run: () => Response }; const commands: Readonly> = { "/help": { description: "List the commands", run: () => reply(help()) }, + "/benchmark": { + description: "Pick scenarios, tasks, and an agent, and run them", + run: () => ({ kind: "benchmark" }), + }, "/exit": { description: "Exit evorca", run: () => ({ kind: "exit" }) }, }; @@ -25,7 +37,8 @@ const commands: Readonly> = { * Returns the outcome of submitting `line` at the time `at`. A line that * starts with `/` runs the command of that name. Any other line is a message * for the agent, which is not connected yet. The appended entries are the - * trimmed line and the reply, both at `at`. A blank line appends nothing. + * trimmed line and the reply, both at `at`. `/benchmark` has no reply, so it + * appends only the line. A blank line appends nothing. */ export function submit(line: string, at: Date): Outcome { const text = line.trim(); @@ -34,6 +47,8 @@ export function submit(line: string, at: Date): Outcome { ? (commands[text]?.run() ?? reply(`Unknown command ${text}. Type /help for commands.`)) : reply("No agent is connected yet."); if (response.kind === "exit") return response; + if (response.kind === "benchmark") + return { kind: "benchmark", entries: [{ role: "you", text, at }] }; return { kind: "append", entries: [ diff --git a/packages/cli/src/evaluate-view.tsx b/packages/cli/src/evaluate-view.tsx index b5d49b5..c2166ef 100644 --- a/packages/cli/src/evaluate-view.tsx +++ b/packages/cli/src/evaluate-view.tsx @@ -21,7 +21,7 @@ import { format } from "node:util"; import { createCliRenderer } from "@opentui/core"; import { createRoot } from "@opentui/react"; -import { palette } from "./shell.js"; +import { palette } from "./parts.js"; import { readRunFolders, type Frame, type RunFolders } from "./watch-folder.js"; import { Watch } from "./watch-view.js"; import type { Selection } from "./watch.js"; diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 2a74f1b..996549b 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -2,7 +2,8 @@ * The `evorca` command, which `pnpm evorca` runs from the repository root. * Without arguments, `evorca` opens the interactive shell full screen, and * exits with code 1 when stdin or stdout is not a terminal. It loads the shell - * on demand, so the other commands never load OpenTUI's native library. + * on demand, so the other commands never load OpenTUI's native library. The + * shell's `/benchmark` runs `openrca` evaluations in the shell's process. * `evorca fetch` downloads the data of a benchmark's scenarios, the answer keys * and the telemetry, into `data/benchmarks///` at the * repository root. @@ -169,7 +170,20 @@ const { positionals, values } = parseCommandLine(); if (positionals.length === 0 && Object.keys(values).length === 0) { if (!process.stdin.isTTY || !process.stdout.isTTY) fail("evorca needs an interactive terminal."); const { openShell } = await import("./shell.js"); - await openShell(); + const dirs = dirsOf("openrca"); + const upstreamUrl = readUpstreamUrl(); + await openShell({ + ...dirs, + evaluate: (request) => + evaluateOpenRca({ + ...dirs, + ...request, + jobs: undefined, + resume: undefined, + apiKey, + upstreamUrl, + }), + }); } else { const [command, ...ids] = positionals; if (!isKeyOf(commands, command)) fail(usage); @@ -187,17 +201,17 @@ if (positionals.length === 0 && Object.keys(values).length === 0) { fail(`Unknown scenario: ${unknown.join(", ")}. Choose from: ${known.join(", ")}`); } + await commands[command]({ benchmark, scenarios, runIds, flags: values, ...dirsOf(name) }); +} + +function dirsOf(benchmark: keyof typeof benchmarks) { const root = new URL("../../../", import.meta.url); - await commands[command]({ - benchmark, - scenarios, - runIds, - flags: values, - dataDir: fileURLToPath(new URL(`data/benchmarks/${name}/`, root)), + return { + dataDir: fileURLToPath(new URL(`data/benchmarks/${benchmark}/`, root)), cacheDir: fileURLToPath(new URL(".cache/", root)), - runsDir: fileURLToPath(new URL(`data/runs/${name}/`, root)), + runsDir: fileURLToPath(new URL(`data/runs/${benchmark}/`, root)), agentsDir: fileURLToPath(new URL("agents/", root)), - }); + }; } async function printProgress(progress: AsyncIterable) { diff --git a/packages/cli/src/openrca.ts b/packages/cli/src/openrca.ts index 4e7cf87..b3c58be 100644 --- a/packages/cli/src/openrca.ts +++ b/packages/cli/src/openrca.ts @@ -190,9 +190,15 @@ type ModelSession = { readonly relay: Relay; }; -const defaultModel: ModelId = "claude-sonnet-5"; -const defaultBudgetUsd = 50; -const defaultTaskBudgetUsd = 2; +/** + * The model of a run that names none, and the budgets in US dollars of a run + * and of each of its tasks. + */ +export const runDefaults: { + readonly model: ModelId; + readonly budgetUsd: number; + readonly taskBudgetUsd: number; +} = { model: "claude-sonnet-5", budgetUsd: 50, taskBudgetUsd: 2 }; /** * Runs an agent on the tasks of `scenarios`, `jobs` tasks at a time, each in @@ -366,13 +372,13 @@ async function evaluateRun( const image = await ensureImage(name, agentsDir); const agentRecord: AgentRecord = { name, ...agent, image: image.tag, imageId: image.id }; - const modelId = request.model ?? sameAs?.model?.id ?? defaultModel; + const modelId = request.model ?? sameAs?.model?.id ?? runDefaults.model; const model: ModelSettings | null = agent.needsModel ? { id: modelId, effort: request.effort ?? sameAs?.model?.effort ?? models[modelId].defaultEffort, taskBudgetUsd: - request.taskBudgetUsd ?? sameAs?.model?.taskBudgetUsd ?? defaultTaskBudgetUsd, + request.taskBudgetUsd ?? sameAs?.model?.taskBudgetUsd ?? runDefaults.taskBudgetUsd, upstreamUrl: request.upstreamUrl, prices: { table: priceTable.id, @@ -404,7 +410,7 @@ async function evaluateRun( } } const budgetUsd = agent.needsModel - ? (request.budgetUsd ?? sameAs?.budgetUsd ?? defaultBudgetUsd) + ? (request.budgetUsd ?? sameAs?.budgetUsd ?? runDefaults.budgetUsd) : null; const seed = recorded?.seed ?? request.seed ?? randomInt(2 ** 32); const jobs = request.jobs ?? sameAs?.jobs ?? defaultJobs; diff --git a/packages/cli/src/parts.tsx b/packages/cli/src/parts.tsx new file mode 100644 index 0000000..854b6e7 --- /dev/null +++ b/packages/cli/src/parts.tsx @@ -0,0 +1,96 @@ +/** + * The parts that every evorca screen draws: the grays, the gutter, the name + * evorca, a line of keys, and the footer. The shell, the picker of + * `/benchmark`, and the view of `evorca watch` share them, so they live apart + * from each. + * + * @module + */ +// @jsxRuntime automatic +// @jsxImportSource @opentui/react +import { sep } from "node:path"; +import { TextAttributes } from "@opentui/core"; +import packageJson from "../package.json" with { type: "json" }; + +/** The grays that evorca draws with, each with equal red, green, and blue. */ +export const palette = { + background: "#0a0a0a", + border: "#333333", + muted: "#707070", + text: "#c8c8c8", + bright: "#f4f4f4", +} as const; + +/** The space in columns between the edge of the screen and the content. */ +export const gutter = 2; +const version = `evorca v${packageJson.version}`; +const separator = " "; + +/** The name evorca in bold, with `evo` dimmer than `rca`. */ +export function Mark() { + return ( + + evo + rca + + ); +} + +/** + * A line of keys and what they do. When the line is wider than `width`, it + * drops hints from the start until it fits. + */ +export function Hints({ + hints, + width, +}: { + readonly hints: readonly (readonly [string, string])[]; + readonly width: number; +}) { + const shown = hints.filter((_, index) => hintsWidth(hints.slice(index)) <= width); + return ( + + {shown.map(([key, label], index) => ( + + {key} + {` ${label}${index < shown.length - 1 ? separator : ""}`} + + ))} + + ); +} + +/** The bottom line: `directory`, shortened to fit `width`, and the version. */ +export function Footer({ + directory, + width, +}: { + readonly directory: string; + readonly width: number; +}) { + return ( + + {shorten(directory, width - 2 * gutter - version.length - 2)} + + + {version} + + + ); +} + +function hintsWidth(shown: readonly (readonly [string, string])[]): number { + return ( + shown.reduce((sum, [key, label]) => sum + key.length + 1 + label.length, 0) + + separator.length * (shown.length - 1) + ); +} + +function shorten(path: string, max: number): string { + const segments = path.split(sep); + for (let start = 0; start < segments.length; start++) { + const shown = start === 0 ? path : `…${sep}${segments.slice(start).join(sep)}`; + if (shown.length <= max) return shown; + } + return `…${path.slice(path.length - Math.max(0, max - 1))}`; +} diff --git a/packages/cli/src/shell.test.tsx b/packages/cli/src/shell.test.tsx index 9906e26..e97bfee 100644 --- a/packages/cli/src/shell.test.tsx +++ b/packages/cli/src/shell.test.tsx @@ -1,10 +1,15 @@ +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; import { RGBA } from "@opentui/core"; import { ManualClock, type TestRendererSetup } from "@opentui/core/testing"; import { testRender } from "@opentui/react/test-utils"; import { act } from "react"; import { afterEach, describe, expect, it, vi } from "vitest"; import packageJson from "../package.json" with { type: "json" }; -import { palette, Shell } from "./shell.js"; +import { palette } from "./parts.js"; +import { Shell, type Bench } from "./shell.js"; const wordmark = [ "#####.#...#..###..####...###...###.", @@ -22,16 +27,21 @@ const secondExample = "Find the root cause of the 5xx burst on payments"; let setup: TestRendererSetup; let clock: ManualClock; -let onExit: () => void; +let onExit: (lines?: readonly string[], code?: number) => void; let wallTime: Date; +let bench: Bench = { + dataDir: "/no/data", + runsDir: "/no/runs", + evaluate: () => Promise.resolve(true), +}; async function open(width = 100, height = 30, directory = "~/incidents") { clock = new ManualClock(); onExit = vi.fn(); wallTime = new Date(2026, 10, 17, 14, 5, 12); setup = await testRender( - wallTime} onExit={onExit} />, - { width, height, clock }, + wallTime} bench={bench} onExit={onExit} />, + { width, height, clock, exitOnCtrlC: false }, ); await setup.renderOnce(); return setup.captureCharFrame(); @@ -260,7 +270,7 @@ describe("Shell", () => { expect(frame).toMatch(/^ +14:05:12 +you +hello +$/m); expect(frame).toMatch(/^ +09:03:07 +you +\/help +$/m); - expect(frame).toMatch(/^ +09:03:07 +evorca +\/help {2}List the commands +$/m); + expect(frame).toMatch(/^ +09:03:07 +evorca +\/help +List the commands +$/m); }); it("stays home on a blank line", async () => { @@ -272,15 +282,17 @@ describe("Shell", () => { expect(frame).not.toMatch(/ you /); }); - it("lists the commands for /help, the second line under the first", async () => { - await open(); + it("lists the commands for /help, each line under the first", async () => { + await open(120); const lines = (await enter("/help")).split("\n"); - const first = lines.findIndex((line) => line.includes("/help List the commands")); - - expect(first).toBeGreaterThan(-1); - expect(lines[first + 1]?.indexOf("/exit Exit evorca")).toBe( - lines[first]?.indexOf("/help List the commands"), - ); + const first = lines.findIndex((line) => / \/help +List the commands/.test(line)); + const commands = lines.slice(first, first + 3); + const columns = (pattern: RegExp) => commands.map((line) => line.search(pattern)); + + expect(commands[1]).toMatch(/ \/benchmark +Pick scenarios, tasks, and an agent, and run them/); + expect(commands[2]).toMatch(/ \/exit +Exit evorca/); + expect(new Set(columns(/\/\w/)).size).toBe(1); + expect(new Set(columns(/(List|Pick|Exit) /)).size).toBe(1); }); it("names an unknown command", async () => { @@ -326,3 +338,218 @@ describe("Shell", () => { expect(onExit).toHaveBeenCalledTimes(1); }); }); + +describe("/benchmark", () => { + const answerKey = "ANSWER-KEY"; + const frames: string[] = []; + let finish: (ok: boolean) => void; + let request: Parameters[0] | undefined; + + afterEach(() => { + expect(frames.join("\n")).not.toContain(answerKey); + frames.length = 0; + request = undefined; + return act(() => setup.dispose()); + }); + + async function openWithData() { + const root = await mkdtemp(join(tmpdir(), "evorca-shell-")); + const query = (indices: readonly string[]) => + [ + "task_index,instruction,scoring_points", + ...indices.map( + (index, row) => `${index},Find the cause of failure ${row}.,${answerKey} ${row}`, + ), + ].join("\n"); + await mkdir(join(root, "data", "telecom", "telemetry"), { recursive: true }); + await mkdir(join(root, "data", "bank"), { recursive: true }); + await writeFile( + join(root, "data", "telecom", "query.csv"), + query(["task_1", "task_4", "task_7"]), + ); + await writeFile(join(root, "data", "bank", "query.csv"), query(["task_2", "task_6"])); + bench = { + dataDir: join(root, "data"), + runsDir: join(root, "runs"), + evaluate: (next) => + new Promise((resolve) => { + request = next; + console.log(`Running ${next.agent} on ${next.scenarios.join(", ")}.`); + next.onRun("20261118T120000Z-random"); + finish = (ok) => { + if (ok) console.log("telecom: 2 answered"); + resolve(ok); + }; + next.signal.addEventListener("abort", () => { + console.error( + "Stopped. Resume with: pnpm evorca evaluate --resume=20261118T120000Z-random", + ); + }); + }), + }; + await open(120, 30); + } + + async function show(): Promise { + await setup.renderOnce(); + const frame = setup.captureCharFrame(); + frames.push(frame); + return frame; + } + + async function until(text: string): Promise { + for (let tries = 0; tries < 100; tries++) { + await act(() => sleep(10)); + const frame = await show(); + if (frame.includes(text)) return frame; + } + throw new Error(`No frame showed ${text}:\n${setup.captureCharFrame()}`); + } + + async function keys(...names: string[]): Promise { + for (const name of names) { + await act(async () => { + if (name === "enter") setup.mockInput.pressEnter(); + else if (name === "escape") { + setup.mockInput.pressEscape(); + clock.setTime(clock.now() + 100); + } else if (name === "ctrl+c") setup.mockInput.pressCtrlC(); + else if (name === "up" || name === "down" || name === "left" || name === "right") { + setup.mockInput.pressArrow(name); + } else await setup.mockInput.typeText(name); + }); + } + await act(() => sleep(20)); + return show(); + } + + async function confirmRun() { + await openWithData(); + await enter("/benchmark"); + await until("Scenarios"); + await keys(" ", "down", " ", "enter"); + await keys("down", "down", "down", "down", "h", "enter"); + await keys("down", "right", "enter"); + return show(); + } + + it("lists each scenario with its tasks and telemetry, and how to fetch one without tasks", async () => { + await openWithData(); + await enter("/benchmark"); + const frame = await until("Scenarios"); + + expect(frame).toMatch(/› • bank +2 tasks, no telemetry/); + expect(frame).toMatch(/ • telecom +3 tasks, with telemetry/); + expect(frame).toMatch( + / × market-cloudbed-1 +Fetch its tasks with pnpm evorca fetch --benchmark=openrca --scenario=market-cloudbed-1 *\n/, + ); + }); + + it("shows each task's row, task index, difficulty, and instruction, and drops a difficulty", async () => { + await openWithData(); + await enter("/benchmark"); + await until("Scenarios"); + await keys(" ", "down", " ", "enter"); + const tasks = await keys("down", "down", "down", "down"); + const hardDropped = await keys("h"); + + expect(tasks).toMatch(/ • telecom +3 of 3 tasks/); + expect(tasks).toMatch(/› • +000 +task_1 +easy +Find the cause of failure 0\./); + expect(tasks).toMatch(/ • +002 +task_7 +hard +Find the cause of failure 2\./); + expect(hardDropped).toMatch(/ • telecom +2 of 3 tasks/); + }); + + it("confirms with the command that repeats the pick", async () => { + const frame = await confirmRun(); + + expect(frame).toMatch(/tasks +4 per run: bank 2 of 2, telecom 2 of 3/); + expect(frame).toMatch(/agent +random, 2 runs, seed \d+/); + expect(frame.replaceAll(/ \\ *\n +/g, " ")).toMatch( + /command +EVORCA_SEED=\d+ pnpm evorca evaluate --benchmark=openrca --agent=random --scenario=bank --scenario=telecom --rows=telecom:0,1 --runs=2 *\n/, + ); + }); + + it("runs the pick in the view of a run, then logs the command and what the run printed", async () => { + await confirmRun(); + const running = await keys("enter"); + + expect(request).toMatchObject({ + agent: "random", + scenarios: ["bank", "telecom"], + rows: new Map([["telecom", [0, 1]]]), + runs: 2, + model: undefined, + }); + expect(running).toContain("Running random on bank, telecom."); + expect(running).toContain("ctrl+c stop"); + + await act(async () => finish(true)); + const logged = await until("telecom: 2 answered"); + const help = await enter("/help"); + + expect(logged).toMatch(/evorca +Running the evaluation that this command repeats:/); + expect(logged).toMatch(/evorca +Running random on bank, telecom\./); + expect(help).toMatch(/\/help +List the commands/); + expect(onExit).not.toHaveBeenCalled(); + }); + + it("stops the run on ctrl+c, and leaves at once on a second ctrl+c", async () => { + await confirmRun(); + await keys("enter"); + const stopping = await keys("ctrl+c"); + + expect(request?.signal.aborted).toBe(true); + expect(stopping).toContain("Stopping. Press ctrl+c again to stop at once."); + expect(onExit).not.toHaveBeenCalled(); + + await keys("ctrl+c"); + + expect(onExit).toHaveBeenCalledWith( + expect.arrayContaining(["Running random on bank, telecom."]), + 130, + ); + }); + + it("logs the stop and goes back to the session once the run has stopped", async () => { + await confirmRun(); + await keys("enter", "ctrl+c"); + await act(async () => finish(false)); + const logged = await until("Stopped. Resume with"); + + expect(logged).toMatch(/evorca +Stopping\. Press ctrl\+c again to stop at once\./); + expect(onExit).not.toHaveBeenCalled(); + }); + + it("asks before /exit stops a run, then leaves once the run has stopped", async () => { + await confirmRun(); + await keys("enter"); + const asked = await enter("/exit"); + frames.push(asked); + + expect(asked).toContain("Type /exit again to stop it and leave evorca."); + expect(request?.signal.aborted).toBe(false); + + await enter("/exit"); + + expect(request?.signal.aborted).toBe(true); + expect(onExit).not.toHaveBeenCalled(); + + await act(async () => finish(false)); + + expect(onExit).toHaveBeenCalledWith( + expect.arrayContaining([ + "› /exit", + "Stopped. Resume with: pnpm evorca evaluate --resume=20261118T120000Z-random", + ]), + ); + }); + + it("goes back to the session when the picker closes", async () => { + await openWithData(); + await enter("/benchmark"); + await until("Scenarios"); + const frame = await keys("escape"); + + expect(frame).toMatch(/evorca +No run started\./); + }); +}); diff --git a/packages/cli/src/shell.tsx b/packages/cli/src/shell.tsx index a6b1bdc..3be04f7 100644 --- a/packages/cli/src/shell.tsx +++ b/packages/cli/src/shell.tsx @@ -6,6 +6,14 @@ * bottom. It applies the outcome that `commands.ts` returns for each line, and * draws only grays. * + * `/benchmark` opens the picker of `benchmark-view.tsx` in place of the + * transcript, then runs the pick in this process and shows it in the view of + * `evorca watch`, with the input line under it. What the run prints joins the + * transcript when the run ends. Ctrl+c stops the run as it stops `evorca + * evaluate`, and a second ctrl+c leaves at once. While a run goes, the input + * line takes only `/exit`, which asks first, then stops the run and leaves + * once it has stopped. Leaving during a run prints what it printed. + * * One clock drives the animations in `motion.ts`. It ticks while one of them * moves, sleeps until the next one moves, and stops when none will. * @@ -16,9 +24,10 @@ */ // @jsxRuntime automatic // @jsxImportSource @opentui/react +import { randomInt } from "node:crypto"; import { homedir } from "node:os"; import { sep } from "node:path"; -import { createCliRenderer, TextAttributes, type InputRenderable } from "@opentui/core"; +import { createCliRenderer, type InputRenderable } from "@opentui/core"; import { createRoot, useAnimationTick, @@ -27,8 +36,10 @@ import { useTerminalDimensions, } from "@opentui/react"; import { memo, useEffect, useRef, useState } from "react"; -import packageJson from "../package.json" with { type: "json" }; +import { commandOf, readCatalog, type Catalog, type Plan } from "./benchmark.js"; +import { BenchmarkPicker } from "./benchmark-view.js"; import { submit, type Entry } from "./commands.js"; +import { captureConsole, followRuns } from "./evaluate-view.js"; import { bootDuration, frameRate, @@ -39,37 +50,38 @@ import { type Led, type Phase, } from "./motion.js"; +import { Footer, gutter, Hints, Mark, palette } from "./parts.js"; +import type { Frame } from "./watch-folder.js"; +import { Watch } from "./watch-view.js"; +import type { Selection } from "./watch.js"; import { panel } from "./wordmark.js"; -/** The grays the shell draws with, each with equal red, green, and blue. */ -export const palette = { - background: "#0a0a0a", - border: "#333333", - muted: "#707070", - text: "#c8c8c8", - bright: "#f4f4f4", -} as const; - type Voice = { readonly speaker: string; readonly text: string }; -type View = - { readonly view: "home" } | { readonly view: "session"; readonly transcript: readonly Entry[] }; +type Screen = + | { readonly kind: "log" } + | { readonly kind: "picker"; readonly catalog: Catalog; readonly seed: number } + | { readonly kind: "run"; readonly run: Run }; + +type Run = { + readonly read: (selection: Selection) => Promise; + readonly stop: AbortController; + readonly entries: Entry[]; + exit: "no" | "asked" | "confirmed"; +}; type Draft = | { readonly kind: "empty"; readonly typingFrom: number; readonly firstExample: number } | { readonly kind: "text"; readonly nextExample: number }; -/** The space in columns between the edge of the screen and the content. */ -export const gutter = 2; const promptWidth = 75; const tagline = "autonomous incident investigation"; -const version = `evorca v${packageJson.version}`; const hints = [ ["enter", "send"], ["/help", "commands"], ["ctrl+c", "exit"], ] as const; -const separator = " "; +const rule = { border: ["bottom" as const], borderColor: palette.border }; const dot = "•"; const wordmarkWidth = (panel[0]?.length ?? 0) * 2 - 1; const voices: Readonly> = { @@ -87,7 +99,7 @@ const inks: Readonly> = { * Draws the shell full screen for the working directory. The shell runs until * the user exits it. */ -export async function openShell(): Promise { +export async function openShell(bench: Bench): Promise { const renderer = await createCliRenderer({ exitOnCtrlC: false, useMouse: false, @@ -97,28 +109,50 @@ export async function openShell(): Promise { new Date()} - onExit={() => renderer.destroy()} + bench={bench} + onExit={(lines = [], code) => { + renderer.destroy(); + void renderer.closed.then(() => { + for (const line of lines) process.stdout.write(`${line}\n`); + if (code !== undefined) process.exit(code); + }); + }} />, ); } +/** + * What `/benchmark` needs: the folders of the benchmark's data and of its + * runs, and `evaluate`, which runs a plan as `evorca evaluate` does. + */ +export type Bench = { + readonly dataDir: string; + readonly runsDir: string; + readonly evaluate: ( + request: Plan & { readonly signal: AbortSignal; readonly onRun: (runId: string) => void }, + ) => Promise; +}; + /** * The props of {@link Shell}. `directory` is the working directory as the user * should read it, and `wallTime` returns the local time to stamp each entry - * of the transcript with. + * of the transcript with. `onExit` closes the shell, then prints `lines` and + * ends the process with `code` if they are given. */ export type ShellProps = { readonly directory: string; readonly wallTime: () => Date; - readonly onExit: () => void; + readonly bench: Bench; + readonly onExit: (lines?: readonly string[], code?: number) => void; }; /** The whole screen. It calls `onExit` on `/exit` and on ctrl+c. */ -export function Shell({ directory, wallTime, onExit }: ShellProps) { +export function Shell({ directory, wallTime, bench, onExit }: ShellProps) { const renderer = useRenderer(); const { width } = useTerminalDimensions(); const [start] = useState(() => renderer.clock.now()); - const [view, setView] = useState({ view: "home" }); + const [transcript, setTranscript] = useState([]); + const [screen, setScreen] = useState({ kind: "log" }); const [draft, setDraft] = useState({ kind: "empty", typingFrom: start + bootDuration, @@ -128,11 +162,13 @@ export function Shell({ directory, wallTime, onExit }: ShellProps) { const input = useRef(null); const now = renderer.clock.now(); - const home = view.view === "home"; + const home = screen.kind === "log" && transcript.length === 0; const showWordmark = home && width >= wordmarkWidth + 2 * gutter; const mark = wordmarkFrame(now - start); const placeholder = - draft.kind === "empty" ? typewriter(now - draft.typingFrom, draft.firstExample) : null; + screen.kind === "log" && draft.kind === "empty" + ? typewriter(now - draft.typingFrom, draft.firstExample) + : null; const phases: Phase[] = []; if (showWordmark) phases.push(mark.phase); if (placeholder) phases.push(placeholder.phase); @@ -146,7 +182,12 @@ export function Shell({ directory, wallTime, onExit }: ShellProps) { }, [renderer, now, rest]); useKeyboard((key) => { - if (key.ctrl && key.name === "c") onExit(); + if (!key.ctrl || key.name !== "c") return; + if (screen.kind !== "run") return onExit(); + const { run } = screen; + if (run.stop.signal.aborted) return onExit(texts(run.entries), 130); + say(run, "Stopping. Press ctrl+c again to stop at once."); + run.stop.abort(); }); function onInput(value: string) { @@ -166,50 +207,134 @@ export function Shell({ directory, wallTime, onExit }: ShellProps) { function onSubmit(line: string) { if (input.current) input.current.value = ""; const outcome = submit(line, wallTime()); + if (screen.kind === "run") { + if (line.trim() !== "") whileRunning(screen.run, line.trim(), outcome.kind === "exit"); + return; + } if (outcome.kind === "exit") return onExit(); - if (outcome.entries.length === 0) return; - setView((view) => ({ - view: "session", - transcript: [...(view.view === "session" ? view.transcript : []), ...outcome.entries], - })); + setTranscript((transcript) => [...transcript, ...outcome.entries]); + if (outcome.kind === "benchmark") openPicker(); } + function whileRunning(run: Run, text: string, exit: boolean) { + run.entries.push({ role: "you", text, at: wallTime() }); + if (!exit) { + return say(run, "A run is going. /exit stops it and leaves, and ctrl+c stops it."); + } + if (run.exit === "no") { + run.exit = "asked"; + return say(run, "A run is going. Type /exit again to stop it and leave evorca."); + } + run.exit = "confirmed"; + run.stop.abort(); + say(run, "Stopping. evorca leaves once the run has stopped."); + } + + function say(run: Run, text: string) { + run.entries.push({ role: "evorca", text, at: wallTime() }); + setScreen({ kind: "run", run }); + } + + function openPicker() { + readCatalog(bench.dataDir, bench.runsDir).then( + (catalog) => setScreen({ kind: "picker", catalog, seed: randomInt(2 ** 32) }), + (error: unknown) => append(error instanceof Error ? error.message : String(error)), + ); + } + + function append(text: string) { + setTranscript((transcript) => [...transcript, { role: "evorca", text, at: wallTime() }]); + } + + async function runPlan(plan: Plan) { + append(`Running the evaluation that this command repeats:\n${commandOf(plan)}`); + const { read, onRun } = followRuns({ ...bench, scenarios: plan.scenarios }); + const run: Run = { read, stop: new AbortController(), entries: [], exit: "no" }; + setScreen({ kind: "run", run }); + const restore = captureConsole(({ text }) => { + run.entries.push({ role: "evorca", text, at: wallTime() }); + }); + try { + await bench.evaluate({ ...plan, signal: run.stop.signal, onRun }); + } catch (error: unknown) { + run.entries.push({ role: "evorca", text: String(error), at: wallTime() }); + } finally { + restore(); + } + if (run.exit === "confirmed") return onExit(texts(run.entries)); + setTranscript((transcript) => [...transcript, ...run.entries]); + setScreen({ kind: "log" }); + } + + const prompt = ( + + + {"› "} + + + + ); + + if (screen.kind === "run") { + const { run } = screen; + return ( + texts(run.entries)} + keys={draft.kind === "empty"} + > + + {prompt} + + + ); + } return ( {home ? : null} - {view.view === "session" ? ( - - ) : ( + {screen.kind === "picker" ? ( + { + append("No run started."); + setScreen({ kind: "log" }); + }} + onStart={(plan) => void runPlan(plan)} + /> + ) : home ? ( + ) : ( + )} - - - - {"› "} - - + {screen.kind === "picker" ? null : ( + + {prompt} + - - + )} {home ? : null}