From 02697a9de50b4e7c483b0a4cac44c32f575cf966 Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:25:56 +0200 Subject: [PATCH] feat(cli): run evaluate tasks in parallel and resume stopped runs evorca evaluate takes --jobs, --rows, and --resume=. It writes run.json before the first task and after each task, by renaming a staging file into place, so a stopped run keeps every finished task. A resumed run skips those tasks, removes containers that a killed run left, and refuses other agent, seed, or rows settings. Each agent's entry sets its container's memory, /tmp, CPUs, processes, and default jobs. The random agent keeps 1 GiB, 64 MiB, one CPU, 128 processes, and one task at a time. --- .changeset/cli-evaluate-resume.md | 5 + packages/cli/README.md | 52 +++- packages/cli/src/agents.test.ts | 25 +- packages/cli/src/agents.ts | 87 +++++-- packages/cli/src/main.test.ts | 24 ++ packages/cli/src/main.ts | 61 ++++- packages/cli/src/openrca.ts | 385 ++++++++++++++++++++---------- packages/cli/src/runs.test.ts | 151 ++++++++++++ packages/cli/src/runs.ts | 156 ++++++++++++ 9 files changed, 778 insertions(+), 168 deletions(-) create mode 100644 .changeset/cli-evaluate-resume.md create mode 100644 packages/cli/src/runs.test.ts create mode 100644 packages/cli/src/runs.ts diff --git a/.changeset/cli-evaluate-resume.md b/.changeset/cli-evaluate-resume.md new file mode 100644 index 0000000..56d0ab9 --- /dev/null +++ b/.changeset/cli-evaluate-resume.md @@ -0,0 +1,5 @@ +--- +"@evorca/cli": minor +--- + +`evorca evaluate` runs several tasks at once with `--jobs`, runs a subset of rows with `--rows`, and resumes a stopped run with `--resume=`. It rewrites `run.json` after every task, and each agent in the `agents` table sets its container's memory, `/tmp`, CPUs, processes, and default `jobs`. diff --git a/packages/cli/README.md b/packages/cli/README.md index fbed1c8..43aa9b2 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -8,13 +8,12 @@ The `evorca` command. From the repository root, `pnpm evorca` runs it without a - The slash commands `/help` and `/exit`. - `evorca fetch`, which downloads the answer keys and the telemetry of a benchmark's scenarios into `data/benchmarks///` at the repository root. - `evorca reproduce`, which downloads only the answer keys and checks that evorca reproduces the scores that a benchmark publishes. It keeps the scorer's checkout and environment in `.cache/` at the repository root. -- `evorca evaluate`, which runs an agent from `agents/` on a benchmark's scenarios, one Docker container per task, scores the answers, and keeps each run in `data/runs///` at the repository root. +- `evorca evaluate`, which runs an agent from `agents/` on a benchmark's scenarios, one Docker container per task and several tasks at a time, scores the answers, and keeps each run in `data/runs///` at the repository root. A run that stops can resume from its last finished task. - One benchmark, `openrca`, through [`@evorca/openrca`](../openrca/README.md). ## Non-goals - An agent in the shell. Every message in the shell gets the same reply until one is connected. -- Parallel tasks and resumed runs. `evorca evaluate` runs one task at a time, and a run that stops starts again from the first task. - Benchmark logic. Each benchmark package owns its scenarios, its files, how to download them, and how to score them. The command only parses the command line and prints results. - Configuration, themes, and mouse input. - Publishing to npm. The package is private, and `pnpm evorca` runs its TypeScript source through tsx. @@ -28,13 +27,19 @@ evorca evorca fetch --benchmark= [--scenario=...] evorca reproduce --benchmark= [--scenario=...] evorca evaluate --benchmark= --agent= [--scenario=...] + [--rows=,...] [--jobs=] +evorca evaluate --benchmark= --resume= [--scenario=...] + [--jobs=] ``` - Without arguments, `evorca` opens the [interactive shell](#use-the-shell). - `--benchmark` is required. The only benchmark is `openrca`. - `--scenario` names one scenario and can repeat. Without it, the command uses every scenario of the benchmark. The `openrca` scenarios are `bank`, `telecom`, `market-cloudbed-1`, and `market-cloudbed-2`. -- `--agent` is required by `evorca evaluate`. The only agent is `random`. -- `EVORCA_SEED`, a non-negative integer in the environment, sets the seed of `evorca evaluate`. Without it, the command picks a seed and prints it. +- `--agent` is required by `evorca evaluate`, unless it resumes a run. The only agent is `random`. +- `--rows` limits `evorca evaluate` to some rows of each scenario's `query.csv`, counted from 0, such as `--rows=0,7,21` for a pilot. Without it, the command runs every task. +- `--jobs` sets how many tasks `evorca evaluate` runs at once. Without it, the command uses the agent's `jobs`. +- `--resume` continues the run with that ID. The run keeps its agent, seed, and rows, and `--jobs` can change. With `--scenario`, the command resumes only those scenarios of the run. +- `EVORCA_SEED`, a non-negative integer in the environment, sets the seed of `evorca evaluate`. Without it, the command picks a seed and prints it, or uses the seed of the run it resumes. `evorca fetch` prints one line for each file that is already present, and two lines for each file that it downloads: one when the download starts and one when it is done. A line has the status, the path, and the size in bytes. A `present` or `fetched` line also has the SHA-256: @@ -48,7 +53,7 @@ fetched data/benchmarks/openrca/telecom/telemetry/2020_04_11/trace/trace_span.c Both commands exit with code 1 and list the valid choices if the command, the benchmark, or a scenario is unknown. They also exit with code 1, and print the reason, if a file on disk does not match its pinned hash, if the missing files need more than the free disk space, or if a download fails. The free-space message gives both sizes in GB. -`evorca evaluate` prints one line per task, then the outcomes and the accuracy of each scenario. It exits with code 1 and lists the agents if `--agent` is missing or unknown. Before it runs any task, it exits with code 1 if a scenario's data is missing, with the `evorca fetch` command that downloads it, or if Docker does not answer. +`evorca evaluate` prints one line per task, then the outcomes and the accuracy of each scenario. It exits with code 1 and lists the agents if `--agent` is missing or unknown. Before it runs any task, it exits with code 1 if a scenario's data is missing, with the `evorca fetch` command that downloads it, if a row in `--rows` does not exist, if Docker does not answer, or if the run to resume is missing or has other settings. It names each setting that differs. ## Use the shell @@ -115,6 +120,7 @@ Agent random, image evorca-agent-random:, sha256: Seed 7. Set EVORCA_SEED=7 to repeat this run. git node v26.13.1, docker 28.2.2, daemon 28.2.2 +Run 20261117T121537Z-random, 1 task at a time. telecom: 51 tasks, run folder data/runs/openrca/telecom/20261117T121537Z-random telecom 1/51 task_2 answered 0.22 s @@ -127,11 +133,25 @@ telecom: 51 answered total 51 tasks strict 5.88% partial 8.80% ``` -A run with the same seed writes the same predictions, byte for byte. Without `EVORCA_SEED`, the command picks a seed and prints it. One seed is one sample of the agent's accuracy, so compare agents over several seeds. +A run with the same seed writes the same predictions, byte for byte, whatever its `--jobs`. Without `EVORCA_SEED`, the command picks a seed and prints it. One seed is one sample of the agent's accuracy, so compare agents over several seeds. + +With `--jobs=4`, four containers run at once, and the lines come in the order that the tasks finish. If the agent's tasks together ask for more memory than Docker has, the command prints a warning before the first task. On Docker Desktop, set the memory under **Settings** > **Resources**. + +For a pilot, run a few rows with `--rows=0,7,21`. The command still writes a `predictions.csv` row for every task, because the evaluator matches predictions to tasks by position, and gives the other rows the empty prediction `{}`. The accuracy and `scores.json` cover only the rows of the run. + +### Resume a run + +To stop a run, press ctrl+c. The command removes the run's containers and prints the command that resumes the run: + +```sh +pnpm evorca evaluate --benchmark=openrca --resume=20261117T121537Z-random +``` + +The resumed run skips each task that has an outcome in `run.json`. It runs the other tasks again from the start, so it first deletes their folders. It removes containers that a killed run left behind. It refuses to resume if the agent's folder, its entry in the `agents` table, `EVORCA_SEED`, or `--rows` differ from the run's. ### Read a run -Each run of a scenario lands in `data/runs////`. The run ID is the start time in UTC and the agent's name, such as `20261117T121537Z-random`: +Each run of a scenario lands in `data/runs////`. The run ID is the start time in UTC and the agent's name, such as `20261117T121537Z-random`. A run of several scenarios uses one run ID for all of them: ```text / @@ -144,16 +164,20 @@ Each run of a scenario lands in `data/runs////`. Th log.txt the container's stdout and stderr ``` -`run.json` holds: +The command writes `run.json` before the first task and again after each task. It writes a new file and renames it into place, so `run.json` is always whole. `run.json` holds: - `runId`, `benchmark`, and `scenario`. -- `agent`: the `name`, `seesCandidates`, `needsTelemetry`, `timeoutSeconds`, the `image` tag, and the `imageId`. +- `agent`: the `name`, `seesCandidates`, `needsTelemetry`, `timeoutSeconds`, `resources`, the `image` tag, and the `imageId`. - `seed`. +- `rows`: the rows of the run, or `null` for every row. +- `jobs`: how many tasks ran at once. - `git`: the `commit` and whether the checkout was `dirty`, with uncommitted changes. - `versions`: Node, and the Docker client and daemon. - `pinnedInputs`: the OpenRCA commit, the Drive archives, and the SHA-256 of each answer key. -- `startedAt` and `endedAt`, in UTC. -- `tasks`: the `row`, `taskIndex`, `outcome`, `exitCode`, and `seconds` of each task. +- `startedAt`, in UTC. +- `resumes`: the `startedAt`, `git`, and `versions` of each later session of the run. +- `endedAt`, in UTC, or `null` while a task has no outcome. +- `tasks`: the `row`, `taskIndex`, `outcome`, `exitCode`, and `seconds` of each task, in row order. ## Run an agent in a container @@ -164,6 +188,8 @@ Each agent has a folder, `agents//` at the repository root, with a `Docker | `seesCandidates` | `/task.json` lists the candidate components and reasons of the scenario. | | `needsTelemetry` | The container gets the scenario's telemetry at `/data`. | | `timeoutSeconds` | `evorca evaluate` kills the container of a task that runs longer than this time. | +| `resources` | The container's `memoryMiB`, `tmpMiB` for `/tmp`, `cpus`, and `pids`. | +| `jobs` | How many tasks run at once without `--jobs`. | RCA-Agent sees the candidates, so an agent with `seesCandidates` answers under the same conditions. The [`@evorca/openrca` README](../openrca/README.md#show-an-agent-the-candidates) explains the choice. @@ -174,10 +200,10 @@ The container of a task sees: - `/task.json`, read-only, with the `scenario`, the `row`, the `taskIndex`, the `instruction`, and, if the agent sees candidates, the `candidates` with their `components` and `reasons`. It never holds the scoring points. - `/out`, writable. The agent writes its answer to `/out/prediction.json` as OpenRCA prediction JSON, such as `{"1": {"root cause occurrence datetime": "2020-04-11 00:05:00", "root cause component": "docker_003", "root cause reason": "CPU fault"}}`. - `/data`, read-only, only if the agent needs telemetry. It is `data/benchmarks/openrca//telemetry/`. The container never sees `query.csv` or `record.csv`, which hold the answers. -- `/tmp`, a 64 MB folder in memory. +- `/tmp`, a folder in memory of `tmpMiB`. Its files count toward the container's memory. - `EVORCA_SEED`, the seed of the run. -The container has no network and a read-only root file system. It runs as user `65534:65534` with no Linux capabilities and `no-new-privileges`, and it gets 1 GB of memory, one CPU, and at most 128 processes. +The container has no network and a read-only root file system. It runs as user `65534:65534` with no Linux capabilities and `no-new-privileges`, and it gets the memory, CPUs, and processes of the agent's `resources`. The `random` agent gets 1024 MiB of memory, a 64 MiB `/tmp`, one CPU, and at most 128 processes, and it runs one task at a time. Each container carries the label `evorca.run=`. Each task ends with one outcome. Only `answered` scores the agent's answer. Every other outcome scores the empty prediction `{}`, which scores 0. diff --git a/packages/cli/src/agents.test.ts b/packages/cli/src/agents.test.ts index 3c331ff..8f33b64 100644 --- a/packages/cli/src/agents.test.ts +++ b/packages/cli/src/agents.test.ts @@ -1,12 +1,13 @@ import { describe, expect, it } from "vitest"; -import { dockerRunArgs, outcomeOf, type Exit, type TaskContainer } from "./agents.js"; +import { agents, dockerRunArgs, outcomeOf, type Exit, type TaskContainer } from "./agents.js"; const dataDir = "/repo/data/benchmarks/openrca"; const taskDir = "/repo/data/runs/openrca/telecom/20261117T120000Z-random/tasks/007"; const container: TaskContainer = { + runId: "20261117T120000Z-random", name: "evorca-20261117T120000Z-random-telecom-7", image: "sha256:0123", - agent: { seesCandidates: true, needsTelemetry: false, timeoutSeconds: 60 }, + agent: agents.random, taskDir, dataDir, scenario: "telecom", @@ -27,9 +28,29 @@ describe("dockerRunArgs", () => { expect(valuesOf(args, "--security-opt")).toEqual(["no-new-privileges"]); expect(valuesOf(args, "--user")).toEqual(["65534:65534"]); expect(valuesOf(args, "--env")).toEqual(["EVORCA_SEED=42"]); + expect(valuesOf(args, "--label")).toEqual(["evorca.run=20261117T120000Z-random"]); expect(args.at(-1)).toBe("sha256:0123"); }); + it("gives the random agent 1 GiB of memory, a 64 MiB /tmp, one CPU, and 128 processes", () => { + const args = dockerRunArgs(container); + + expect(valuesOf(args, "--memory")).toEqual(["1024m"]); + expect(valuesOf(args, "--tmpfs")).toEqual(["/tmp:rw,noexec,nosuid,size=64m"]); + expect(valuesOf(args, "--cpus")).toEqual(["1"]); + expect(valuesOf(args, "--pids-limit")).toEqual(["128"]); + }); + + it("applies the limits of the agent's resources", () => { + const resources = { memoryMiB: 3072, tmpMiB: 1536, cpus: 2, pids: 256 }; + const args = dockerRunArgs({ ...container, agent: { ...container.agent, resources } }); + + expect(valuesOf(args, "--memory")).toEqual(["3072m"]); + expect(valuesOf(args, "--tmpfs")).toEqual(["/tmp:rw,noexec,nosuid,size=1536m"]); + expect(valuesOf(args, "--cpus")).toEqual(["2"]); + expect(valuesOf(args, "--pids-limit")).toEqual(["256"]); + }); + it("mounts the task read-only and the out folder writable, without telemetry", () => { expect(valuesOf(dockerRunArgs(container), "--mount")).toEqual([ `type=bind,src=${taskDir}/task.json,dst=/task.json,readonly`, diff --git a/packages/cli/src/agents.ts b/packages/cli/src/agents.ts index 968fffc..a33456a 100644 --- a/packages/cli/src/agents.ts +++ b/packages/cli/src/agents.ts @@ -19,20 +19,45 @@ import { parsePrediction, type OpenRcaScenario } from "@evorca/openrca"; const run = promisify(execFile); const containerUser = "65534:65534"; +/** The Docker label that holds the run ID on every container that a run starts. */ +export const runLabel = "evorca.run"; + +/** + * The limits of one task's container: its memory and its `/tmp` folder in MiB, + * its CPUs, and its processes. `/tmp` lives in memory and counts toward + * `memoryMiB`. + */ +export type Resources = { + readonly memoryMiB: number; + readonly tmpMiB: number; + readonly cpus: number; + readonly pids: number; +}; + /** * How `evorca evaluate` runs an agent. `seesCandidates` adds the scenario's * candidate components and reasons to `/task.json`. `needsTelemetry` mounts the - * scenario's telemetry at `/data`. `timeoutSeconds` bounds each task. + * scenario's telemetry at `/data`. `timeoutSeconds` bounds each task, and + * `resources` limits its container. `jobs` is how many tasks run at once + * unless `--jobs` says otherwise. */ export type Agent = { readonly seesCandidates: boolean; readonly needsTelemetry: boolean; readonly timeoutSeconds: number; + readonly resources: Resources; + readonly jobs: number; }; /** The agents by name. Each agent's image builds from `agents//` at the repository root. */ export const agents = { - random: { seesCandidates: true, needsTelemetry: false, timeoutSeconds: 60 }, + random: { + seesCandidates: true, + needsTelemetry: false, + timeoutSeconds: 60, + resources: { memoryMiB: 1024, tmpMiB: 64, cpus: 1, pids: 128 }, + jobs: 1, + }, } satisfies Record; /** The name of an agent, such as `random`. */ @@ -88,13 +113,26 @@ export async function dockerVersions(): Promise< } /** - * One task's container. `image` is an image ID, and `taskDir` holds the task's - * `task.json`. `runContainer` creates the task's `out/` folder in `taskDir`. + * Returns the memory of the Docker daemon's machine in MiB, which on Docker + * Desktop is the memory of its virtual machine. + * + * @throws Error if the daemon does not answer. + */ +export async function dockerMemoryMiB(): Promise { + const info = await run("docker", ["info", "--format", "{{.MemTotal}}"]); + return Math.floor(Number(info.stdout.trim()) / 2 ** 20); +} + +/** + * One task's container. `runId` labels it, `image` is an image ID, and + * `taskDir` holds the task's `task.json`. `runContainer` creates the task's + * `out/` folder in `taskDir`. */ export type TaskContainer = { + readonly runId: string; readonly name: string; readonly image: string; - readonly agent: Agent; + readonly agent: Pick; readonly taskDir: string; readonly dataDir: string; readonly scenario: OpenRcaScenario; @@ -104,13 +142,15 @@ export type TaskContainer = { /** * Returns the arguments of `docker run` for one task. The container runs as an * unprivileged user with no network, no capabilities, a read-only root file - * system, and limits on memory, CPU, and processes. It gets `task.json` at - * `/task.json`, read-only, and `out/` at `/out`, writable. If the agent needs - * telemetry, it gets `//telemetry/` at `/data`, read-only. - * The seed is in `EVORCA_SEED`. + * system, and the agent's limits on memory, `/tmp`, CPU, and processes. It gets + * `task.json` at `/task.json`, read-only, and `out/` at `/out`, writable. If + * the agent needs telemetry, it gets `//telemetry/` at + * `/data`, read-only. The seed is in `EVORCA_SEED`, and the `evorca.run` label + * holds the run ID. */ export function dockerRunArgs(container: TaskContainer): string[] { - const { name, image, agent, taskDir, dataDir, scenario, seed } = container; + const { runId, name, image, agent, taskDir, dataDir, scenario, seed } = container; + const { memoryMiB, tmpMiB, cpus, pids } = agent.resources; const telemetry = agent.needsTelemetry ? ["--mount", `type=bind,src=${join(dataDir, scenario, "telemetry")},dst=/data,readonly`] : []; @@ -120,11 +160,13 @@ export function dockerRunArgs(container: TaskContainer): string[] { "--init", "--name", name, + "--label", + `${runLabel}=${runId}`, "--network", "none", "--read-only", "--tmpfs", - "/tmp:rw,noexec,nosuid,size=64m", + `/tmp:rw,noexec,nosuid,size=${tmpMiB}m`, "--cap-drop", "ALL", "--security-opt", @@ -132,11 +174,11 @@ export function dockerRunArgs(container: TaskContainer): string[] { "--user", containerUser, "--memory", - "1g", + `${memoryMiB}m`, "--cpus", - "1", + String(cpus), "--pids-limit", - "128", + String(pids), "--mount", `type=bind,src=${join(taskDir, "task.json")},dst=/task.json,readonly`, "--mount", @@ -169,7 +211,11 @@ export async function runContainer(container: TaskContainer, logPath: string): P // The container's user does not own the folder on the host. await chmod(outDir, 0o777); await using log = await open(logPath, "w"); - const child = spawn("docker", dockerRunArgs(container), { stdio: ["ignore", log.fd, log.fd] }); + // Detached, so a ctrl+c in the terminal reaches evorca alone, which then removes the containers. + const child = spawn("docker", dockerRunArgs(container), { + stdio: ["ignore", log.fd, log.fd], + detached: true, + }); let timedOut = false; const timer = setTimeout(() => { timedOut = true; @@ -190,6 +236,17 @@ export async function runContainer(container: TaskContainer, logPath: string): P export type Outcome = "answered" | "no-prediction" | "invalid-prediction" | "nonzero-exit" | "timeout"; +/** + * Removes every container that carries the run's label, running or not. A run + * calls it when it stops, and a resumed run calls it before its first task. + */ +export async function removeRunContainers(runId: string): Promise { + const filter = `label=${runLabel}=${runId}`; + const listed = await run("docker", ["ps", "--all", "--quiet", "--filter", filter]); + const ids = listed.stdout.split("\n").filter((id) => id !== ""); + if (ids.length > 0) await run("docker", ["rm", "--force", ...ids]); +} + /** * Returns a task's outcome and the diagnosis to score, from how its container * ended and the text of `/out/prediction.json`, undefined if the file is missing. diff --git a/packages/cli/src/main.test.ts b/packages/cli/src/main.test.ts index 459fd9e..d93f823 100644 --- a/packages/cli/src/main.test.ts +++ b/packages/cli/src/main.test.ts @@ -53,6 +53,30 @@ describe("evorca evaluate", () => { expect(code).toBe(1); expect(stderr).toContain("random"); }); + + it.each([ + ["--jobs=0", "--jobs must be a positive integer"], + ["--jobs=two", "--jobs must be a positive integer"], + ["--rows=0,,7", "--rows must list row numbers"], + ["--rows=-1", "--rows must list row numbers"], + ])("exits with code 1 for %s", async (flag, message) => { + const { code, stderr } = await evorca( + "evaluate", + "--benchmark=openrca", + "--agent=random", + flag, + ); + + expect(code).toBe(1); + expect(stderr).toContain(message); + }); + + it("exits with code 1 for a run ID that is not one", async () => { + const { code, stderr } = await evorca("evaluate", "--benchmark=openrca", "--resume=../x"); + + expect(code).toBe(1); + expect(stderr).toContain("--resume takes a run ID"); + }); }); describe.each(["fetch", "reproduce", "evaluate"])("evorca %s", (command) => { diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 8010105..0021922 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -15,7 +15,6 @@ * * @module */ -import { randomInt } from "node:crypto"; import { relative } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; @@ -39,7 +38,7 @@ type Benchmark = (typeof benchmarks)[keyof typeof benchmarks]; type Invocation = { readonly benchmark: Benchmark; readonly scenarios: readonly (keyof Benchmark["scenarios"])[]; - readonly agent: string | undefined; + readonly flags: Flags; readonly dataDir: string; readonly cacheDir: string; readonly runsDir: string; @@ -54,8 +53,21 @@ const commands = { await printProgress(benchmark.fetch({ scenarios, dataDir, files: benchmark.answerKeys })); if (!(await benchmark.reproduce({ scenarios, dataDir, cacheDir }))) process.exitCode = 1; }, - async evaluate({ benchmark, scenarios, agent, ...dirs }: Invocation) { - const options = { ...dirs, scenarios, agent: choose(agents, agent, "agent"), seed: readSeed() }; + async evaluate({ benchmark, scenarios, flags, ...dirs }: Invocation) { + const resume = flags.resume === undefined ? undefined : parseRunId(flags.resume); + const agent = + flags.agent === undefined && resume !== undefined + ? undefined + : choose(agents, flags.agent, "agent"); + const options = { + ...dirs, + scenarios, + agent, + resume, + seed: readSeed(), + rows: flags.rows === undefined ? undefined : parseRows(flags.rows), + jobs: flags.jobs === undefined ? undefined : parseCount(flags.jobs, "--jobs"), + }; if (!(await benchmark.evaluate(options))) process.exitCode = 1; }, }; @@ -63,7 +75,10 @@ const commands = { const usage = [ "Usage: evorca", ` evorca <${Object.keys(commands).join("|")}> --benchmark= [--scenario=...]`, - " [--agent=]", + " evorca evaluate --benchmark= --agent= [--scenario=...]", + " [--rows=,...] [--jobs=]", + " evorca evaluate --benchmark= --resume= [--scenario=...]", + " [--jobs=]", ].join("\n"); const { positionals, values } = parseCommandLine(); @@ -89,7 +104,7 @@ if (positionals.length === 0 && Object.keys(values).length === 0) { await commands[command]({ benchmark, scenarios, - agent: values.agent, + flags: values, dataDir: fileURLToPath(new URL(`data/benchmarks/${name}/`, root)), cacheDir: fileURLToPath(new URL(".cache/", root)), runsDir: fileURLToPath(new URL(`data/runs/${name}/`, root)), @@ -108,6 +123,13 @@ async function printProgress(progress: AsyncIterable) { } } +type Flags = { + readonly agent?: string | undefined; + readonly rows?: string | undefined; + readonly jobs?: string | undefined; + readonly resume?: string | undefined; +}; + function parseCommandLine() { try { return parseArgs({ @@ -116,6 +138,9 @@ function parseCommandLine() { benchmark: { type: "string" }, scenario: { type: "string", multiple: true }, agent: { type: "string" }, + rows: { type: "string" }, + jobs: { type: "string" }, + resume: { type: "string" }, }, }); } catch (error: unknown) { @@ -123,15 +148,35 @@ function parseCommandLine() { } } -function readSeed(): number { +function readSeed(): number | undefined { const seed = process.env.EVORCA_SEED; - if (seed === undefined) return randomInt(2 ** 32); + if (seed === undefined) return undefined; if (!/^\d+$/.test(seed) || !Number.isSafeInteger(Number(seed))) { fail(`EVORCA_SEED must be a non-negative integer, not ${JSON.stringify(seed)}.`); } return Number(seed); } +function parseRows(text: string): number[] { + if (!/^\d+(,\d+)*$/.test(text)) { + fail(`--rows must list row numbers, such as --rows=0,7,21, not ${JSON.stringify(text)}.`); + } + return [...new Set(text.split(",").map(Number))].sort((a, b) => a - b); +} + +function parseCount(text: string, flag: string): number { + if (!/^[1-9]\d*$/.test(text)) + fail(`${flag} must be a positive integer, not ${JSON.stringify(text)}.`); + return Number(text); +} + +function parseRunId(text: string): string { + if (!/^\d{8}T\d{6}Z-[a-z0-9-]+$/.test(text)) { + fail(`--resume takes a run ID, such as 20261117T121537Z-random, not ${JSON.stringify(text)}.`); + } + return text; +} + function choose(table: T, key: string | undefined, option: string) { if (isKeyOf(table, key)) return key; const problem = key === undefined ? `Missing --${option}` : `Unknown ${option}: ${key}`; diff --git a/packages/cli/src/openrca.ts b/packages/cli/src/openrca.ts index 6d2e4c6..9a5a592 100644 --- a/packages/cli/src/openrca.ts +++ b/packages/cli/src/openrca.ts @@ -7,17 +7,19 @@ * @module */ import { execFile } from "node:child_process"; +import { randomInt } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join, relative } from "node:path"; import { promisify } from "node:util"; import type { Diagnosis } from "@evorca/core"; import { openRcaCandidates, + parsePrediction, readTasks, writePredictions, type OpenRcaScenario, - type TaskIndex, + type OpenRcaTask, } from "@evorca/openrca"; import { evaluatorVersions, @@ -32,14 +34,25 @@ import { } from "@evorca/openrca/scoring"; import { agents, + dockerMemoryMiB, dockerVersions, ensureImage, outcomeOf, + removeRunContainers, runContainer, - type Agent, type AgentName, - type Outcome, } from "./agents.js"; +import { + conflicts, + readRuns, + runPool, + tasksToRun, + writeJson, + type AgentRecord, + type GitState, + type RunRecord, + type TaskRecord, +} from "./runs.js"; const run = promisify(execFile); @@ -94,70 +107,68 @@ export async function reproduceOpenRca({ return mismatches.length === 0; } -/** What `run.json` records about the run of one agent on one scenario. */ -type RunRecord = { - readonly runId: string; - readonly benchmark: "openrca"; - readonly scenario: OpenRcaScenario; - readonly agent: Agent & { - readonly name: AgentName; - readonly image: string; - readonly imageId: string; - }; - readonly seed: number; - readonly git: { readonly commit: string; readonly dirty: boolean }; - readonly pinnedInputs: typeof pinnedInputs; - readonly versions: { - readonly node: string; - readonly docker: { readonly client: string; readonly server: string }; - }; - readonly startedAt: string; - readonly endedAt: string; - readonly tasks: readonly TaskRecord[]; -}; - /** - * What `run.json` records about one task. `exitCode` is null after a timeout or - * when a signal ended the `docker run` client. + * What `evorca evaluate` asks for. `resume` names a run to continue. A setting + * that is undefined comes from that run, or else from its default: the agent's + * `jobs`, every row, and a random seed. */ -type TaskRecord = { - readonly row: number; - readonly taskIndex: TaskIndex; - readonly outcome: Outcome; - readonly exitCode: number | null; - readonly seconds: number; +export type EvaluateRequest = { + readonly scenarios: readonly OpenRcaScenario[]; + readonly agent: AgentName | undefined; + readonly seed: number | undefined; + readonly rows: readonly number[] | undefined; + readonly jobs: number | undefined; + readonly resume: string | undefined; +}; + +type Dirs = { + readonly dataDir: string; + readonly cacheDir: string; + readonly runsDir: string; + readonly agentsDir: string; }; -type Setup = Pick; +/** One scenario of a run: its tasks, the rows that the run covers, and its folder. */ +type ScenarioPlan = { + readonly scenario: OpenRcaScenario; + readonly tasks: readonly OpenRcaTask[]; + readonly selected: readonly OpenRcaTask[]; + readonly runDir: string; + readonly record: RunRecord; +}; /** - * Runs `agent` on every task of `scenarios`, one task at a time, each in its - * own container. It scores the answers with the official evaluator and prints - * a line per task and the accuracy per scenario. Each scenario's run lands in - * `///`. Before any task, it prints the problem and - * returns false if a scenario lacks the data that the agent reads, or if Docker - * does not answer. Otherwise it returns true. + * Runs an agent on the tasks of `scenarios`, `jobs` tasks at a time, each in + * its own container, or continues the run that `resume` names. It scores the + * answers with the official evaluator and prints a line per task and the + * accuracy per scenario. Each scenario's run lands in + * `///`, and its `run.json` is rewritten after every + * task. A resumed run skips the tasks that have an outcome. Before any task, it + * prints the problem and returns false if the run to resume is missing or has + * other settings, if a row does not exist, if a scenario lacks the data that the + * agent reads, or if Docker does not answer. Otherwise it returns true. + * + * On SIGINT or SIGTERM, it removes the run's containers and exits, and the run + * can resume. * * @throws Error if git, an image build, a container start, or the evaluator fails. */ -export async function evaluateOpenRca({ - scenarios, - agent: name, - seed, - dataDir, - cacheDir, - runsDir, - agentsDir, -}: { - readonly scenarios: readonly OpenRcaScenario[]; - readonly agent: AgentName; - readonly seed: number; - readonly dataDir: string; - readonly cacheDir: string; - readonly runsDir: string; - readonly agentsDir: string; -}): Promise { - const agent: Agent = agents[name]; +export async function evaluateOpenRca(request: EvaluateRequest & Dirs): Promise { + const { dataDir, cacheDir, runsDir, agentsDir } = request; + const resumed = + request.resume === undefined + ? undefined + : await readRuns(runsDir, request.resume, request.scenarios); + if (resumed?.size === 0) { + console.error(`No run ${request.resume} in ${relative(process.cwd(), runsDir)}.`); + return false; + } + const recorded = resumed?.values().next().value; + const name = recorded?.agent.name ?? request.agent; + if (name === undefined) throw new Error("evaluateOpenRca needs an agent or a run to resume."); + const scenarios = resumed === undefined ? request.scenarios : [...resumed.keys()]; + const { jobs: defaultJobs, ...agent } = agents[name]; + const needs = (scenario: OpenRcaScenario) => [ join(dataDir, scenario, "query.csv"), ...(agent.needsTelemetry ? [join(dataDir, scenario, "telemetry")] : []), @@ -171,6 +182,17 @@ export async function evaluateOpenRca({ console.error(` pnpm evorca fetch --benchmark=openrca ${flags}`); return false; } + const tasks = new Map(); + for (const scenario of scenarios) tasks.set(scenario, await readTasks(scenario, dataDir)); + const rows = request.rows ?? recorded?.rows ?? null; + for (const [scenario, scenarioTasks] of tasks) { + const unknown = rows?.filter((row) => row >= scenarioTasks.length) ?? []; + if (unknown.length > 0) { + const range = `rows 0 to ${scenarioTasks.length - 1}`; + console.error(`${scenario} has no row ${unknown.join(", ")}. Its tasks are ${range}.`); + return false; + } + } const docker = await dockerVersions(); if (docker === undefined) { console.error("Docker is not running. Start Docker Desktop and try again."); @@ -178,104 +200,207 @@ export async function evaluateOpenRca({ } const image = await ensureImage(name, agentsDir); + const agentRecord: AgentRecord = { name, ...agent, image: image.tag, imageId: image.id }; + if (recorded !== undefined) { + const problems = conflicts(recorded, { agent: agentRecord, seed: request.seed, rows }); + if (problems.length > 0) { + console.error(`Run ${recorded.runId} cannot resume with other settings:`); + for (const problem of problems) console.error(` ${problem}`); + return false; + } + } + const seed = recorded?.seed ?? request.seed ?? randomInt(2 ** 32); + const jobs = request.jobs ?? defaultJobs; + const runId = recorded?.runId ?? `${new Date().toISOString().replace(/[-:]|\.\d+/g, "")}-${name}`; const git = await gitState(agentsDir); - const setup: Setup = { - runId: `${new Date().toISOString().replace(/[-:]|\.\d+/g, "")}-${name}`, - agent: { name, ...agent, image: image.tag, imageId: image.id }, - seed, - git, - versions: { node: process.version, docker }, - }; + const versions = { node: process.version, docker }; console.log(`Agent ${name}, image ${image.tag}, ${image.id}`); console.log(`Seed ${seed}. Set EVORCA_SEED=${seed} to repeat this run.`); console.log(`git ${git.commit}${git.dirty ? " with uncommitted changes" : ""}`); console.log(`node ${process.version}, docker ${docker.client}, daemon ${docker.server}`); - const scores: ScoredRow[] = []; - for (const scenario of scenarios) { - scores.push(...(await evaluateScenario(scenario, setup, { dataDir, cacheDir, runsDir }))); + const pace = `${jobs} ${jobs === 1 ? "task" : "tasks"} at a time`; + console.log(`${recorded === undefined ? "Run" : "Resuming run"} ${runId}, ${pace}.`); + const memoryMiB = await dockerMemoryMiB(); + if (jobs * agent.resources.memoryMiB > memoryMiB) { + const asked = `${jobs} tasks at ${agent.resources.memoryMiB} MiB`; + console.log(`Warning: ${asked} need more than Docker's ${memoryMiB} MiB of memory.`); } - if (scenarios.length > 1) { - console.log("\nAll scenarios:"); - printSummary(summarize(scores)); + + if (recorded !== undefined) await removeRunContainers(runId); + const startedAt = new Date().toISOString(); + const plans: ScenarioPlan[] = []; + for (const [scenario, scenarioTasks] of tasks) { + const runDir = join(runsDir, scenario, runId); + const previous = resumed?.get(scenario); + const record: RunRecord = + previous === undefined + ? { + runId, + benchmark: "openrca", + scenario, + agent: agentRecord, + seed, + rows, + jobs, + git, + pinnedInputs, + versions, + startedAt, + resumes: [], + endedAt: null, + tasks: [], + } + : { + ...previous, + jobs, + resumes: [...previous.resumes, { startedAt, git, versions }], + endedAt: null, + }; + const selected = + rows === null ? scenarioTasks : scenarioTasks.filter((task) => rows.includes(task.row)); + for (const task of tasksToRun(selected, record.tasks)) { + await rm(taskDirOf(runDir, task.row), { recursive: true, force: true }); + } + await mkdir(runDir, { recursive: true }); + await writeJson(join(runDir, "run.json"), record); + plans.push({ scenario, tasks: scenarioTasks, selected, runDir, record }); + } + + const stop = new AbortController(); + const onSignal = (signal: NodeJS.Signals) => { + stop.abort(); + console.error( + `\nStopped. Resume with: pnpm evorca evaluate --benchmark=openrca --resume=${runId}`, + ); + void removeRunContainers(runId).finally(() => process.exit(signal === "SIGINT" ? 130 : 143)); + }; + process.once("SIGINT", onSignal).once("SIGTERM", onSignal); + try { + const scores: ScoredRow[] = []; + for (const plan of plans) { + const scenarioScores = await evaluateScenario(plan, { jobs, dataDir, cacheDir }, stop.signal); + if (scenarioScores === undefined) return false; + scores.push(...scenarioScores); + } + if (plans.length > 1) { + console.log("\nAll scenarios:"); + printSummary(summarize(scores)); + } + return true; + } finally { + process.off("SIGINT", onSignal).off("SIGTERM", onSignal); } - return true; } +/** + * Runs the tasks of one scenario that have no outcome, rewrites `run.json` + * after each, then scores the run. It returns the scores of the run's rows, or + * undefined if `signal` aborts. + */ async function evaluateScenario( - scenario: OpenRcaScenario, - { runId, ...setup }: Setup, + { scenario, tasks, selected, runDir, record }: ScenarioPlan, { + jobs, dataDir, cacheDir, - runsDir, - }: { readonly dataDir: string; readonly cacheDir: string; readonly runsDir: string }, -): Promise { - const tasks = await readTasks(scenario, dataDir); - const runDir = join(runsDir, scenario, runId); - console.log( - `\n${scenario}: ${tasks.length} tasks, run folder ${relative(process.cwd(), runDir)}`, + }: { readonly jobs: number; readonly dataDir: string; readonly cacheDir: string }, + signal: AbortSignal, +): Promise { + const records = new Map(record.tasks.map((task) => [task.row, task])); + const pending = tasksToRun(selected, record.tasks); + const done = records.size > 0 ? `, ${records.size} done` : ""; + const folder = relative(process.cwd(), runDir); + console.log(`\n${scenario}: ${selected.length} tasks${done}, run folder ${folder}`); + let saving = Promise.resolve(); + const save = (endedAt: string | null) => { + const tasks = [...records.values()].sort((a, b) => a.row - b.row); + const snapshot: RunRecord = { ...record, endedAt, tasks }; + saving = saving.then(() => writeJson(join(runDir, "run.json"), snapshot)); + return saving; + }; + + await runPool( + pending, + jobs, + async (task) => { + const taskRecord = await runTask(task, record, { runDir, dataDir }).catch( + (error: unknown) => { + if (signal.aborted) return undefined; + throw error; + }, + ); + if (taskRecord === undefined || signal.aborted) return; + records.set(task.row, taskRecord); + const { taskIndex, outcome, seconds } = taskRecord; + const progress = `${records.size}/${selected.length} ${taskIndex}`; + console.log(`${scenario} ${progress} ${outcome} ${seconds.toFixed(2)} s`); + await save(null); + }, + signal, ); - const startedAt = new Date().toISOString(); - const records: TaskRecord[] = []; + if (signal.aborted) return undefined; + await save(new Date().toISOString()); + const diagnoses = new Map(); - for (const task of tasks) { - const taskDir = join(runDir, "tasks", String(task.row).padStart(3, "0")); - await mkdir(taskDir, { recursive: true }); - const candidates = setup.agent.seesCandidates - ? { candidates: openRcaCandidates[scenario] } - : {}; - await writeFile(join(taskDir, "task.json"), json({ ...task, ...candidates })); - const started = performance.now(); - const exit = await runContainer( - { - name: `evorca-${runId}-${scenario}-${task.row}`, - image: setup.agent.imageId, - agent: setup.agent, - taskDir, - dataDir, - scenario, - seed: setup.seed, - }, - join(taskDir, "log.txt"), - ); - const seconds = Math.round(performance.now() - started) / 1000; - const prediction = await readFile(join(taskDir, "out", "prediction.json"), "utf8").catch( - () => undefined, - ); - const { outcome, diagnosis } = outcomeOf(exit, prediction); - diagnoses.set(task.row, diagnosis); - const exitCode = exit.kind === "exited" ? exit.code : null; - records.push({ row: task.row, taskIndex: task.taskIndex, outcome, exitCode, seconds }); - const progress = `${records.length}/${tasks.length} ${task.taskIndex}`; - console.log(`${scenario} ${progress} ${outcome} ${seconds.toFixed(2)} s`); + for (const { row, outcome } of records.values()) { + if (outcome !== "answered") continue; + const text = await readFile(join(taskDirOf(runDir, row), "out", "prediction.json"), "utf8"); + diagnoses.set(row, parsePrediction(text) ?? []); } - const endedAt = new Date().toISOString(); - const predictionsPath = join(runDir, "predictions.csv"); await writeFile(predictionsPath, writePredictions(tasks, diagnoses)); - const record: RunRecord = { - runId, - benchmark: "openrca", - scenario, - ...setup, - pinnedInputs, - startedAt, - endedAt, - tasks: records, - }; - await writeFile(join(runDir, "run.json"), json(record)); - const scores = await scoreWithOfficialEvaluator({ predictionsPath, scenario, dataDir, cacheDir }); + const rows = new Set(selected.map((task) => task.row)); + const scores = ( + await scoreWithOfficialEvaluator({ predictionsPath, scenario, dataDir, cacheDir }) + ).filter((score) => rows.has(score.row)); const summary = summarize(scores); await writeFile(join(runDir, "scores.json"), json({ summary, scores })); - const outcomes = Map.groupBy(records, (task) => task.outcome); + const outcomes = Map.groupBy(records.values(), (task) => task.outcome); const counts = [...outcomes].map(([outcome, tasks]) => `${tasks.length} ${outcome}`); console.log(`\n${scenario}: ${counts.join(", ")}`); printSummary(summary); return scores; } -async function gitState(dir: string): Promise { +/** Runs one task in its own container and returns its record. */ +async function runTask( + task: OpenRcaTask, + { runId, scenario, agent, seed }: RunRecord, + { runDir, dataDir }: { readonly runDir: string; readonly dataDir: string }, +): Promise { + const taskDir = taskDirOf(runDir, task.row); + await mkdir(taskDir, { recursive: true }); + const candidates = agent.seesCandidates ? { candidates: openRcaCandidates[scenario] } : {}; + await writeFile(join(taskDir, "task.json"), json({ ...task, ...candidates })); + const started = performance.now(); + const exit = await runContainer( + { + runId, + name: `evorca-${runId}-${scenario}-${task.row}`, + image: agent.imageId, + agent, + taskDir, + dataDir, + scenario, + seed, + }, + join(taskDir, "log.txt"), + ); + const seconds = Math.round(performance.now() - started) / 1000; + const prediction = await readFile(join(taskDir, "out", "prediction.json"), "utf8").catch( + () => undefined, + ); + const { outcome } = outcomeOf(exit, prediction); + const exitCode = exit.kind === "exited" ? exit.code : null; + return { row: task.row, taskIndex: task.taskIndex, outcome, exitCode, seconds }; +} + +function taskDirOf(runDir: string, row: number): string { + return join(runDir, "tasks", String(row).padStart(3, "0")); +} + +async function gitState(dir: string): Promise { const git = async (...args: string[]) => (await run("git", ["-C", dir, ...args])).stdout.trim(); return { commit: await git("rev-parse", "HEAD"), diff --git a/packages/cli/src/runs.test.ts b/packages/cli/src/runs.test.ts new file mode 100644 index 0000000..ab01484 --- /dev/null +++ b/packages/cli/src/runs.test.ts @@ -0,0 +1,151 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readdir, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { describe, expect, it } from "vitest"; +import { agents } from "./agents.js"; +import { + conflicts, + runPool, + tasksToRun, + writeJson, + type Conditions, + type TaskRecord, +} from "./runs.js"; + +const { jobs: _, ...random } = agents.random; +const recorded: Conditions = { + agent: { name: "random", ...random, image: "evorca-agent-random:abc", imageId: "sha256:1" }, + seed: 7, + rows: [0, 7, 21], +}; + +describe("conflicts", () => { + it("accepts a resume that asks for nothing or for the run's own settings", () => { + expect(conflicts(recorded, {})).toEqual([]); + expect( + conflicts(recorded, { seed: 7, rows: [0, 7, 21], agent: { ...recorded.agent } }), + ).toEqual([]); + }); + + it("names each setting that differs from the run's", () => { + const problems = conflicts(recorded, { + seed: 8, + rows: [0, 7], + agent: { ...recorded.agent, imageId: "sha256:2" }, + }); + + expect(problems).toHaveLength(3); + expect(problems[0]).toMatch(/^agent: /); + expect(problems[1]).toBe("seed: the run has 7, not 8"); + expect(problems[2]).toBe("rows: the run has [0,7,21], not [0,7]"); + }); + + it("rejects a subset of rows on a run of every row", () => { + expect(conflicts({ ...recorded, rows: null }, { rows: [0] })).toEqual([ + "rows: the run has null, not [0]", + ]); + }); +}); + +describe("tasksToRun", () => { + const record = (row: number): TaskRecord => ({ + row, + taskIndex: "task_1", + outcome: "answered", + exitCode: 0, + seconds: 1, + }); + + it("keeps the tasks without an outcome, in task order", () => { + const tasks = [0, 1, 2, 3, 4].map((row) => ({ row })); + + expect(tasksToRun(tasks, [record(3), record(0)])).toEqual([{ row: 1 }, { row: 2 }, { row: 4 }]); + }); + + it("runs every task of a new run and none of a finished one", () => { + const tasks = [0, 1].map((row) => ({ row })); + + expect(tasksToRun(tasks, [])).toEqual(tasks); + expect(tasksToRun(tasks, [record(0), record(1)])).toEqual([]); + }); +}); + +describe("runPool", () => { + const rows = Array.from({ length: 24 }, (_, row) => row); + + function answer(seed: number, row: number): string { + return createHash("sha256").update(`${seed}/${row}`).digest("hex").slice(0, 12); + } + + async function runAll(jobs: number) { + const answers = new Map(); + let running = 0; + let most = 0; + await runPool(rows, jobs, async (row) => { + running += 1; + most = Math.max(most, running); + await sleep(Math.random() * 5); + answers.set(row, answer(7, row)); + running -= 1; + }); + return { answers: [...answers].sort(([a], [b]) => a - b), most }; + } + + it("gives the same answers with 1 job and with 4", async () => { + const one = await runAll(1); + const four = await runAll(4); + + expect(four.answers).toEqual(one.answers); + expect(one.answers).toHaveLength(rows.length); + expect(one.most).toBe(1); + expect(four.most).toBe(4); + }); + + it("starts no task after one throws, waits for the others, and throws the error", async () => { + const started: number[] = []; + const finished: number[] = []; + const run = runPool(rows, 3, async (row) => { + started.push(row); + if (row === 4) throw new Error("docker run failed"); + await sleep(10); + finished.push(row); + }); + + await expect(run).rejects.toThrow("docker run failed"); + expect(started).toEqual([0, 1, 2, 3, 4]); + expect(finished).toEqual([0, 1, 2, 3]); + }); + + it("starts no task after the signal aborts", async () => { + const stop = new AbortController(); + const started: number[] = []; + await runPool( + rows, + 2, + async (row) => { + started.push(row); + if (row === 3) stop.abort(); + await sleep(1); + }, + stop.signal, + ); + + expect(started).toEqual([0, 1, 2, 3]); + }); +}); + +describe("writeJson", () => { + it("writes indented JSON and leaves no staging file", async () => { + const dir = await mkdtemp(join(tmpdir(), "evorca-runs-")); + const path = join(dir, "run.json"); + await writeJson(path, { tasks: [] }); + await writeJson(path, { tasks: [{ row: 0 }] }); + + expect(await readFile(path, "utf8")).toBe( + '{\n "tasks": [\n {\n "row": 0\n }\n ]\n}\n', + ); + expect(await readdir(dir)).toEqual(["run.json"]); + }); +}); diff --git a/packages/cli/src/runs.ts b/packages/cli/src/runs.ts new file mode 100644 index 0000000..9a3438e --- /dev/null +++ b/packages/cli/src/runs.ts @@ -0,0 +1,156 @@ +/** + * The record of an `evorca evaluate` run, `run.json`, and what a resumed run + * needs from it. `evorca evaluate` rewrites `run.json` after every task, so a + * run that stops can resume from its last finished task with the settings it + * started with. + * + * @module + */ +import { readFile, rename, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import type { OpenRcaScenario, TaskIndex } from "@evorca/openrca"; +import type { pinnedInputs } from "@evorca/openrca/scoring"; +import type { Agent, AgentName, Outcome } from "./agents.js"; + +/** + * What `run.json` records about one task. `exitCode` is null after a timeout or + * when a signal ended the `docker run` client. + */ +export type TaskRecord = { + readonly row: number; + readonly taskIndex: TaskIndex; + readonly outcome: Outcome; + readonly exitCode: number | null; + readonly seconds: number; +}; + +/** The agent of a run: its entry in the agents table, its name, and its image. */ +export type AgentRecord = Omit & { + readonly name: AgentName; + readonly image: string; + readonly imageId: string; +}; + +/** The git commit of a session of a run, and whether the checkout had uncommitted changes. */ +export type GitState = { readonly commit: string; readonly dirty: boolean }; + +/** The Node version and the Docker versions of a session of a run. */ +export type Versions = { + readonly node: string; + readonly docker: { readonly client: string; readonly server: string }; +}; + +/** + * What `run.json` records about the run of one agent on one scenario. `rows` is + * null when the run covers every task. `jobs` is how many tasks ran at once in + * the latest session. `resumes` lists each later session of the run, and + * `endedAt` is null until every task has an outcome. + */ +export type RunRecord = { + readonly runId: string; + readonly benchmark: "openrca"; + readonly scenario: OpenRcaScenario; + readonly agent: AgentRecord; + readonly seed: number; + readonly rows: readonly number[] | null; + readonly jobs: number; + readonly git: GitState; + readonly pinnedInputs: typeof pinnedInputs; + readonly versions: Versions; + readonly startedAt: string; + readonly resumes: readonly { + readonly startedAt: string; + readonly git: GitState; + readonly versions: Versions; + }[]; + readonly endedAt: string | null; + readonly tasks: readonly TaskRecord[]; +}; + +/** The settings of a run that its results depend on. A resumed run keeps each of them. */ +export type Conditions = Pick; + +const conditionKeys = ["agent", "seed", "rows"] as const satisfies readonly (keyof Conditions)[]; + +/** + * Returns one line for each setting that a resumed run asks for and that + * differs from the run's own. An undefined setting keeps the run's. + */ +export function conflicts(recorded: Conditions, requested: Partial): string[] { + return conditionKeys.flatMap((key) => { + const value = requested[key]; + if (value === undefined || isDeepStrictEqual(value, recorded[key])) return []; + return [`${key}: the run has ${JSON.stringify(recorded[key])}, not ${JSON.stringify(value)}`]; + }); +} + +/** Returns the tasks of `tasks` that have no outcome in `records`, in the order of `tasks`. */ +export function tasksToRun( + tasks: readonly T[], + records: readonly TaskRecord[], +): T[] { + const done = new Set(records.map((record) => record.row)); + return tasks.filter((task) => !done.has(task.row)); +} + +/** + * Calls `work` on each item, at most `jobs` at a time, and starts the items in + * order. Once `signal` aborts or a call throws, it starts no more items. It + * waits for the calls in flight, then throws the first error. + */ +export async function runPool( + items: readonly T[], + jobs: number, + work: (item: T) => Promise, + signal?: AbortSignal, +): Promise { + // The workers share one iterator, so each item goes to exactly one worker. + const queue = items.values(); + let failure: { readonly error: unknown } | undefined; + const worker = async () => { + for (const item of queue) { + if (failure !== undefined || signal?.aborted) return; + await work(item).catch((error: unknown) => { + failure ??= { error }; + }); + } + }; + await Promise.all(Array.from({ length: Math.min(jobs, items.length) }, worker)); + if (failure !== undefined) throw failure.error; +} + +/** + * Writes `value` to `path` as indented JSON. It writes a staging file and + * renames it into place, so a reader or a crash never sees half a file. Calls + * for the same path must not overlap. + */ +export async function writeJson(path: string, value: unknown): Promise { + const staging = `${path}.tmp`; + await writeFile(staging, `${JSON.stringify(value, null, 2)}\n`); + await rename(staging, path); +} + +/** + * Reads the `run.json` of run `runId` in the folder of each of `scenarios` that + * has one, and returns them by scenario. + * + * @throws Error if a `run.json` exists but cannot be read or parsed. + */ +export async function readRuns( + runsDir: string, + runId: string, + scenarios: readonly OpenRcaScenario[], +): Promise> { + const runs = new Map(); + for (const scenario of scenarios) { + const text = await readFile(join(runsDir, scenario, runId, "run.json"), "utf8").catch( + (error: unknown) => { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return undefined; + throw error; + }, + ); + if (text !== undefined) runs.set(scenario, JSON.parse(text)); + } + return runs; +}