From 2adb345cb1bb3ce94a3b7bf8a55c0e9086670fa8 Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:42:07 +0200 Subject: [PATCH 1/6] feat(agent-react): add the task and answer model Parse /task.json, define the strict submit_answer tool, check an answer against the fields that the task asks for, and write it as OpenRCA prediction JSON. --- agents/react/package-lock.json | 97 +++++++++++++++ agents/react/package.json | 17 +++ agents/react/src/answer.ts | 206 +++++++++++++++++++++++++++++++ agents/react/src/task.ts | 65 ++++++++++ agents/react/test/answer.test.ts | 133 ++++++++++++++++++++ agents/react/tsconfig.json | 10 ++ pnpm-lock.yaml | 70 ++++++++++- 7 files changed, 596 insertions(+), 2 deletions(-) create mode 100644 agents/react/package-lock.json create mode 100644 agents/react/package.json create mode 100644 agents/react/src/answer.ts create mode 100644 agents/react/src/task.ts create mode 100644 agents/react/test/answer.test.ts create mode 100644 agents/react/tsconfig.json diff --git a/agents/react/package-lock.json b/agents/react/package-lock.json new file mode 100644 index 0000000..db8ce4d --- /dev/null +++ b/agents/react/package-lock.json @@ -0,0 +1,97 @@ +{ + "name": "@evorca/agent-react", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@evorca/agent-react", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "@anthropic-ai/sdk": "1.4.0" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-1.4.0.tgz", + "integrity": "sha512-tyuibnhQL57kLOx5oomY4Ld7jOVOPNC5v3IiyRtR3l+Friv66tI4VbuZYF2NMel/T0WMuQMBFOMKDBUbUh8gPA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.1", + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.14", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.14.tgz", + "integrity": "sha512-Y5tqevQaUAZhhj82BpYMNC1ssvLQTVAfwKdm8iy8qwmhvC2/CpcAu8OC+QEtjM7pMk7knx83u1Qj8gzkicyelw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.10.0.tgz", + "integrity": "sha512-lMkyes8iHrG+mk6WH7X70MdCLI2RS0FwNBY1PyQ8UemaijE2m5mv5A0Niy3HtbYTFrgkUEQjFMXmSAk2hQDuPQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/standardwebhooks": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.1.1.tgz", + "integrity": "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + } + } +} diff --git a/agents/react/package.json b/agents/react/package.json new file mode 100644 index 0000000..9a86997 --- /dev/null +++ b/agents/react/package.json @@ -0,0 +1,17 @@ +{ + "name": "@evorca/agent-react", + "version": "0.0.0", + "private": true, + "description": "The ReAct agent, the evorca baseline without self-improvement.", + "license": "Apache-2.0", + "type": "module", + "scripts": { + "typecheck": "tsc --noEmit", + "test": "vitest run", + "extract-background": "tsx scripts/extract-background.ts", + "smoke": "tsx scripts/smoke.ts" + }, + "dependencies": { + "@anthropic-ai/sdk": "1.4.0" + } +} diff --git a/agents/react/src/answer.ts b/agents/react/src/answer.ts new file mode 100644 index 0000000..96bc59a --- /dev/null +++ b/agents/react/src/answer.ts @@ -0,0 +1,206 @@ +/** + * The `submit_answer` tool, and the answer that the model gives through it. + * + * The tool takes one root cause per failure, each with a datetime, a component, + * and a reason, any of them null. Its schema is strict, so the API returns only + * inputs that match it, and with candidates the component and the reason are + * enums of them. {@link readAnswer} still checks every input, because the model + * can give a datetime in the wrong format or leave out a field that the task + * scores. A rejected answer keeps its valid fields as a partial answer. + * + * OpenRCA's evaluator finds each root cause in the prediction with a regular + * expression that expects the fields in the order datetime, component, reason, + * so {@link formatPrediction} writes them in that order. + * + * @module + */ +import type Anthropic from "@anthropic-ai/sdk"; +import { isRecord, type Candidates, type Task, type TaskIndex } from "./task.ts"; + +/** A field of a root cause. */ +export type Field = "datetime" | "component" | "reason"; + +/** One root cause of an answer. A field is absent when the answer leaves it out. */ +export type Cause = { readonly [field in Field]?: string }; + +/** An answer read from `submit_answer`, or why it was rejected and what of it is valid. */ +export type Reading = + | { readonly valid: true; readonly causes: readonly Cause[] } + | { + readonly valid: false; + readonly problems: readonly string[]; + readonly partial: readonly Cause[]; + }; + +const fields = ["datetime", "component", "reason"] as const satisfies readonly Field[]; + +const inputNames: Readonly> = { + datetime: "occurrence_datetime", + component: "component", + reason: "reason", +}; + +const predictionNames: Readonly> = { + datetime: "root cause occurrence datetime", + component: "root cause component", + reason: "root cause reason", +}; + +const scoredFields: Readonly> = { + task_1: ["datetime"], + task_2: ["reason"], + task_3: ["component"], + task_4: ["datetime", "reason"], + task_5: ["datetime", "component"], + task_6: ["component", "reason"], + task_7: ["datetime", "component", "reason"], +}; + +/** The name of the answer tool. */ +export const submitAnswer = "submit_answer"; + +/** + * Returns the definition of the `submit_answer` tool. With candidates, the + * component and the reason must be one of them. + */ +export function submitAnswerTool(candidates: Candidates | undefined): Anthropic.Tool { + const choice = (names: readonly string[] | undefined) => + names === undefined ? { type: "string" } : { type: "string", enum: names }; + const nullable = (schema: object, description: string) => ({ + anyOf: [schema, { type: "null" }], + description, + }); + return { + name: submitAnswer, + description: + "Submit your final answer to the issue: the root cause of each failure. Call it once, when you have found the root causes. It ends your investigation.", + strict: true, + input_schema: { + type: "object", + properties: { + root_causes: { + type: "array", + description: "One root cause per failure, as many as the issue says occurred.", + items: { + type: "object", + properties: { + [inputNames.datetime]: nullable( + { type: "string" }, + "When the root cause occurred, in UTC+8, in the format YYYY-MM-DD HH:MM:SS. null if the issue does not ask for it.", + ), + [inputNames.component]: nullable( + choice(candidates?.components), + "The root cause component. null if the issue does not ask for it.", + ), + [inputNames.reason]: nullable( + choice(candidates?.reasons), + "The root cause reason. null if the issue does not ask for it.", + ), + }, + required: fields.map((field) => inputNames[field]), + additionalProperties: false, + }, + }, + }, + required: ["root_causes"], + additionalProperties: false, + }, + }; +} + +/** + * Reads the input of a `submit_answer` call for `task`. The answer is valid if + * it has at least one root cause, every field it gives is valid, and every root + * cause gives each field that the task scores. A datetime is valid in the form + * `YYYY-MM-DD HH:MM:SS` for a time that exists. With candidates, a component or + * a reason is valid if it is one of them, and otherwise if it is not empty. + */ +export function readAnswer(input: unknown, task: Task): Reading { + const rootCauses = isRecord(input) ? input.root_causes : undefined; + if (!Array.isArray(rootCauses) || rootCauses.length === 0) { + return { + valid: false, + problems: ["root_causes must list at least one root cause."], + partial: [], + }; + } + const problems: string[] = []; + const causes = rootCauses.map((value: unknown, i): Cause => { + const label = `Root cause ${i + 1}`; + if (!isRecord(value)) { + problems.push(`${label} is not an object.`); + return {}; + } + const cause: { [field in Field]?: string } = {}; + for (const field of fields) { + const given = value[inputNames[field]]; + if (given === null || given === undefined) continue; + const problem = checkField(field, given, task.candidates); + if (problem === undefined && typeof given === "string") cause[field] = given; + else problems.push(`${label}: ${problem}`); + } + for (const field of scoredFields[task.taskIndex]) { + if (cause[field] === undefined && value[inputNames[field]] == null) { + problems.push(`${label} has no ${inputNames[field]}, which the issue asks for.`); + } + } + return cause; + }); + if (problems.length === 0) return { valid: true, causes }; + return { + valid: false, + problems, + partial: causes.some((cause) => Object.keys(cause).length > 0) ? causes : [], + }; +} + +function checkField( + field: Field, + given: unknown, + candidates: Candidates | undefined, +): string | undefined { + const name = inputNames[field]; + if (typeof given !== "string") return `${name} must be a string or null.`; + if (field === "datetime") { + return isDatetime(given) + ? undefined + : `${name} ${JSON.stringify(given)} is not a datetime in the format YYYY-MM-DD HH:MM:SS.`; + } + const allowed = field === "component" ? candidates?.components : candidates?.reasons; + if (allowed !== undefined && !allowed.includes(given)) { + return `${name} ${JSON.stringify(given)} is not one of the possible root cause ${field}s.`; + } + return given.trim() === "" ? `${name} is empty.` : undefined; +} + +function isDatetime(text: string): boolean { + const parts = /^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})$/ + .exec(text) + ?.slice(1) + .map(Number); + if (parts === undefined) return false; + const [year = 0, month = 0, day = 0, hour = 0, minute = 0, second = 0] = parts; + const time = Date.UTC(year, month - 1, day, hour, minute, second); + return new Date(time).toISOString().slice(0, 19).replace("T", " ") === text; +} + +/** + * Formats root causes as OpenRCA prediction JSON, numbered from 1, with each + * cause's fields in the order that the evaluator expects, such as + * `{"1":{"root cause occurrence datetime":"2020-04-11 00:05:00"}}`. + */ +export function formatPrediction(causes: readonly Cause[]): string { + return JSON.stringify( + Object.fromEntries( + causes.map((cause, i) => [ + String(i + 1), + Object.fromEntries( + fields.flatMap((field) => { + const value = cause[field]; + return value === undefined ? [] : [[predictionNames[field], value]]; + }), + ), + ]), + ), + ); +} diff --git a/agents/react/src/task.ts b/agents/react/src/task.ts new file mode 100644 index 0000000..ef986ea --- /dev/null +++ b/agents/react/src/task.ts @@ -0,0 +1,65 @@ +/** + * The task that `evorca evaluate` gives the agent in `/task.json`, parsed at + * the boundary so the rest of the agent trusts its shape. + * + * @module + */ + +const taskIndices = ["task_1", "task_2", "task_3", "task_4", "task_5", "task_6", "task_7"] as const; + +/** The kind of an OpenRCA task, which decides the root cause fields that it scores. */ +export type TaskIndex = (typeof taskIndices)[number]; + +/** The root cause components and reasons that an answer can name. */ +export type Candidates = { + readonly components: readonly string[]; + readonly reasons: readonly string[]; +}; + +/** One OpenRCA task, as `/task.json` holds it. `candidates` is absent unless the agent sees them. */ +export type Task = { + readonly scenario: string; + readonly row: number; + readonly taskIndex: TaskIndex; + readonly instruction: string; + readonly candidates?: Candidates; +}; + +/** + * Parses the text of `/task.json`. + * + * @throws Error if the text is not JSON or lacks a field of {@link Task}. + */ +export function parseTask(text: string): Task { + const value: unknown = JSON.parse(text); + if (!isRecord(value)) throw new Error("task.json is not an object"); + const { scenario, row, taskIndex, instruction, candidates } = value; + if (typeof scenario !== "string") throw new Error("task.json has no scenario"); + if (typeof row !== "number" || !Number.isInteger(row)) throw new Error("task.json has no row"); + const index = taskIndices.find((candidate) => candidate === taskIndex); + if (index === undefined) throw new Error(`task.json has an unknown taskIndex: ${taskIndex}`); + if (typeof instruction !== "string") throw new Error("task.json has no instruction"); + return { + scenario, + row, + taskIndex: index, + instruction, + ...(candidates === undefined ? {} : { candidates: parseCandidates(candidates) }), + }; +} + +function parseCandidates(value: unknown): Candidates { + if (!isRecord(value) || !isStrings(value.components) || !isStrings(value.reasons)) { + throw new Error("task.json has candidates without lists of components and reasons"); + } + return { components: value.components, reasons: value.reasons }; +} + +function isStrings(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} + +/** Whether `value` is a JSON object. */ +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/agents/react/test/answer.test.ts b/agents/react/test/answer.test.ts new file mode 100644 index 0000000..d08a6de --- /dev/null +++ b/agents/react/test/answer.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from "vitest"; +import { parsePrediction } from "../../../packages/openrca/src/prediction.ts"; +import { formatPrediction, readAnswer, submitAnswerTool } from "../src/answer.ts"; +import type { Task } from "../src/task.ts"; + +const task: Task = { + scenario: "telecom", + row: 3, + taskIndex: "task_7", + instruction: "On April 11, 2020, between 00:00 and 00:30, two failures occurred.", + candidates: { components: ["os_001", "docker_003"], reasons: ["CPU fault", "network delay"] }, +}; + +const cause = ( + occurrence_datetime: string | null, + component: string | null, + reason: string | null, +) => ({ occurrence_datetime, component, reason }); + +describe("readAnswer", () => { + it("accepts one root cause per failure with every field the task asks for", () => { + const reading = readAnswer( + { + root_causes: [ + cause("2020-04-11 00:05:00", "docker_003", "CPU fault"), + cause("2020-04-11 00:20:00", "os_001", "network delay"), + ], + }, + task, + ); + expect(reading).toEqual({ + valid: true, + causes: [ + { datetime: "2020-04-11 00:05:00", component: "docker_003", reason: "CPU fault" }, + { datetime: "2020-04-11 00:20:00", component: "os_001", reason: "network delay" }, + ], + }); + }); + + it.each([ + ["task_1", cause("2020-04-11 00:05:00", null, null)], + ["task_2", cause(null, null, "CPU fault")], + ["task_3", cause(null, "os_001", null)], + ["task_4", cause("2020-04-11 00:05:00", null, "CPU fault")], + ["task_5", cause("2020-04-11 00:05:00", "os_001", null)], + ["task_6", cause(null, "os_001", "CPU fault")], + ] as const)("accepts only the fields that %s asks for", (taskIndex, given) => { + expect(readAnswer({ root_causes: [given] }, { ...task, taskIndex }).valid).toBe(true); + }); + + it("rejects an answer without a field that the task asks for, and keeps the rest", () => { + const reading = readAnswer( + { root_causes: [cause(null, "os_001", null)] }, + { ...task, taskIndex: "task_5" }, + ); + expect(reading).toEqual({ + valid: false, + problems: ["Root cause 1 has no occurrence_datetime, which the issue asks for."], + partial: [{ component: "os_001" }], + }); + }); + + it("rejects an invalid datetime, an unknown candidate, and an empty list", () => { + const reading = readAnswer( + { + root_causes: [ + cause("2020-04-11T00:05:00Z", "docker_003", "disk full"), + cause("2020-02-30 00:00:00", "os_001", "CPU fault"), + ], + }, + task, + ); + expect(reading).toEqual({ + valid: false, + problems: [ + 'Root cause 1: occurrence_datetime "2020-04-11T00:05:00Z" is not a datetime in the format YYYY-MM-DD HH:MM:SS.', + 'Root cause 1: reason "disk full" is not one of the possible root cause reasons.', + 'Root cause 2: occurrence_datetime "2020-02-30 00:00:00" is not a datetime in the format YYYY-MM-DD HH:MM:SS.', + ], + partial: [{ component: "docker_003" }, { component: "os_001", reason: "CPU fault" }], + }); + expect(readAnswer({ root_causes: [] }, task)).toEqual({ + valid: false, + problems: ["root_causes must list at least one root cause."], + partial: [], + }); + expect(readAnswer("nonsense", task).valid).toBe(false); + }); + + it("accepts any non-empty component and reason without candidates", () => { + const { candidates: _, ...blind } = task; + const reading = readAnswer( + { root_causes: [cause("2020-04-11 00:05:00", "node-7", "gremlins")] }, + blind, + ); + expect(reading.valid).toBe(true); + expect(readAnswer({ root_causes: [cause("2020-04-11 00:05:00", " ", "x")] }, blind).valid).toBe( + false, + ); + }); +}); + +describe("formatPrediction", () => { + it("writes OpenRCA prediction JSON that the runner parses back", () => { + const causes = [ + { datetime: "2020-04-11 00:05:00", component: "docker_003", reason: "CPU fault" }, + { component: "os_001" }, + ]; + const json = formatPrediction(causes); + expect(json).toBe( + '{"1":{"root cause occurrence datetime":"2020-04-11 00:05:00","root cause component":"docker_003","root cause reason":"CPU fault"},"2":{"root cause component":"os_001"}}', + ); + expect(parsePrediction(json)).toEqual([ + { + occurredAt: Date.UTC(2020, 3, 10, 16, 5) / 1000, + component: "docker_003", + reason: "CPU fault", + }, + { occurredAt: undefined, component: "os_001", reason: undefined }, + ]); + }); +}); + +describe("submitAnswerTool", () => { + it("is strict, and limits the component and the reason to the candidates", () => { + const tool = submitAnswerTool(task.candidates); + expect(tool.strict).toBe(true); + const item = JSON.stringify(tool.input_schema); + expect(item).toContain('"enum":["os_001","docker_003"]'); + expect(item).toContain('"enum":["CPU fault","network delay"]'); + expect(JSON.stringify(submitAnswerTool(undefined).input_schema)).not.toContain("enum"); + }); +}); diff --git a/agents/react/tsconfig.json b/agents/react/tsconfig.json new file mode 100644 index 0000000..784dfcb --- /dev/null +++ b/agents/react/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "allowImportingTsExtensions": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true + }, + "include": ["src", "test", "scripts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 807edd4..290a807 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -25,10 +25,16 @@ importers: version: 6.0.3 vitest: specifier: ^5.0.5 - version: 5.0.5(@types/node@26.6.6)(vite@8.3.5(@types/node@26.6.6)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) + version: 5.0.5(@opentelemetry/api@1.10.0)(@types/node@26.6.6)(vite@8.3.5(@types/node@26.6.6)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) agents/random: {} + agents/react: + dependencies: + '@anthropic-ai/sdk': + specifier: 1.4.0 + version: 1.4.0 + packages/cli: dependencies: '@evorca/core': @@ -89,6 +95,19 @@ importers: packages: + '@anthropic-ai/sdk@1.4.0': + resolution: {integrity: sha512-tyuibnhQL57kLOx5oomY4Ld7jOVOPNC5v3IiyRtR3l+Friv66tI4VbuZYF2NMel/T0WMuQMBFOMKDBUbUh8gPA==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@babel/runtime@7.29.14': + resolution: {integrity: sha512-Y5tqevQaUAZhhj82BpYMNC1ssvLQTVAfwKdm8iy8qwmhvC2/CpcAu8OC+QEtjM7pMk7knx83u1Qj8gzkicyelw==} + engines: {node: '>=6.9.0'} + '@changesets/apply-release-plan@8.1.1': resolution: {integrity: sha512-nk2Hbq3M9NeHMdaIRBDvawKrQJzXB9sGyBUE5yWnIKtvdcNCPj75zBekJYr0gTT2ecJH3eFeTrc8eisPImcv0Q==} engines: {node: ^22.11 || ^24 || >=26} @@ -336,6 +355,10 @@ packages: resolution: {integrity: sha512-6QEf6yqFbETdwGITKq57aYoPfX/3K8XFNwsAlx0C1M7o8cb79sv1M3w+tWuWvIcSbNqrLF7OD7YpZMVVz335hQ==} engines: {node: '>=20.0.0'} + '@opentelemetry/api@1.10.0': + resolution: {integrity: sha512-lMkyes8iHrG+mk6WH7X70MdCLI2RS0FwNBY1PyQ8UemaijE2m5mv5A0Niy3HtbYTFrgkUEQjFMXmSAk2hQDuPQ==} + engines: {node: '>=8.0.0'} + '@opentui/core-darwin-arm64@0.9.2': resolution: {integrity: sha512-6ASALGMPpmpsoGKYUpJYF+HJs1lZiw9zYsUhInCRYuVvBxYkBqV7MSz+1HFsTTd/ewqFuNnV/JMlEieMXZqiEA==} cpu: [arm64] @@ -489,6 +512,9 @@ packages: '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} + '@stablelib/base64@1.0.1': + resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} + '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -584,6 +610,9 @@ packages: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} + fast-sha256@1.3.0: + resolution: {integrity: sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -621,6 +650,10 @@ packages: jju@1.4.0: resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==} + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + jsonc-parser@3.3.1: resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} @@ -768,6 +801,9 @@ packages: resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} engines: {node: '>=0.10.0'} + standardwebhooks@1.1.1: + resolution: {integrity: sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==} + std-env@4.3.0: resolution: {integrity: sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==} @@ -791,6 +827,9 @@ packages: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + tsx@4.23.15: resolution: {integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==} engines: {node: '>=18.0.0'} @@ -939,6 +978,14 @@ packages: snapshots: + '@anthropic-ai/sdk@1.4.0': + dependencies: + '@opentelemetry/api': 1.10.0 + json-schema-to-ts: 3.1.1 + standardwebhooks: 1.1.1 + + '@babel/runtime@7.29.14': {} + '@changesets/apply-release-plan@8.1.1': dependencies: '@changesets/config': 4.0.1 @@ -1157,6 +1204,8 @@ snapshots: tinyglobby: 0.2.17 yaml: 2.9.1 + '@opentelemetry/api@1.10.0': {} + '@opentui/core-darwin-arm64@0.9.2': optional: true @@ -1261,6 +1310,8 @@ snapshots: '@rolldown/pluginutils@1.0.1': {} + '@stablelib/base64@1.0.1': {} + '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -1360,6 +1411,8 @@ snapshots: expect-type@1.4.0: {} + fast-sha256@1.3.0: {} + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -1385,6 +1438,11 @@ snapshots: jju@1.4.0: {} + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.14 + ts-algebra: 2.0.0 + jsonc-parser@3.3.1: {} launch-editor@2.14.2: @@ -1511,6 +1569,11 @@ snapshots: source-map-js@1.2.2: {} + standardwebhooks@1.1.1: + dependencies: + '@stablelib/base64': 1.0.1 + fast-sha256: 1.3.0 + std-env@4.3.0: {} string-width@7.2.0: @@ -1532,6 +1595,8 @@ snapshots: fdir: 6.5.0(picomatch@4.0.7) picomatch: 4.0.7 + ts-algebra@2.0.0: {} + tsx@4.23.15: dependencies: esbuild: 0.28.2 @@ -1556,7 +1621,7 @@ snapshots: tsx: 4.23.15 yaml: 2.9.1 - vitest@5.0.5(@types/node@26.6.6)(vite@8.3.5(@types/node@26.6.6)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)): + vitest@5.0.5(@opentelemetry/api@1.10.0)(@types/node@26.6.6)(vite@8.3.5(@types/node@26.6.6)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)): dependencies: '@types/chai': 5.2.3 '@vitest/mocker': 5.0.5(vite@8.3.5(@types/node@26.6.6)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1)) @@ -1573,6 +1638,7 @@ snapshots: vite: 8.3.5(@types/node@26.6.6)(esbuild@0.28.2)(tsx@4.23.15)(yaml@2.9.1) why-is-node-running: 3.2.1 optionalDependencies: + '@opentelemetry/api': 1.10.0 '@types/node': 26.6.6 transitivePeerDependencies: - msw From 7b8385ae939d635cfaa8123e63367c9d5fc2c1f1 Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:42:12 +0200 Subject: [PATCH 2/6] feat(agent-react): add the persistent bash session One bash process runs the model's commands in turn. A command past the timeout ends the session's process group, and long output keeps its start and its end. --- agents/react/src/bash.ts | 193 +++++++++++++++++++++++++++++++++ agents/react/test/bash.test.ts | 119 ++++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 agents/react/src/bash.ts create mode 100644 agents/react/test/bash.test.ts diff --git a/agents/react/src/bash.ts b/agents/react/src/bash.ts new file mode 100644 index 0000000..56b083f --- /dev/null +++ b/agents/react/src/bash.ts @@ -0,0 +1,193 @@ +/** + * The bash session behind the `bash` tool. One bash process runs the model's + * commands one at a time, so the working directory, variables, and functions + * persist between calls. + * + * Each command runs through `eval` with stdin from `/dev/null`, so a syntax + * error or a command that reads stdin cannot swallow what follows. After the + * command, bash prints a marker with a random nonce and the exit status, and + * the output ends where the marker starts. stderr goes to stdout, in order. + * + * A command that runs past the timeout kills the session's whole process + * group, and so does a restart. The next command then starts a new session. + * Output longer than the limit keeps its start and its end around a note. + * + * @module + */ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { randomBytes } from "node:crypto"; + +/** Where the session runs and its limits. */ +export type BashOptions = { + readonly cwd: string; + readonly env: NodeJS.ProcessEnv; + readonly timeoutMs: number; + readonly outputChars: number; +}; + +/** + * How a command ended: it finished with an exit status, it ran past the + * timeout, or the shell itself exited, as after `exit`. + */ +export type CommandEnd = + | { readonly kind: "finished"; readonly exitCode: number } + | { readonly kind: "timeout" } + | { readonly kind: "session-exit"; readonly code: number | null }; + +/** What a command printed, cut to the output limit, and how it ended. */ +export type CommandResult = { readonly output: string; readonly end: CommandEnd }; + +/** A persistent bash session. It starts a shell on the first command. */ +export class BashSession { + readonly #options: BashOptions; + #shell: Shell | undefined; + + constructor(options: BashOptions) { + this.#options = options; + } + + /** Runs `command` in the session and returns its output and how it ended. */ + async run(command: string): Promise { + this.#shell ??= new Shell(this.#options); + const result = await this.#shell.run(command); + if (result.end.kind !== "finished") this.restart(); + return result; + } + + /** Ends the session. The next command starts a new one. */ + restart(): void { + this.#shell?.kill(); + this.#shell = undefined; + } +} + +class Shell { + readonly #child: ChildProcessWithoutNullStreams; + readonly #options: BashOptions; + readonly #endMarker: string; + + constructor(options: BashOptions) { + this.#options = options; + this.#endMarker = `\n__evorca_${randomBytes(12).toString("hex")}__ `; + this.#child = spawn("bash", ["--noprofile", "--norc"], { + cwd: options.cwd, + env: options.env, + detached: true, + }); + this.#child.stdin.on("error", () => {}); + this.#child.stdout.setEncoding("utf8"); + this.#child.stderr.setEncoding("utf8"); + // A runaway command should be the out-of-memory victim, not the agent. + this.#child.stdin.write("exec 2>&1\n{ echo 1000 > /proc/self/oom_score_adj; } 2>/dev/null\n"); + } + + run(command: string): Promise { + const capture = new Capture(this.#options.outputChars, this.#endMarker); + const child = this.#child; + return new Promise((resolve) => { + const finish = (end: CommandEnd) => { + clearTimeout(timer); + child.stdout.off("data", onData); + child.stderr.off("data", onData); + child.off("close", onClose); + resolve({ output: capture.output(), end }); + }; + const onData = (chunk: string) => { + const exitCode = capture.push(chunk); + if (exitCode !== undefined) finish({ kind: "finished", exitCode }); + }; + const onClose = (code: number | null) => finish({ kind: "session-exit", code }); + const timer = setTimeout(() => finish({ kind: "timeout" }), this.#options.timeoutMs); + if (child.exitCode !== null || child.signalCode !== null) { + finish({ kind: "session-exit", code: child.exitCode }); + return; + } + child.stdout.on("data", onData); + child.stderr.on("data", onData); + child.once("close", onClose); + const quoted = `'${command.replaceAll("'", `'\\''`)}'`; + const marker = this.#endMarker.slice(1, -1); + child.stdin.write(`eval ${quoted} < /dev/null\nprintf '\\n%s %d\\n' ${marker} "$?"\n`); + }); + } + + kill(): void { + if (this.#child.pid === undefined) return; + try { + process.kill(-this.#child.pid, "SIGKILL"); + } catch (error) { + if (!(error instanceof Error && "code" in error && error.code === "ESRCH")) throw error; + } + } +} + +/** + * Collects a command's output until the end marker. It never holds more than + * a few times the output limit: past that it drops the middle, keeping the + * start, the end, and a count of what it dropped. + */ +class Capture { + readonly #limit: number; + readonly #endMarker: string; + #text = ""; + #dropped = 0; + + constructor(limit: number, endMarker: string) { + this.#limit = limit; + this.#endMarker = endMarker; + } + + /** Adds a chunk of output. Returns the exit status once the marker line is complete. */ + push(chunk: string): number | undefined { + this.#text += chunk; + const at = this.#text.indexOf(this.#endMarker); + if (at >= 0) { + const newline = this.#text.indexOf("\n", at + this.#endMarker.length); + if (newline < 0) return undefined; + const exitCode = Number(this.#text.slice(at + this.#endMarker.length, newline)); + this.#text = this.#text.slice(0, at); + return exitCode; + } + if (this.#text.length > 4 * this.#limit) { + const half = Math.floor(this.#limit / 2); + const tail = half + this.#endMarker.length + 16; + this.#dropped += this.#text.length - half - tail; + this.#text = this.#text.slice(0, half) + this.#text.slice(-tail); + } + return undefined; + } + + /** The output so far, with its middle replaced by a note if it is longer than the limit. */ + output(): string { + const total = this.#text.length + this.#dropped; + if (total <= this.#limit) return this.#text.toWellFormed(); + const half = Math.floor(this.#limit / 2); + const omitted = total - 2 * half; + const head = this.#text.slice(0, half); + const tail = this.#text.slice(-half); + return `${head}\n\n[... ${omitted} characters omitted ...]\n\n${tail}`.toWellFormed(); + } +} + +/** + * Formats a command's result as the text of its tool result, with a note when + * the command failed, timed out, or ended the session. + */ +export function describeResult(result: CommandResult, timeoutSeconds: number): string { + const { output, end } = result; + const note = (text: string) => + `${output}${output === "" || output.endsWith("\n") ? "" : "\n"}[${text}]`; + switch (end.kind) { + case "finished": + if (end.exitCode !== 0) return note(`exit code ${end.exitCode}`); + return output === "" ? "(no output)" : output; + case "timeout": + return note( + `The command ran longer than ${timeoutSeconds} seconds and was stopped. The bash session restarted: its working directory and variables are lost, but files remain.`, + ); + case "session-exit": + return note( + `The bash session exited with code ${end.code}. The next command starts a new session.`, + ); + } +} diff --git a/agents/react/test/bash.test.ts b/agents/react/test/bash.test.ts new file mode 100644 index 0000000..f915167 --- /dev/null +++ b/agents/react/test/bash.test.ts @@ -0,0 +1,119 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { BashSession, describeResult, type BashOptions } from "../src/bash.ts"; + +let cwd: string; +let session: BashSession; + +const open = (options: Partial = {}) => + new BashSession({ cwd, env: process.env, timeoutMs: 5_000, outputChars: 1_000, ...options }); + +beforeEach(async () => { + cwd = await mkdtemp(join(tmpdir(), "evorca-bash-")); + session = open(); +}); + +afterEach(async () => { + session.restart(); + await rm(cwd, { recursive: true, force: true }); +}); + +describe("BashSession", () => { + it("returns stdout and stderr in order, with the exit status", async () => { + const result = await session.run( + "echo out; echo err >&2; echo out2; exit_code() { return 3; }; exit_code", + ); + expect(result).toEqual({ output: "out\nerr\nout2\n", end: { kind: "finished", exitCode: 3 } }); + }); + + it("keeps the working directory, variables, and functions between commands", async () => { + await session.run("mkdir -p work && cd work && export COLOR=teal && greet() { echo hi $1; }"); + const result = await session.run('basename "$PWD"; echo "$COLOR"; greet there'); + expect(result.output).toBe("work\nteal\nhi there\n"); + }); + + it("keeps output without a final newline as it is", async () => { + expect((await session.run("printf abc")).output).toBe("abc"); + expect((await session.run("true")).output).toBe(""); + }); + + it("survives a syntax error, quotes, and a command that reads stdin", async () => { + const broken = await session.run("echo 'unterminated"); + expect(broken.end).toEqual({ kind: "finished", exitCode: 2 }); + const quoted = await session.run(`echo "it's" 'a "test"'`); + expect(quoted.output).toBe(`it's a "test"\n`); + const stdin = await session.run("cat; echo after"); + expect(stdin.output).toBe("after\n"); + const heredoc = await session.run("python3 - <<'EOF'\nprint(6 * 7)\nEOF"); + expect(heredoc.output).toBe("42\n"); + }); + + it("stops a command at the timeout and starts a new session for the next one", async () => { + session = open({ timeoutMs: 300 }); + await session.run("export MARK=before"); + const started = Date.now(); + const slow = await session.run("echo partial; sleep 30"); + expect(Date.now() - started).toBeLessThan(5_000); + expect(slow).toEqual({ output: "partial\n", end: { kind: "timeout" } }); + const next = await session.run('echo "mark=$MARK"'); + expect(next.output).toBe("mark=\n"); + }); + + it("kills the timed-out command's children", async () => { + session = open({ timeoutMs: 300 }); + await session.run("sleep 30 & echo $! > pid; wait"); + const check = await session.run('kill -0 "$(cat pid)" 2>/dev/null && echo alive || echo gone'); + expect(check.output).toBe("gone\n"); + }); + + it("restarts on request and after the shell exits", async () => { + await session.run("export MARK=before"); + session.restart(); + expect((await session.run('echo "mark=$MARK"')).output).toBe("mark=\n"); + await session.run("export MARK=again"); + expect(await session.run("echo bye; exit 4")).toEqual({ + output: "bye\n", + end: { kind: "session-exit", code: 4 }, + }); + expect((await session.run('echo "mark=$MARK"')).output).toBe("mark=\n"); + }); + + it("keeps the start and the end of long output around a note", async () => { + const result = await session.run("seq 1 100000"); + const full = `${Array.from({ length: 100_000 }, (_, i) => i + 1).join("\n")}\n`; + const omitted = full.length - 1_000; + expect(result.output).toBe( + `${full.slice(0, 500)}\n\n[... ${omitted} characters omitted ...]\n\n${full.slice(-500)}`, + ); + expect(result.end).toEqual({ kind: "finished", exitCode: 0 }); + }); + + it("keeps output at the limit whole", async () => { + const result = await session.run("head -c 1000 /dev/zero | tr '\\0' x"); + expect(result.output).toBe("x".repeat(1000)); + }); +}); + +describe("describeResult", () => { + it.each([ + [{ output: "", end: { kind: "finished", exitCode: 0 } }, "(no output)"], + [{ output: "ok\n", end: { kind: "finished", exitCode: 0 } }, "ok\n"], + [{ output: "", end: { kind: "finished", exitCode: 1 } }, "[exit code 1]"], + [ + { output: "no such file\n", end: { kind: "finished", exitCode: 2 } }, + "no such file\n[exit code 2]", + ], + [ + { output: "", end: { kind: "timeout" } }, + "[The command ran longer than 120 seconds and was stopped. The bash session restarted: its working directory and variables are lost, but files remain.]", + ], + [ + { output: "bye\n", end: { kind: "session-exit", code: 0 } }, + "bye\n[The bash session exited with code 0. The next command starts a new session.]", + ], + ] as const)("describes %j", (result, text) => { + expect(describeResult(result, 120)).toBe(text); + }); +}); From 842a92a48452797d550f1e1808af96403e0c44f2 Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:42:16 +0200 Subject: [PATCH 3/6] feat(agent-react): build the prompt from RCA-Agent's domain knowledge extract-background makes background.json from RCA-Agent's prompts at the pinned OpenRCA commit. buildPrompt adds the environment and the answer format, and keeps the system prompt and the tools the same for every task of a scenario. --- agents/react/background.json | 252 +++++++++++++++++++++ agents/react/scripts/extract-background.ts | 191 ++++++++++++++++ agents/react/src/prompt.ts | 141 ++++++++++++ agents/react/test/prompt.test.ts | 102 +++++++++ 4 files changed, 686 insertions(+) create mode 100644 agents/react/background.json create mode 100644 agents/react/scripts/extract-background.ts create mode 100644 agents/react/src/prompt.ts create mode 100644 agents/react/test/prompt.test.ts diff --git a/agents/react/background.json b/agents/react/background.json new file mode 100644 index 0000000..51d11cd --- /dev/null +++ b/agents/react/background.json @@ -0,0 +1,252 @@ +{ + "source": "OpenRCA 2a481ae63cb045c3eadbeca9eddaa10ec1151979: rca/baseline/rca_agent/prompt/ and rca/baseline/rca_agent/executor.py", + "rules": "## RULES OF FAILURE DIAGNOSIS:\n\nWhat you SHOULD do:\n\n1. **Follow the workflow of `preprocess -> anomaly detection -> fault identification -> root cause localization` for failure diagnosis.** \n 1.1. Preprocess:\n - Aggregate each KPI of each components that are possible to be the root cause component to obtain multiple time series classified by 'component-KPI' (e.g., service_A-cpu_usage_pct).\n - Then, calculate global thresholds (e.g., global P95, where 'global' means the threshold of all 'component-KPI' time series within a whole metric file) for each 'component-KPI' time series. - Finally, filter data within the given time duration for all time series to perform further analysis.\n - Since the root cause component must be selected from the provided possible root cause components, all other level's components (e.g., service mesh components, middleware components, etc.) should be ignored.\n 1.2. Anomaly detection: \n - An anomaly is typically a data point that exceeds the global threshold.\n - Look for anomalies below a certain threshold (e.g., <=P95, <=P15, or <=P5) in traffic KPIs or business KPIs (e.g., success rate (ss)) since some network failures can cause a sudden drop on them due to packet loss.\n - Loose the global threshold (e.g., from >=P95 to >=P90, or from <=P95 to <=P15, <=P5) if you really cannot find any anomalies.\n 1.3. Fault identification: \n - A 'fault' is a consecutive sub-series of a specific component-KPI time series. Thus, fault identification is the process of identifying which components experienced faults, on which resources, and at what occurrence time points.\n - Filter out isolated noise spikes to locate faults.\n - Faults where the maximum (or minimum) value in the sub-series only slightly exceeds (or falls below) the threshold (e.g., threshold breach <= 50% of the extremal), it’s likely a false positive caused by random KPI fluctuations, and should be excluded.\n 1.4. Root cause localization: \n - The objective of root cause localization is to determine which identified 'fault' is the root cause of the failure. The root cause occurrence time, component, and reason can be derived from the first piece of data point of that fault.\n - If multiple faulty components are identified at **different levels** (e.g., some being containers and others nodes), and all of them are potential root cause candidates, while the issue itself describes a **single failure**, the root cause level should be determined by the fault that shows the most significant deviation from the threshold (i.e., >> 50%). However, this method is only applicable to identify the root cause level, not the root cause component. If there are multiple faulty components at the same level, you should use traces and logs to identify the root cause component.\n - If multiple service-level faulty components are identified, the root cause component is typically the last (the most downstream in a call chain) **faulty** service within a trace. Use traces to identify the root cause component among multiple faulty services.\n - If multiple container-level faulty components are identified, the root cause component is typically the last (the most downstream in a call chain) **faulty** container within a trace. Use traces to identify the root cause component among multiple faulty container.\n - If multiple node-level faulty components are identified and the issue doesn't specify **a single failure**, each of these nodes might be the root cause of separate failures. Otherwise, the predominant nodes with the most faults is the root cause component. The node-level failure do not propagate, and trace only captures communication between all containers or all services.\n - If only one component's one resource KPI has one fault occurred in a specific time, that fault is the root cause. Otherwise, you should use traces and logs to identify the root cause component and reason.\n2. **Follow the order of `threshold calculation -> data extraction -> metric analyis -> trace analysis -> log analysis` for failure diagnosis.** \n 2.0. Before analysis: You should extract and filter the data to include those within the failure duration only after the global threshold has been calculated. After these two steps, you can perform metric analysis, trace analysis, and log analysis.\n 2.1. Metric analysis: Use metrics to calculate whether each KPIs of each component has consecutive anomalies beyond the global threshold is the fastest way to find the faults. Since there are a large number of traces and logs, metrics analysis should first be used to narrow down the search space of duration and components.\n 2.2. Trace analysis: Use traces can further localize which container-level or service-level faulty component is the root cause components when there are multiple faulty components at the same level (container or service) identified by metrics analysis.\n 2.3. Log analysis: Use logs can further localize which resource is the root cause reason when there are multiple faulty resource KPIs of a component identified by metrics analysis. Logs can also help to identify the root cause component among multiple faulty components at the same level.\n 2.4. Always confirm whether the target key or field is valid (e.g., component's name, KPI's name, trace ID, log ID, etc.) when a retrieval result is empty.\n\nWhat you SHOULD NOT do:\n\n1. **DO NOT use the local data (filtered/cached series in specific time duration) to calculate the global threshold of aggregated 'component-KPI' time series.** Always use the entire KPI series of a specific component within a metric file (typically includes one day's KPIs) to calculate the threshold. To obtain global threshold, you can first aggregate each component's each KPI to calculate their threshold, and then retrieve the objective time duration of aggregated 'component-KPI' to perform anomaly detection and spike filtering.\n2. **DO NOT calculate threshold AFTER filtering data within the given time duration.** Always calculate global thresholds using the entire KPI series of a specific component within a metric file BEFORE filtering data within the given time duration.\n3. **DO NOT query a specific KPI without knowing which KPIs are available.** Different systems may have completely different KPI naming conventions. If you want to query a specific KPI, first ensure that you are aware of all the available KPIs.\n4. **DO NOT mistakenly identify a healthy (non-faulty) service at the downstream end of a trace that includes faulty components as the root cause.** The root cause component should be the most downstream **faulty** service to appear within the trace call chain, which must first and foremost be a FAULTY component identified by metrics analysis.\n5. **DO NOT focus solely on warning or error logs during log analysis. Many info logs contain critical information about service operations and interactions between services, which can be valuable for root cause analysis.**\n\n## RULES OF PYTHON CODE WRITING:\n\n1. Do not calculate threshold AFTER filtering data within the given time duration. Always calculate global thresholds using the entire KPI series of a specific component within a metric file BEFORE filtering data within the given time duration.\n2. All issues use **UTC+8** time. However, the local machine's default timezone is unknown. Please use `pytz.timezone('Asia/Shanghai')` to explicityly set the timezone to UTC+8.", + "scenarios": { + "bank": { + "schema": "## TELEMETRY DIRECTORY STRUCTURE:\n\n- You can access the telemetry directory in our microservices system: `/data/`.\n\n- This directory contains subdirectories organized by a date (e.g., `/data/2021_03_05/`). \n\n- Within each date-specific directory, you’ll find these subdirectories: `metric`, `trace`, and `log` (e.g., `/data/2021_03_05/metric/`).\n\n- The telemetry data in those subdirectories is stored in CSV format (e.g., `/data/2021_03_05/metric/metric_container.csv`).\n\n## DATA SCHEMA\n\n1. **Metric Files**:\n \n 1. `metric_app.csv`:\n\n ```csv\n timestamp,rr,sr,cnt,mrt,tc\n 1614787440,100.0,100.0,22,53.27,ServiceTest1\n ```\n\n 2. `metric_container.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1614787200,Tomcat04,OSLinux-CPU_CPU_CPUCpuUtil,26.2957\n ```\n\n2. **Trace Files**:\n\n 1. `trace_span.csv`:\n\n ```csv\n timestamp,cmdb_id,parent_id,span_id,trace_id,duration\n 1614787199628,dockerA2,369-bcou-dle-way1-c514cf30-43410@0824-2f0e47a816-17492,21030300016145905763,gw0120210304000517192504,19\n ```\n\n3. **Log Files**:\n\n 1. `log_service.csv`:\n\n ```csv\n log_id,timestamp,cmdb_id,log_name,value\n 8c7f5908ed126abdd0de6dbdd739715c,1614787201,Tomcat01,gc,\"3748789.580: [GC (CMS Initial Mark) [1 CMS-initial-mark: 2462269K(3145728K)] 3160896K(4089472K), 0.1985754 secs] [Times: user=0.59 sys=0.00, real=0.20 secs] \"\n ```\n\n{cand}\n\n## CLARIFICATION OF TELEMETRY DATA:\n\n1. This microservice system is a banking platform.\n\n2. The `metric_app.csv` file only contains four KPIs: rr, sr, cnt, and mrt,. In contrast, `metric_container.csv` records a variety of KPIs, such as CPU usage and memory usage. The specific names of these KPIs can be found in the `kpi_name` field.\n\n3. In different telemetry files, the timestamp units and cmdb_id formats may vary:\n\n- Metric: Timestamp units are in seconds (e.g., 1614787440).\n\n- Trace: Timestamp units are in milliseconds (e.g., 1614787199628).\n\n- Log: Timestamp units are in seconds (e.g., 1614787201).\n\n4. Please use the UTC+8 time zone in all analysis steps since system is deployed in China/Hong Kong/Singapore.", + "cand": "## POSSIBLE ROOT CAUSE REASONS:\n \n- high CPU usage\n- high memory usage \n- network latency \n- network packet loss\n- high disk I/O read usage \n- high disk space usage\n- high JVM CPU load \n- JVM Out of Memory (OOM) Heap\n\n## POSSIBLE ROOT CAUSE COMPONENTS:\n\n- apache01\n- apache02\n- Tomcat01\n- Tomcat02\n- Tomcat04\n- Tomcat03\n- MG01\n- MG02\n- IG01\n- IG02\n- Mysql01\n- Mysql02\n- Redis01\n- Redis02", + "candidates": { + "components": [ + "apache01", + "apache02", + "Tomcat01", + "Tomcat02", + "Tomcat04", + "Tomcat03", + "MG01", + "MG02", + "IG01", + "IG02", + "Mysql01", + "Mysql02", + "Redis01", + "Redis02" + ], + "reasons": [ + "high CPU usage", + "high memory usage", + "network latency", + "network packet loss", + "high disk I/O read usage", + "high disk space usage", + "high JVM CPU load", + "JVM Out of Memory (OOM) Heap" + ] + } + }, + "telecom": { + "schema": "## TELEMETRY DIRECTORY STRUCTURE:\n\n- You can access the telemetry directory in our microservices system: `/data/`\n\n- This directory contains subdirectories organized by a date (e.g., `/data/2020_04_11/`). \n\n- Within each date-specific directory, you’ll find these subdirectories: `metric` and `trace` (e.g., `/data/2020_04_11/metric/`).\n\n- The telemetry data in those subdirectories is stored in CSV format (e.g., `/data/2020_04_11/metric/metric_container.csv`).\n\n## DATA SCHEMA\n\n1. **Metric Files**:\n \n 1. `metric_app.csv`:\n\n ```csv\n serviceName,startTime,avg_time,num,succee_num,succee_rate\n osb_001,1586534400000,0.333,1,1,1.0\n ```\n\n 2. `metric_container.csv`:\n\n ```csv\n itemid,name,bomc_id,timestamp,value,cmdb_id\n 999999996381330,container_mem_used,ZJ-004-060,1586534423000,59.000000,docker_008\n ```\n\n 3. `metric_middleware.csv`:\n\n ```csv\n itemid,name,bomc_id,timestamp,value,cmdb_id\n 999999996508323,connected_clients,ZJ-005-024,1586534672000,25,redis_003\n ```\n\n 4. `metric_node.csv`:\n\n ```csv\n itemid,name,bomc_id,timestamp,value,cmdb_id\n 999999996487783,CPU_iowait_time,ZJ-001-010,1586534683000,0.022954,os_017\n ```\n\n 5. `metric_service.csv`:\n\n ```csv\n itemid,name,bomc_id,timestamp,value,cmdb_id\n 999999998650974,MEM_Total,ZJ-002-055,1586534694000,381.902264,db_003\n ```\n\n2. **Trace Files**:\n\n 1. `trace_span.csv`:\n\n ```csv\n callType,startTime,elapsedTime,success,traceId,id,pid,cmdb_id,dsName,serviceName\n JDBC,1586534400335,2.0,True,01df517164d1c0365586,407d617164d1c14f2613,6e02217164d1c14b2607,docker_006,db_003,\n LOCAL,1586534400331,6.0,True,01df517164d1c0365586,6e02217164d1c14b2607,8432217164d1c1442597,docker_006,db_003,local_method_017\n RemoteProcess,1586534400324,55.0,True,01df517164d1c0365586,8432217164d1c1442597,b755e17164d1c13f5066,docker_006,,csf_005\n FlyRemote,1586534400149,7.0,TRUE,fa1e817164d1c0375444,da74117164d1c0955052,b959f17164d1c08c5050,docker_003,,fly_remote_001\n OSB,1586534660846,376.0,True,d9c4817164d5baee6924,77d1117164d5baee6925,None,os_021,,osb_001\n ```\n\n{cand}\n\n## CLARIFICATION OF TELEMETRY DATA:\n\n1. This service system is a telecom database system.\n\n2. The `metric_app.csv` file only contains five KPIs: startTime, avg_time, num, succee_num, succee_rate. In contrast, other metric files record a variety of KPIs, such as CPU usage and memory usage. The specific names of these KPIs can be found in the `name` field.\n\n3. In all telemetry files, the timestamp units and cmdb_id formats remain consistent:\n\n- Metric: Timestamp units are in milliseconds (e.g., 1586534423000).\n\n- Trace: Timestamp units are in milliseconds (e.g., 1586534400335).\n\n4. Please use the UTC+8 time zone in all analysis steps since system is deployed in China/Hong Kong/Singapore.", + "cand": "## POSSIBLE ROOT CAUSE REASONS:\n \n- CPU fault\n- network delay\n- network loss \n- db connection limit \n- db close\n\n## POSSIBLE ROOT CAUSE COMPONENTS:\n\n(if the root cause is at the node level, i.e., the root cause is a specific node)\n\n- os_001\n- os_002\n- os_003\n- os_004\n- os_005\n- os_006\n- os_007\n- os_008\n- os_009\n- os_010\n- os_011\n- os_012\n- os_013\n- os_014\n- os_015\n- os_016\n- os_017\n- os_018\n- os_019\n- os_020\n- os_021\n- os_022\n\n(if the root cause is at the pod level, i.e., the root cause is a specific container)\n\n- docker_001\n- docker_002\n- docker_003\n- docker_004\n- docker_005\n- docker_006\n- docker_007\n- docker_008\n\n(if the root cause is at the service level, i.e., if all pods of a specific service are faulty, the root cause is the service itself)\n\n- db_001\n- db_002\n- db_003\n- db_004\n- db_005\n- db_006\n- db_007\n- db_008\n- db_009\n- db_010\n- db_011\n- db_012\n- db_013", + "candidates": { + "components": [ + "os_001", + "os_002", + "os_003", + "os_004", + "os_005", + "os_006", + "os_007", + "os_008", + "os_009", + "os_010", + "os_011", + "os_012", + "os_013", + "os_014", + "os_015", + "os_016", + "os_017", + "os_018", + "os_019", + "os_020", + "os_021", + "os_022", + "docker_001", + "docker_002", + "docker_003", + "docker_004", + "docker_005", + "docker_006", + "docker_007", + "docker_008", + "db_001", + "db_002", + "db_003", + "db_004", + "db_005", + "db_006", + "db_007", + "db_008", + "db_009", + "db_010", + "db_011", + "db_012", + "db_013" + ], + "reasons": ["CPU fault", "network delay", "network loss", "db connection limit", "db close"] + } + }, + "market-cloudbed-1": { + "schema": "## TELEMETRY DIRECTORY STRUCTURE:\n\n- You can access the telemetry directory of `cloudbed-1` in our microservices system: `/data/`.\n\n- This directory contains subdirectories organized by a date (e.g., `/data/2022_03_20/`). \n\n- Within each date-specific directory, you’ll find these subdirectories: `metric`, `trace`, and `log` (e.g., `/data/2022_03_20/metric/`).\n\n- The telemetry data in those subdirectories is stored in CSV format (e.g., `/data/2022_03_20/metric/metric_container.csv`).\n\n## DATA SCHEMA\n\n1. **Metric Files**:\n \n 1. `metric_container.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647781200,node-6.adservice2-0,container_fs_writes_MB./dev/vda,0.0\n ```\n\n 2. `metric_mesh.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647790380,cartservice-1.source.cartservice.redis-cart,istio_tcp_sent_bytes.-,1255.0\n ```\n\n 3. `metric_node.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647705600,node-1,system.cpu.iowait,0.31\n ```\n\n 4. `metric_runtime.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647730800,adservice.ts:8088,java_nio_BufferPool_TotalCapacity.direct,57343.0\n ```\n\n 5. `metric_service.csv`:\n\n ```csv\n service,timestamp,rr,sr,mrt,count\n adservice-grpc,1647716400,100.0,100.0,2.429508196728182,61\n ```\n\n2. **Trace Files**:\n\n 1. `trace_span.csv`:\n\n ```csv\n timestamp,cmdb_id,span_id,trace_id,duration,type,status_code,operation_name,parent_span\n 1647705600361,frontend-0,a652d4d10e9478fc,9451fd8fdf746a80687451dae4c4e984,49877,rpc,0,hipstershop.CheckoutService/PlaceOrder,952754a738a11675\n ```\n\n3. **Log Files**:\n\n 1. `log_proxy.csv`:\n\n ```csv\n log_id,timestamp,cmdb_id,log_name,value\n KN43pn8BmS57GQLkQUdP,1647761110,cartservice-1,log_cartservice-service_application,etCartAsync called with userId=3af80013-c2c1-4ae6-86d0-1d9d308e6f5b\n ```\n\n 2. `log_service.csv`:\n\n ```csv\n log_id,timestamp,cmdb_id,log_name,value\n GIvpon8BDiVcQfZwJ5a9,1647705660,currencyservice-0,log_currencyservice-service_application,\"severity: info, message: Getting supported currencies...\"\n ```\n\n{cand}\n\n## CLARIFICATION OF TELEMETRY DATA:\n\n1. This microservice system is a E-commerce platform which includes a failover mechanism, with each service deployed across four pods. In this system, a container (pod) can be deployed in different nodes. If the root cause component is a single pod of a specific service (e.g., node-1.adservice-0), the failure may not significantly impact the corresponding service metrics. In contrast, if the root cause component is a service itself (e.g., adservice), which means all pods of this service are faulty, the corresponding service metrics will be significantly impacted. Moreover, such fault could be propagate through the call chain, resulting in other service's metrics faulty. Note that `Pod` equals to `Container` in this system.\n\n2. The `metric_service.csv` file only contains four KPIs: rr, sr, mrt, and count. In contrast, other metric files record a variety of KPIs, such as CPU usage and memory usage. The specific names of these KPIs can be found in the `kpi_name` field.\n\n3. Note that the `cmdb_id` is the name of specific components, including nodes, pods, services, etc.\n\n- Metrics:\n - Runtime: The application name and port, e.g., `adservice.ts:8088`\n - Service: The service name and protocol, e.g., `adservic-grpc`\n - Container: The pod name combined with a node name, e.g., `node-1.adservice-0`\n - Node: The node name, e.g., `node-1`\n - Mesh: The service-to-service connection identifier within the mesh, e.g.,`cartservice-1.source.cartservice.redis-cart`\n\n- Traces: The pod name, e.g., `adservice-0`\n\n- Logs: The pod name, e.g., `adservice-0`\n\n4. In different telemetry files, the timestamp units and cmdb_id formats may vary:\n\n- Metric: Timestamp units are in seconds (e.g., 1647781200). cmdb_id varies by metric file:\n - In container metrics: `-x.-x` (e.g., `node-1.adservice-0`)\n - In node metrics: `-x` (e.g., `node-1`)\n - In service metrics: `-grpc` (e.g., `adservice-grpc`)\n\n- Trace: Timestamp units are in milliseconds (e.g., 1647705600361). cmdb_id is consistently `-x` (e.g., frontend-0).\n\n- Log: Timestamp units are in seconds (e.g., 1647705660). cmdb_id is consistently `-x` (e.g., currencyservice-0).\n\n5. Please use the UTC+8 time zone in all analysis steps since system is deployed in China/Hong Kong/Singapore.", + "cand": "## POSSIBLE ROOT CAUSE COMPONENTS:\n\n(if the root cause is at the node level, i.e., the root cause is a specific node)\n- node-1\n- node-2\n- node-3\n- node-4\n- node-5\n- node-6\n\n(if the root cause is at the pod level, i.e., the root cause is a specific container)\n\n- frontend-0\n- frontend-1\n- frontend-2\n- frontend2-0\n- shippingservice-0\n- shippingservice-1\n- shippingservice-2\n- shippingservice2-0\n- checkoutservice-0\n- checkoutservice-1\n- checkoutservice-2\n- checkoutservice2-0\n- currencyservice-0\n- currencyservice-1\n- currencyservice-2\n- currencyservice2-0\n- adservice-0\n- adservice-1\n- adservice-2\n- adservice2-0\n- emailservice-0\n- emailservice-1\n- emailservice-2\n- emailservice2-0\n- cartservice-0\n- cartservice-1\n- cartservice-2\n- cartservice2-0\n- productcatalogservice-0\n- productcatalogservice-1\n- productcatalogservice-2\n- productcatalogservice2-0\n- recommendationservice-0\n- recommendationservice-1\n- recommendationservice-2\n- recommendationservice2-0\n- paymentservice-0\n- paymentservice-1\n- paymentservice-2\n- paymentservice2-0\n\n(if the root cause is at the service level, i.e., if all pods of a specific service are faulty, the root cause is the service itself)\n\n- frontend\n- shippingservice\n- checkoutservice\n- currencyservice\n- adservice\n- emailservice\n- cartservice\n- productcatalogservice\n- recommendationservice\n- paymentservice\n\n## POSSIBLE ROOT CAUSE REASONS:\n\n- container CPU load\n- container memory load\n- container network packet retransmission \n- container network packet corruption\n- container network latency \n- container packet loss \n- container process termination\n- container read I/O load\n- container write I/O load\n- node CPU load\n- node CPU spike\n- node memory consumption\n- node disk read I/O consumption \n- node disk write I/O consumption \n- node disk space consumption", + "candidates": { + "components": [ + "node-1", + "node-2", + "node-3", + "node-4", + "node-5", + "node-6", + "frontend-0", + "frontend-1", + "frontend-2", + "frontend2-0", + "shippingservice-0", + "shippingservice-1", + "shippingservice-2", + "shippingservice2-0", + "checkoutservice-0", + "checkoutservice-1", + "checkoutservice-2", + "checkoutservice2-0", + "currencyservice-0", + "currencyservice-1", + "currencyservice-2", + "currencyservice2-0", + "adservice-0", + "adservice-1", + "adservice-2", + "adservice2-0", + "emailservice-0", + "emailservice-1", + "emailservice-2", + "emailservice2-0", + "cartservice-0", + "cartservice-1", + "cartservice-2", + "cartservice2-0", + "productcatalogservice-0", + "productcatalogservice-1", + "productcatalogservice-2", + "productcatalogservice2-0", + "recommendationservice-0", + "recommendationservice-1", + "recommendationservice-2", + "recommendationservice2-0", + "paymentservice-0", + "paymentservice-1", + "paymentservice-2", + "paymentservice2-0", + "frontend", + "shippingservice", + "checkoutservice", + "currencyservice", + "adservice", + "emailservice", + "cartservice", + "productcatalogservice", + "recommendationservice", + "paymentservice" + ], + "reasons": [ + "container CPU load", + "container memory load", + "container network packet retransmission", + "container network packet corruption", + "container network latency", + "container packet loss", + "container process termination", + "container read I/O load", + "container write I/O load", + "node CPU load", + "node CPU spike", + "node memory consumption", + "node disk read I/O consumption", + "node disk write I/O consumption", + "node disk space consumption" + ] + } + }, + "market-cloudbed-2": { + "schema": "## TELEMETRY DIRECTORY STRUCTURE:\n\n- You can access the telemetry directory of `cloudbed-2` in our microservices system: `/data/`.\n\n- This directory contains subdirectories organized by a date (e.g., `/data/2022_03_20/`). \n\n- Within each date-specific directory, you’ll find these subdirectories: `metric`, `trace`, and `log` (e.g., `/data/2022_03_20/metric/`).\n\n- The telemetry data in those subdirectories is stored in CSV format (e.g., `/data/2022_03_20/metric/metric_container.csv`).\n\n## DATA SCHEMA\n\n1. **Metric Files**:\n \n 1. `metric_container.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647781200,node-6.adservice2-0,container_fs_writes_MB./dev/vda,0.0\n ```\n\n 2. `metric_mesh.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647790380,cartservice-1.source.cartservice.redis-cart,istio_tcp_sent_bytes.-,1255.0\n ```\n\n 3. `metric_node.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647705600,node-1,system.cpu.iowait,0.31\n ```\n\n 4. `metric_runtime.csv`:\n\n ```csv\n timestamp,cmdb_id,kpi_name,value\n 1647730800,adservice.ts:8088,java_nio_BufferPool_TotalCapacity.direct,57343.0\n ```\n\n 5. `metric_service.csv`:\n\n ```csv\n service,timestamp,rr,sr,mrt,count\n adservice-grpc,1647716400,100.0,100.0,2.429508196728182,61\n ```\n\n2. **Trace Files**:\n\n 1. `trace_span.csv`:\n\n ```csv\n timestamp,cmdb_id,span_id,trace_id,duration,type,status_code,operation_name,parent_span\n 1647705600361,frontend-0,a652d4d10e9478fc,9451fd8fdf746a80687451dae4c4e984,49877,rpc,0,hipstershop.CheckoutService/PlaceOrder,952754a738a11675\n ```\n\n3. **Log Files**:\n\n 1. `log_proxy.csv`:\n\n ```csv\n log_id,timestamp,cmdb_id,log_name,value\n KN43pn8BmS57GQLkQUdP,1647761110,cartservice-1,log_cartservice-service_application,etCartAsync called with userId=3af80013-c2c1-4ae6-86d0-1d9d308e6f5b\n ```\n\n 2. `log_service.csv`:\n\n ```csv\n log_id,timestamp,cmdb_id,log_name,value\n GIvpon8BDiVcQfZwJ5a9,1647705660,currencyservice-0,log_currencyservice-service_application,\"severity: info, message: Getting supported currencies...\"\n ```\n\n{cand}\n\n## CLARIFICATION OF TELEMETRY DATA:\n\n1. This microservice system is a E-commerce platform which includes a failover mechanism, with each service deployed across four pods. In this system, a container (pod) can be deployed in different nodes. If the root cause component is a single pod of a specific service (e.g., node-1.adservice-0), the failure may not significantly impact the corresponding service metrics. In contrast, if the root cause component is a service itself (e.g., adservice), which means all pods of this service are faulty, the corresponding service metrics will be significantly impacted. Moreover, such fault could be propagate through the call chain, resulting in other service's metrics faulty. Note that `Pod` equals to `Container` in this system.\n\n2. The `metric_service.csv` file only contains four KPIs: rr, sr, mrt, and count. In contrast, other metric files record a variety of KPIs, such as CPU usage and memory usage. The specific names of these KPIs can be found in the `kpi_name` field.\n\n3. Note that the `cmdb_id` is the name of specific components, including nodes, pods, services, etc.\n\n- Metrics:\n - Runtime: The application name and port, e.g., `adservice.ts:8088`\n - Service: The service name and protocol, e.g., `adservic-grpc`\n - Container: The pod name combined with a node name, e.g., `node-1.adservice-0`\n - Node: The node name, e.g., `node-1`\n - Mesh: The service-to-service connection identifier within the mesh, e.g.,`cartservice-1.source.cartservice.redis-cart`\n\n- Traces: The pod name, e.g., `adservice-0`\n\n- Logs: The pod name, e.g., `adservice-0`\n\n4. In different telemetry files, the timestamp units and cmdb_id formats may vary:\n\n- Metric: Timestamp units are in seconds (e.g., 1647781200). cmdb_id varies by metric file:\n - In container metrics: `-x.-x` (e.g., `node-1.adservice-0`)\n - In node metrics: `-x` (e.g., `node-1`)\n - In service metrics: `-grpc` (e.g., `adservice-grpc`)\n\n- Trace: Timestamp units are in milliseconds (e.g., 1647705600361). cmdb_id is consistently `-x` (e.g., frontend-0).\n\n- Log: Timestamp units are in seconds (e.g., 1647705660). cmdb_id is consistently `-x` (e.g., currencyservice-0).\n\n5. Please use the UTC+8 time zone in all analysis steps since system is deployed in China/Hong Kong/Singapore.", + "cand": "## POSSIBLE ROOT CAUSE COMPONENTS:\n\n(if the root cause is at the node level, i.e., the root cause is a specific node)\n- node-1\n- node-2\n- node-3\n- node-4\n- node-5\n- node-6\n\n(if the root cause is at the pod level, i.e., the root cause is a specific container)\n\n- frontend-0\n- frontend-1\n- frontend-2\n- frontend2-0\n- shippingservice-0\n- shippingservice-1\n- shippingservice-2\n- shippingservice2-0\n- checkoutservice-0\n- checkoutservice-1\n- checkoutservice-2\n- checkoutservice2-0\n- currencyservice-0\n- currencyservice-1\n- currencyservice-2\n- currencyservice2-0\n- adservice-0\n- adservice-1\n- adservice-2\n- adservice2-0\n- emailservice-0\n- emailservice-1\n- emailservice-2\n- emailservice2-0\n- cartservice-0\n- cartservice-1\n- cartservice-2\n- cartservice2-0\n- productcatalogservice-0\n- productcatalogservice-1\n- productcatalogservice-2\n- productcatalogservice2-0\n- recommendationservice-0\n- recommendationservice-1\n- recommendationservice-2\n- recommendationservice2-0\n- paymentservice-0\n- paymentservice-1\n- paymentservice-2\n- paymentservice2-0\n\n(if the root cause is at the service level, i.e., if all pods of a specific service are faulty, the root cause is the service itself)\n\n- frontend\n- shippingservice\n- checkoutservice\n- currencyservice\n- adservice\n- emailservice\n- cartservice\n- productcatalogservice\n- recommendationservice\n- paymentservice\n\n## POSSIBLE ROOT CAUSE REASONS:\n\n- container CPU load\n- container memory load\n- container network packet retransmission \n- container network packet corruption\n- container network latency \n- container packet loss \n- container process termination\n- container read I/O load\n- container write I/O load\n- node CPU load\n- node CPU spike\n- node memory consumption\n- node disk read I/O consumption \n- node disk write I/O consumption \n- node disk space consumption", + "candidates": { + "components": [ + "node-1", + "node-2", + "node-3", + "node-4", + "node-5", + "node-6", + "frontend-0", + "frontend-1", + "frontend-2", + "frontend2-0", + "shippingservice-0", + "shippingservice-1", + "shippingservice-2", + "shippingservice2-0", + "checkoutservice-0", + "checkoutservice-1", + "checkoutservice-2", + "checkoutservice2-0", + "currencyservice-0", + "currencyservice-1", + "currencyservice-2", + "currencyservice2-0", + "adservice-0", + "adservice-1", + "adservice-2", + "adservice2-0", + "emailservice-0", + "emailservice-1", + "emailservice-2", + "emailservice2-0", + "cartservice-0", + "cartservice-1", + "cartservice-2", + "cartservice2-0", + "productcatalogservice-0", + "productcatalogservice-1", + "productcatalogservice-2", + "productcatalogservice2-0", + "recommendationservice-0", + "recommendationservice-1", + "recommendationservice-2", + "recommendationservice2-0", + "paymentservice-0", + "paymentservice-1", + "paymentservice-2", + "paymentservice2-0", + "frontend", + "shippingservice", + "checkoutservice", + "currencyservice", + "adservice", + "emailservice", + "cartservice", + "productcatalogservice", + "recommendationservice", + "paymentservice" + ], + "reasons": [ + "container CPU load", + "container memory load", + "container network packet retransmission", + "container network packet corruption", + "container network latency", + "container packet loss", + "container process termination", + "container read I/O load", + "container write I/O load", + "node CPU load", + "node CPU spike", + "node memory consumption", + "node disk read I/O consumption", + "node disk write I/O consumption", + "node disk space consumption" + ] + } + } + } +} diff --git a/agents/react/scripts/extract-background.ts b/agents/react/scripts/extract-background.ts new file mode 100644 index 0000000..ccd56f9 --- /dev/null +++ b/agents/react/scripts/extract-background.ts @@ -0,0 +1,191 @@ +/** + * `pnpm --filter @evorca/agent-react extract-background` writes + * `background.json`, the domain knowledge that RCA-Agent, OpenRCA's published + * baseline, gives its model. It reads RCA-Agent's prompts at the pinned OpenRCA + * commit, which it checks out in `.cache/` at the repository root. + * + * - From `prompt/basic_prompt_.py`, each scenario's `schema`, which + * describes the telemetry, and `cand`, which lists the candidates. The schema + * keeps a `{cand}` placeholder, because `/task.json` decides whether the + * agent sees candidates. Its paths name `/data/`, where the container mounts + * the scenario's telemetry, instead of `dataset//telemetry/`. The + * Market schema names the one cloudbed whose telemetry the container holds. + * - From `prompt/agent_prompt.py`, the `rules` of failure diagnosis, without + * four rules that only fit RCA-Agent's split into a controller that writes + * no code and an executor that runs it: no code in the controller's + * response, no timestamp conversion by the controller, no plots from the + * executor, and no files or bash. One rule names the executor, and here it + * does not. + * - From `executor.py`, the two rules of the executor's code that concern the + * data rather than the IPython kernel: thresholds before filtering, and UTC+8. + * + * The script fails if any text it changes is not where it expects, and if a + * scenario's candidates differ from `@evorca/openrca`'s `candidates.json`. + * + * @module + */ +import { isDeepStrictEqual } from "node:util"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { format, resolveConfig } from "prettier"; +import { openRcaCandidates } from "../../../packages/openrca/src/candidates.ts"; +import { openRcaCommit } from "../../../packages/openrca/src/pinned.ts"; +import { openRcaScenarios } from "../../../packages/openrca/src/scenarios.ts"; +import { ensureOpenRca } from "../../../packages/openrca/src/scoring/evaluator.ts"; +import type { Background, ScenarioBackground } from "../src/prompt.ts"; +import type { Candidates } from "../src/task.ts"; + +const cacheDir = fileURLToPath(new URL("../../../.cache/", import.meta.url)); +const outPath = fileURLToPath(new URL("../background.json", import.meta.url)); +const agentDir = join(await ensureOpenRca(cacheDir), "rca", "baseline", "rca_agent"); +const source = (file: string) => readFile(join(agentDir, file), "utf8"); + +const knownCandidates: Readonly> = openRcaCandidates; + +const droppedRules: Readonly> = { + 1: "**DO NOT include any programming language (Python) in your response.**", + 2: "**DO NOT convert the timestamp to datetime or convert the datetime to timestamp by yourself.**", + 4: "**DO NOT visualize the data or draw pictures or graphs via Python.**", + 5: "**DO NOT save anything in the local file system.**", +}; +const keptCodeRules = [9, 10]; + +const rules = [ + diagnosisRules(pythonString(await source("prompt/agent_prompt.py"), "rules")), + codeRules(pythonString(await source("executor.py"), "rule")), +].join("\n\n"); + +const scenarios = Object.fromEntries( + await Array.fromAsync( + Object.entries(openRcaScenarios), + async ([scenario, system]): Promise<[string, ScenarioBackground]> => { + const prompt = await source(`prompt/basic_prompt_${system.split("/")[0]}.py`); + const cand = pythonString(prompt, "cand"); + const candidates = readCandidates(cand); + if (!isDeepStrictEqual(candidates, knownCandidates[scenario])) { + throw new Error(`The candidates of ${scenario} differ from candidates.json`); + } + return [ + scenario, + { schema: schemaFor(system, pythonString(prompt, "schema")), cand, candidates }, + ]; + }, + ), +); + +const background: Background = { + source: `OpenRCA ${openRcaCommit}: rca/baseline/rca_agent/prompt/ and rca/baseline/rca_agent/executor.py`, + rules, + scenarios, +}; +const json = JSON.stringify(background); +await writeFile( + outPath, + await format(json, { ...(await resolveConfig(outPath)), filepath: outPath }), +); +console.log(`Wrote ${outPath}`); + +function pythonString(source: string, name: string): string { + const value = new RegExp(`^${name} = f?"""(.*?)"""`, "ms").exec(source)?.[1]; + if (value === undefined) throw new Error(`No string ${name}`); + if (value.includes("\\")) throw new Error(`The string ${name} has an escape`); + return value; +} + +function schemaFor(system: string, schema: string): string { + if ((schema.match(/[{}]/g) ?? []).join("") !== "{}" || !schema.includes("{cand}")) { + throw new Error(`The schema of ${system} has braces other than {cand}`); + } + const cloudbed = /^Market\/(cloudbed-\d)$/.exec(system)?.[1]; + const named = + cloudbed === undefined + ? schema + : replaceOnce( + schema, + "- You can access the telemetry directories of two cloudbed (i.e., `cloudbed-1` and `cloudbed-2`) in our microservices system: `dataset/Market/cloudbed-1/telemetry/` and `dataset/Market/cloudbed-2/telemetry/`.", + `- You can access the telemetry directory of \`${cloudbed}\` in our microservices system: \`/data/\`.`, + ); + const telemetry = + cloudbed === undefined + ? `dataset/${system}/telemetry/` + : "dataset/Market/cloudbed-1/telemetry/"; + if (!named.includes(telemetry)) + throw new Error(`The schema of ${system} never names ${telemetry}`); + const result = named.replaceAll(telemetry, "/data/"); + if (result.includes("dataset/")) throw new Error(`The schema of ${system} still names dataset/`); + return result; +} + +function diagnosisRules(rules: string): string { + const [should, shouldNot] = splitOnce(rules, "\n\nWhat you SHOULD NOT do:\n\n"); + const items = shouldNot.split("\n").map((line, i) => { + const prefix = `${i + 1}. `; + if (!line.startsWith(prefix)) throw new Error(`Rule ${i + 1} does not start with "${prefix}"`); + return line.slice(prefix.length); + }); + for (const [number, start] of Object.entries(droppedRules)) { + if (!items[Number(number) - 1]?.startsWith(start)) + throw new Error(`Rule ${number} is not "${start}"`); + } + const kept = items.filter((_, i) => droppedRules[i + 1] === undefined); + return [ + replaceOnce( + should, + "when Executor's retrieval result is empty", + "when a retrieval result is empty", + ), + "What you SHOULD NOT do:", + kept.map((item, i) => `${i + 1}. ${item}`).join("\n"), + ].join("\n\n"); +} + +function codeRules(rule: string): string { + const [heading, list] = splitOnce(rule.trim(), "\n\n"); + const items = new Map( + list.split("\n").map((line) => { + const match = /^(\d+)\. (.*)$/.exec(line); + if (match === null) + throw new Error(`Executor rule line does not start with a number: ${line}`); + return [Number(match[1]), match[2] ?? ""]; + }), + ); + return [ + heading, + keptCodeRules + .map((number, i) => { + const item = items.get(number); + if (item === undefined) throw new Error(`No executor rule ${number}`); + return `${i + 1}. ${item.trim()}`; + }) + .join("\n"), + ].join("\n\n"); +} + +function readCandidates(cand: string): Candidates { + const sections = new Map(); + let section: string[] = []; + for (const line of cand.split("\n")) { + const heading = /^## POSSIBLE ROOT CAUSE (COMPONENTS|REASONS):/.exec(line)?.[1]; + if (heading !== undefined) { + section = []; + sections.set(heading, section); + } else if (line.startsWith("- ")) { + section.push(line.slice(2).trim()); + } + } + return { components: sections.get("COMPONENTS") ?? [], reasons: sections.get("REASONS") ?? [] }; +} + +function replaceOnce(text: string, from: string, to: string): string { + const [before, after] = splitOnce(text, from); + return `${before}${to}${after}`; +} + +function splitOnce(text: string, separator: string): [string, string] { + const at = text.indexOf(separator); + if (at < 0 || text.indexOf(separator, at + 1) >= 0) { + throw new Error(`Expected exactly one ${JSON.stringify(separator.slice(0, 60))}`); + } + return [text.slice(0, at), text.slice(at + separator.length)]; +} diff --git a/agents/react/src/prompt.ts b/agents/react/src/prompt.ts new file mode 100644 index 0000000..e02f1ef --- /dev/null +++ b/agents/react/src/prompt.ts @@ -0,0 +1,141 @@ +/** + * The agent's prompt for a task: the system prompt, the tools, and the first + * user message. + * + * The domain knowledge in the system prompt is RCA-Agent's, from + * `background.json`, which `scripts/extract-background.ts` writes from + * OpenRCA's prompts. The rest of the system prompt describes the container, + * the tools, and how the answer is scored. It names no component, KPI, or + * failure beyond what RCA-Agent's prompts name. + * + * The system prompt and the tools depend only on the scenario, its candidates, + * and the limits, so every request of a task, and every task of a scenario, + * starts with the same bytes, which prompt caching needs. The issue goes in the + * first user message. + * + * @module + */ +import { createHash } from "node:crypto"; +import type Anthropic from "@anthropic-ai/sdk"; +import { submitAnswerTool } from "./answer.ts"; +import type { Candidates, Task } from "./task.ts"; + +/** + * RCA-Agent's domain knowledge for one scenario. `schema` describes the + * telemetry and holds a `{cand}` placeholder where RCA-Agent's prompt lists + * the candidates. `cand` is that list as RCA-Agent writes it, and `candidates` + * holds its entries. + */ +export type ScenarioBackground = { + readonly schema: string; + readonly cand: string; + readonly candidates: Candidates; +}; + +/** RCA-Agent's domain knowledge: its rules, and the background of each scenario. */ +export type Background = { + readonly source: string; + readonly rules: string; + readonly scenarios: Readonly>; +}; + +/** The limits that the agent works within, which the system prompt states. */ +export type Limits = { + readonly maxSteps: number; + readonly commandTimeoutSeconds: number; + readonly outputChars: number; +}; + +/** The parts of a request that stay the same for a whole task. */ +export type Prompt = { + readonly system: string; + readonly tools: readonly Anthropic.Messages.ToolUnion[]; + readonly instruction: string; + readonly sha256: string; +}; + +/** The name of the bash tool. */ +export const bash = "bash"; + +const placeholder = "{cand}"; + +/** + * Builds the prompt for `task`. With candidates equal to RCA-Agent's, the + * system prompt lists them as RCA-Agent's prompt does. With other candidates, + * it lists those, and without candidates it lists none. + * + * @throws Error if `background` has no scenario named like the task's. + */ +export function buildPrompt(task: Task, background: Background, limits: Limits): Prompt { + const scenario = background.scenarios[task.scenario]; + if (scenario === undefined) throw new Error(`No background for scenario ${task.scenario}`); + const cand = candidatesSection(task.candidates, scenario); + const domain = + cand === undefined + ? scenario.schema.replace(`\n\n${placeholder}\n\n`, "\n\n") + : scenario.schema.replace(placeholder, cand); + const system = [ + intro, + environment(limits), + "There is some domain knowledge for you:", + domain, + background.rules, + answer(task.candidates !== undefined), + ].join("\n\n"); + const tools = [{ type: "bash_20250124", name: bash } as const, submitAnswerTool(task.candidates)]; + const instruction = `The issue you are going to solve is:\n\n${task.instruction}\n\nSolve the issue step-by-step.`; + const sha256 = createHash("sha256").update(JSON.stringify({ system, tools })).digest("hex"); + return { system, tools, instruction, sha256 }; +} + +function candidatesSection( + candidates: Candidates | undefined, + scenario: ScenarioBackground, +): string | undefined { + if (candidates === undefined) return undefined; + if (sameCandidates(candidates, scenario.candidates)) return scenario.cand; + const list = (names: readonly string[]) => names.map((name) => `- ${name}`).join("\n"); + return `## POSSIBLE ROOT CAUSE COMPONENTS:\n\n${list(candidates.components)}\n\n## POSSIBLE ROOT CAUSE REASONS:\n\n${list(candidates.reasons)}`; +} + +function sameCandidates(a: Candidates, b: Candidates): boolean { + const same = (x: readonly string[], y: readonly string[]) => + x.length === y.length && x.every((name, i) => name === y[i]); + return same(a.components, b.components) && same(a.reasons, b.reasons); +} + +const intro = + "You are a DevOps assistant for failure diagnosis. To solve the given issue, you analyze the telemetry files of the target system by running commands with the bash tool, and you approximate the answer step-by-step from the results. When you have found the root causes, you report them with the submit_answer tool."; + +function environment(limits: Limits): string { + return `## YOUR ENVIRONMENT + +- The bash tool runs commands in a Linux container without network access. The bash session keeps its working directory, variables, and files between calls. Each \`python3\` call is a new process, so Python variables do not persist: save results that you want to reuse to files in \`/tmp/\`. +- The telemetry directory is \`/data/\`, which is read-only. \`/tmp/\` is writable, and the bash session starts there. Files in \`/tmp/\` count toward your memory limit. +- Python 3 is installed with pandas, pytz, and DuckDB. +- Some telemetry files are larger than 1 GB, more than fits in your memory. Query large CSV files with DuckDB, which streams them from disk, and load only the rows or aggregates that you need into pandas. For example: \`python3 -c "import duckdb; duckdb.sql(\\"SELECT cmdb_id, count(*) FROM read_csv('/data//metric/.csv') GROUP BY cmdb_id\\").show()"\`. DuckDB sets its memory limit and its threads from the container's limits, and it spills to \`.tmp/\` in the working directory, so run it from \`/tmp/\`. +- A command that runs longer than ${limits.commandTimeoutSeconds} seconds is stopped, and the bash session restarts: its working directory and variables are lost, but files remain. +- Output longer than ${limits.outputChars} characters is cut in the middle, so print only what you need. +- You can make at most ${limits.maxSteps} tool calls. After the last one, you can only submit your answer.`; +} + +function answer(withCandidates: boolean): string { + const select = withCandidates + ? [ + "- Select each root cause component and reason from the possible root cause components and reasons above.", + ] + : []; + return [ + "## YOUR ANSWER", + "", + "Call submit_answer once, when you have found the root causes. It ends your investigation.", + "", + "- Give one root cause per failure, as many as the issue says occurred.", + "- Give only the fields that the issue asks for, and set the others to null.", + "- Give the root cause occurrence datetime in UTC+8, in the format `YYYY-MM-DD HH:MM:SS`.", + ...select, + "- Do not be too conservative in selecting the root cause components and reasons. Be decisive to infer a possible answer based on your current observation.", + "", + "The answer is scored field by field. A datetime is correct within one minute of the true root cause occurrence time, and a component or a reason is correct only if it is exactly the true one. If the number of root causes differs from the number of failures, every field scores zero. The order of the root causes does not matter.", + ].join("\n"); +} diff --git a/agents/react/test/prompt.test.ts b/agents/react/test/prompt.test.ts new file mode 100644 index 0000000..1b0e3ef --- /dev/null +++ b/agents/react/test/prompt.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { openRcaCandidates } from "../../../packages/openrca/src/candidates.ts"; +import json from "../background.json" with { type: "json" }; +import { buildPrompt, type Background, type Limits } from "../src/prompt.ts"; +import type { Task } from "../src/task.ts"; + +const background: Background = json; +const limits: Limits = { maxSteps: 50, commandTimeoutSeconds: 120, outputChars: 16_000 }; +const candidates: Readonly> = openRcaCandidates; + +const task: Task = { + scenario: "telecom", + row: 0, + taskIndex: "task_7", + instruction: "On April 11, 2020, between 00:00 and 00:30, a failure occurred.", + candidates: openRcaCandidates.telecom, +}; + +describe("background.json", () => { + it("has the candidates of candidates.json for every scenario", () => { + expect(Object.keys(background.scenarios)).toEqual(Object.keys(openRcaCandidates)); + for (const [scenario, { candidates: listed }] of Object.entries(background.scenarios)) { + expect(listed).toEqual(candidates[scenario]); + } + }); + + it("points every schema at /data/ and keeps one candidates placeholder", () => { + for (const { schema } of Object.values(background.scenarios)) { + expect(schema).toContain("`/data/"); + expect(schema).not.toContain("dataset/"); + expect(schema.split("{cand}")).toHaveLength(2); + } + expect(background.scenarios["market-cloudbed-2"]?.schema).toContain( + "You can access the telemetry directory of `cloudbed-2` in our microservices system: `/data/`.", + ); + }); + + it("keeps no rule that forbids writing code, files, or bash", () => { + expect(background.rules).not.toContain("DO NOT include any programming language"); + expect(background.rules).not.toContain("DO NOT save anything in the local file system"); + expect(background.rules).not.toContain("Executor"); + expect(background.rules).toContain("pytz.timezone('Asia/Shanghai')"); + }); +}); + +describe("buildPrompt", () => { + it("lists RCA-Agent's candidates as RCA-Agent's prompt does", () => { + const { system } = buildPrompt(task, background, limits); + expect(system).toContain(background.scenarios.telecom?.cand); + expect(system).toContain( + "(if the root cause is at the node level, i.e., the root cause is a specific node)", + ); + expect(system).toContain( + "Select each root cause component and reason from the possible root cause", + ); + expect(system).not.toContain("{cand}"); + }); + + it("lists other candidates as they are, and none without candidates", () => { + const other = buildPrompt( + { ...task, candidates: { components: ["os_001", "os_002"], reasons: ["CPU fault"] } }, + background, + limits, + ); + expect(other.system).toContain( + "## POSSIBLE ROOT CAUSE COMPONENTS:\n\n- os_001\n- os_002\n\n## POSSIBLE ROOT CAUSE REASONS:\n\n- CPU fault", + ); + const { candidates: _, ...blind } = task; + const none = buildPrompt(blind, background, limits); + expect(none.system).not.toContain("POSSIBLE ROOT CAUSE"); + expect(none.system).not.toContain("{cand}"); + expect(none.system).not.toContain("Select each root cause component"); + }); + + it("gives every task of a scenario the same system prompt and tools", () => { + const first = buildPrompt(task, background, limits); + const second = buildPrompt( + { ...task, row: 9, taskIndex: "task_2", instruction: "Another issue." }, + background, + limits, + ); + expect(second.system).toBe(first.system); + expect(JSON.stringify(second.tools)).toBe(JSON.stringify(first.tools)); + expect(second.sha256).toBe(first.sha256); + expect(second.instruction).not.toBe(first.instruction); + expect(first.instruction).toBe( + `The issue you are going to solve is:\n\n${task.instruction}\n\nSolve the issue step-by-step.`, + ); + }); + + it("states the limits and declares the bash tool by its type", () => { + const { system, tools } = buildPrompt(task, background, { ...limits, maxSteps: 7 }); + expect(system).toContain("You can make at most 7 tool calls."); + expect(system).toContain("longer than 120 seconds"); + expect(system).toContain("longer than 16000 characters"); + expect(tools[0]).toEqual({ type: "bash_20250124", name: "bash" }); + }); + + it("throws for a scenario without a background", () => { + expect(() => buildPrompt({ ...task, scenario: "nowhere" }, background, limits)).toThrow(); + }); +}); From 9bad8feb4ef551449ae3eca61911370aefe512af Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:42:21 +0200 Subject: [PATCH 4/6] feat(agent-react): add the ReAct loop investigate runs the model until it submits an answer. It wraps up at the step cap, the soft budget, or the soft deadline, and ends on a refusal or an API error with the best answer so far. runTask writes prediction.json, trajectory.jsonl, and summary.json. --- agents/react/src/loop.ts | 360 +++++++++++++++++++++++++ agents/react/src/main.ts | 247 +++++++++++++++++ agents/react/src/usage.ts | 61 +++++ agents/react/test/loop.test.ts | 475 +++++++++++++++++++++++++++++++++ agents/react/test/mock-api.ts | 140 ++++++++++ 5 files changed, 1283 insertions(+) create mode 100644 agents/react/src/loop.ts create mode 100644 agents/react/src/main.ts create mode 100644 agents/react/src/usage.ts create mode 100644 agents/react/test/loop.test.ts create mode 100644 agents/react/test/mock-api.ts diff --git a/agents/react/src/loop.ts b/agents/react/src/loop.ts new file mode 100644 index 0000000..698f72e --- /dev/null +++ b/agents/react/src/loop.ts @@ -0,0 +1,360 @@ +/** + * The ReAct loop: the model reasons, calls a tool, reads the result, and + * repeats until it submits an answer. + * + * Each request sends the same system prompt and tools and appends to the same + * history, so the cache serves every earlier turn. The system prompt carries a + * cache breakpoint of its own, which the next task of the scenario reads, and + * the request's automatic breakpoint follows the end of the history. + * + * The loop is in one of two phases. While investigating, the model may run + * commands. A model that ends its turn without a tool call is asked once to + * submit, and the loop gives up the second time. When the model has made its + * last allowed tool call, has spent the soft budget, or is past the soft + * deadline, the loop wraps up: it asks for the answer and refuses commands, + * and it gives up after two more responses without a valid answer. + * + * A response cut off at `max_tokens` is discarded and asked again with the + * next larger limit, because its last block may be an incomplete tool call. A + * refusal ends the loop. So does an API error that the SDK gives up on, such + * as the model proxy's `evorca_budget_exhausted` 403, after which no model is + * left to ask. Every outcome keeps the best answer so far: the submitted one, + * or the valid fields of the last rejected one that had any. + * + * @module + */ +import Anthropic, { APIError, PermissionDeniedError } from "@anthropic-ai/sdk"; +import { readAnswer, submitAnswer, type Cause } from "./answer.ts"; +import { describeResult, type BashSession, type CommandEnd } from "./bash.ts"; +import { bash, type Limits, type Prompt } from "./prompt.ts"; +import { isRecord, type Task } from "./task.ts"; +import { addUsage, costUsd, noUsage, type Prices, type Usage } from "./usage.ts"; + +/** The effort levels of `output_config.effort`. */ +export const efforts = ["low", "medium", "high", "xhigh", "max"] as const; + +/** One effort level. */ +export type Effort = (typeof efforts)[number]; + +/** + * How the loop calls the model. `maxTokens` lists the output limits to try in + * order, the next one after a response stops at `max_tokens`. `softBudgetUsd` + * and `deadline`, a time in milliseconds since the epoch, start the wrap-up. + */ +export type Settings = { + readonly model: string; + readonly effort: Effort; + readonly prices: Prices; + readonly maxTokens: readonly [number, ...number[]]; + readonly softBudgetUsd?: number; + readonly deadline?: number; +}; + +/** Why the loop began to wrap up. */ +export type WrapUpReason = "steps" | "budget" | "time"; + +/** Why the loop ended. */ +export type StopCause = + | "submitted" + | "no-answer" + | "budget-exhausted" + | "refusal" + | "max-tokens" + | "context-window" + | "api-error"; + +/** An API error that ended the loop. */ +export type ApiErrorSummary = { + readonly status: number | undefined; + readonly errorType: string | null; + readonly message: string; +}; + +/** + * How the loop ended. `causes` is the submitted answer, or else the valid + * fields of the last rejected one that had any, or else empty. `steps` counts + * the tool calls that ran, and `turns` the model responses, discarded ones + * included. + */ +export type Outcome = { + readonly stopCause: StopCause; + readonly wrapUp: WrapUpReason | null; + readonly answer: "submitted" | "partial" | "none"; + readonly causes: readonly Cause[]; + readonly steps: number; + readonly turns: number; + readonly usage: Usage; + readonly costUsd: number; + readonly error?: ApiErrorSummary; + readonly refusal?: Anthropic.Messages.RefusalStopDetails | null; +}; + +/** One entry of the trajectory, without its timestamp. */ +export type Event = + | { readonly type: "user"; readonly text: string } + | { + readonly type: "response"; + readonly turn: number; + readonly maxTokens: number; + readonly seconds: number; + readonly costUsd: number; + readonly message: Anthropic.Message; + } + | { + readonly type: "tool"; + readonly step: number | null; + readonly id: string; + readonly name: string; + readonly input: unknown; + readonly output: string; + readonly isError: boolean; + readonly seconds: number; + readonly end?: CommandEnd; + } + | ({ readonly type: "api-error" } & ApiErrorSummary); + +/** What {@link investigate} works with. `now` returns milliseconds since the epoch. */ +export type LoopOptions = { + readonly client: Anthropic; + readonly prompt: Prompt; + readonly task: Task; + readonly shell: BashSession; + readonly settings: Settings; + readonly limits: Limits; + readonly log: (event: Event) => void; + readonly now?: () => number; +}; + +type Phase = + | { readonly kind: "investigating"; readonly nudged: boolean } + | { readonly kind: "wrapping-up"; readonly reason: WrapUpReason; readonly responsesLeft: number }; + +type ToolResult = { readonly output: string; readonly isError: boolean; readonly end?: CommandEnd }; + +const budgetExhausted = "evorca_budget_exhausted"; +const wrapUpResponses = 2; + +const nudge = + "You ended your turn without calling submit_answer. Call submit_answer now with your answer, or continue the investigation with bash if you need more evidence."; + +/** Runs the loop for one task until it ends, and returns how it ended. */ +export async function investigate(options: LoopOptions): Promise { + const { client, prompt, task, shell, settings, limits, log } = options; + const now = options.now ?? Date.now; + const messages: Anthropic.MessageParam[] = []; + let phase: Phase = { kind: "investigating", nudged: false }; + let usage = noUsage; + let steps = 0; + let turns = 0; + let partial: readonly Cause[] = []; + + const send = (results: Anthropic.ToolResultBlockParam[], text?: string) => { + if (text !== undefined) log({ type: "user", text }); + const blocks = text === undefined ? results : [...results, { type: "text" as const, text }]; + messages.push({ role: "user", content: blocks }); + }; + const wrapUp = (reason: WrapUpReason): Phase => ({ + kind: "wrapping-up", + reason, + responsesLeft: wrapUpResponses, + }); + const end = (stopCause: StopCause, extra: Partial = {}): Outcome => { + const causes = extra.causes ?? partial; + return { + stopCause, + wrapUp: phase.kind === "wrapping-up" ? phase.reason : null, + answer: stopCause === "submitted" ? "submitted" : causes.length > 0 ? "partial" : "none", + causes, + steps, + turns, + usage, + costUsd: costUsd(usage, settings.prices), + ...extra, + }; + }; + + const respond = async (): Promise => { + let message: Anthropic.Message | undefined; + for (const maxTokens of settings.maxTokens) { + const started = now(); + message = await client.messages.create( + { + model: settings.model, + max_tokens: maxTokens, + system: [{ type: "text", text: prompt.system, cache_control: { type: "ephemeral" } }], + tools: [...prompt.tools], + messages, + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: settings.effort }, + cache_control: { type: "ephemeral" }, + }, + // Without a timeout, the SDK refuses a non-streaming request that it expects to take over + // 10 minutes, at an hour per 128,000 max_tokens. + { timeout: Math.max(10 * 60_000, (60 * 60_000 * maxTokens) / 128_000) }, + ); + turns += 1; + usage = addUsage(usage, message.usage); + log({ + type: "response", + turn: turns, + maxTokens, + seconds: (now() - started) / 1000, + costUsd: costUsd(addUsage(noUsage, message.usage), settings.prices), + message, + }); + if (message.stop_reason !== "max_tokens") break; + } + if (message === undefined) throw new Error("No output limit to try"); + return message; + }; + + const limitReached = (): WrapUpReason | undefined => { + if ( + settings.softBudgetUsd !== undefined && + costUsd(usage, settings.prices) >= settings.softBudgetUsd + ) { + return "budget"; + } + if (settings.deadline !== undefined && now() >= settings.deadline) return "time"; + return undefined; + }; + + const runBash = async (input: unknown): Promise => { + if (isRecord(input) && input.restart === true) { + shell.restart(); + return { output: "The bash session was restarted.", isError: false }; + } + if (!isRecord(input) || typeof input.command !== "string") { + return { output: "The bash tool takes a command, or restart: true.", isError: true }; + } + const result = await shell.run(input.command); + return { + output: describeResult(result, limits.commandTimeoutSeconds), + isError: result.end.kind !== "finished", + end: result.end, + }; + }; + + log({ type: "user", text: prompt.instruction }); + messages.push({ role: "user", content: prompt.instruction }); + + for (;;) { + let response: Anthropic.Message; + try { + response = await respond(); + } catch (error) { + if (!(error instanceof APIError)) throw error; + const summary: ApiErrorSummary = { + status: error.status, + errorType: error.type, + message: error.message, + }; + log({ type: "api-error", ...summary }); + return end(isBudgetExhausted(error) ? "budget-exhausted" : "api-error", { error: summary }); + } + switch (response.stop_reason) { + case "refusal": + return end("refusal", { refusal: response.stop_details }); + case "max_tokens": + return end("max-tokens"); + case "model_context_window_exceeded": + return end("context-window"); + } + // The API rejects an empty assistant message, and merges the user messages around a gap. + if (response.content.length > 0) + messages.push({ role: "assistant", content: response.content }); + + const wasWrappingUp = phase.kind === "wrapping-up"; + if (phase.kind === "investigating") { + const reason = limitReached(); + if (reason !== undefined) phase = wrapUp(reason); + } + const calls = response.content.filter((block) => block.type === "tool_use"); + const results: Anthropic.ToolResultBlockParam[] = []; + for (const call of calls) { + const started = now(); + let step: number | null = null; + let result: ToolResult; + if (call.name === submitAnswer) { + step = ++steps; + const reading = readAnswer(call.input, task); + if (reading.valid) { + log(toolEvent(call, step, { output: "Accepted.", isError: false }, started, now())); + return end("submitted", { causes: reading.causes }); + } + if (reading.partial.length > 0) partial = reading.partial; + result = { + output: `Your answer was not accepted:\n${reading.problems.map((problem) => `- ${problem}`).join("\n")}\nCall submit_answer again with a corrected answer.`, + isError: true, + }; + } else if (call.name === bash && phase.kind === "investigating") { + step = ++steps; + result = await runBash(call.input); + } else if (call.name === bash) { + result = { + output: "Not run: you cannot run more commands. Call submit_answer now.", + isError: true, + }; + } else { + result = { output: `There is no tool named ${call.name}.`, isError: true }; + } + log(toolEvent(call, step, result, started, now())); + results.push({ + type: "tool_result", + tool_use_id: call.id, + content: result.output, + is_error: result.isError, + }); + if (phase.kind === "investigating" && steps >= limits.maxSteps) phase = wrapUp("steps"); + } + + if (phase.kind === "wrapping-up") { + if (wasWrappingUp) { + if (phase.responsesLeft <= 1) return end("no-answer"); + phase = { ...phase, responsesLeft: phase.responsesLeft - 1 }; + } + send(results, wrapUpMessage(phase.reason, limits)); + } else if (calls.length === 0) { + if (phase.nudged) return end("no-answer"); + phase = { kind: "investigating", nudged: true }; + send(results, nudge); + } else { + send(results); + } + } +} + +function isBudgetExhausted(error: APIError): boolean { + if (!(error instanceof PermissionDeniedError)) return false; + const type: string | null = error.type; + return type === budgetExhausted; +} + +function toolEvent( + call: Anthropic.ToolUseBlock, + step: number | null, + result: ToolResult, + started: number, + ended: number, +): Event { + return { + type: "tool", + step, + id: call.id, + name: call.name, + input: call.input, + output: result.output, + isError: result.isError, + seconds: (ended - started) / 1000, + ...(result.end === undefined ? {} : { end: result.end }), + }; +} + +function wrapUpMessage(reason: WrapUpReason, limits: Limits): string { + const why = { + steps: `You have made all ${limits.maxSteps} tool calls.`, + budget: "Your budget for this issue is almost spent.", + time: "Your time for this issue is almost up.", + }[reason]; + return `${why} Call submit_answer now with your best answer from what you have found so far. You cannot run more commands.`; +} diff --git a/agents/react/src/main.ts b/agents/react/src/main.ts new file mode 100644 index 0000000..79264d1 --- /dev/null +++ b/agents/react/src/main.ts @@ -0,0 +1,247 @@ +/** + * The `react` agent, the baseline of an evorca evaluation: Claude in a ReAct + * loop with a bash tool over the telemetry and a tool to submit the answer, + * with no memory between tasks and no self-improvement. + * + * The image runs this file. It reads the task from `/task.json` and its + * settings from the environment, investigates the telemetry in `/data`, and + * writes three files to `/out`: `prediction.json`, the answer as OpenRCA + * prediction JSON, if it has one; `trajectory.jsonl`, one line per event as + * it happens; and `summary.json`, how the task ended and what it cost. The + * process exits with 0 whenever the loop ends, with or without an answer, and + * with 1 only if the agent itself fails. + * + * The environment: + * - `ANTHROPIC_BASE_URL` and `ANTHROPIC_API_KEY`, which the SDK reads. + * - `EVORCA_MODEL`, `claude-sonnet-5` by default, and `EVORCA_EFFORT`, `high` + * by default. + * - `EVORCA_TASK_BUDGET_USD`, if set: the loop wraps up at 80% of it. + * - `EVORCA_TASK_TIMEOUT_SECONDS`, if set: the loop wraps up at 80% of it. + * - `EVORCA_SEED`, which the summary records. The model takes no seed. + * + * @module + */ +import { appendFileSync } from "node:fs"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import Anthropic from "@anthropic-ai/sdk"; +import background from "../background.json" with { type: "json" }; +import { formatPrediction } from "./answer.ts"; +import { BashSession } from "./bash.ts"; +import { efforts, investigate, type Effort, type Event, type Outcome } from "./loop.ts"; +import { buildPrompt, type Background, type Limits, type Prompt } from "./prompt.ts"; +import { parseTask, type Task } from "./task.ts"; +import { prices, type Prices } from "./usage.ts"; + +/** The agent's settings from the environment. */ +export type Config = { + readonly model: string; + readonly prices: Prices; + readonly effort: Effort; + readonly budgetUsd?: number; + readonly timeoutSeconds?: number; + readonly seed?: string; +}; + +/** What `summary.json` holds. */ +export type Summary = Omit & { + readonly model: string; + readonly effort: Effort; + readonly promptSha256: string; + readonly seed: string | null; + readonly estimatedCostUsd: number; + readonly budgetUsd: number | null; + readonly softBudgetUsd: number | null; + readonly maxSteps: number; + readonly startedAt: string; + readonly endedAt: string; + readonly seconds: number; +}; + +/** + * The agent's limits. RCA-Agent runs at most 25 controller steps, each of + * which may execute code twice, and stops a cell after 60 seconds on two CPUs. + */ +export const limits: Limits = { maxSteps: 50, commandTimeoutSeconds: 120, outputChars: 16_000 }; + +const maxTokens = [16_000, 64_000] as const; +const wrapUpShare = 0.8; + +/** + * Reads the agent's settings from `env`. + * + * @throws Error if the model has no prices, the effort is unknown, or the + * budget or the timeout is not a positive number. + */ +export function readConfig(env: NodeJS.ProcessEnv): Config { + const model = env.EVORCA_MODEL ?? "claude-sonnet-5"; + const modelPrices = prices[model]; + if (modelPrices === undefined) throw new Error(`No prices for EVORCA_MODEL ${model}`); + const effortName = env.EVORCA_EFFORT ?? "high"; + const effort = efforts.find((level) => level === effortName); + if (effort === undefined) throw new Error(`Unknown EVORCA_EFFORT ${effortName}`); + const budgetUsd = positive(env, "EVORCA_TASK_BUDGET_USD"); + const timeoutSeconds = positive(env, "EVORCA_TASK_TIMEOUT_SECONDS"); + return { + model, + prices: modelPrices, + effort, + ...(budgetUsd === undefined ? {} : { budgetUsd }), + ...(timeoutSeconds === undefined ? {} : { timeoutSeconds }), + ...(env.EVORCA_SEED === undefined ? {} : { seed: env.EVORCA_SEED }), + }; +} + +function positive(env: NodeJS.ProcessEnv, name: string): number | undefined { + const text = env[name]; + if (text === undefined || text === "") return undefined; + const value = Number(text); + if (!Number.isFinite(value) || value <= 0) + throw new Error(`${name} is not a positive number: ${text}`); + return value; +} + +/** What {@link runTask} needs. The shell runs in `shellCwd` with `shellEnv`. */ +export type TaskRun = { + readonly task: Task; + readonly config: Config; + readonly client: Anthropic; + readonly outDir: string; + readonly shellCwd: string; + readonly shellEnv: NodeJS.ProcessEnv; + readonly limits?: Limits; + readonly now?: () => number; +}; + +/** + * Runs the agent on one task, writes its files to `outDir`, and returns the + * summary. + * + * @throws Error if the agent fails, as distinct from the model or the API. + */ +export async function runTask(run: TaskRun): Promise { + const { task, config, client, outDir } = run; + const now = run.now ?? Date.now; + const taskLimits = run.limits ?? limits; + const startedAt = now(); + const prompt = buildPrompt(task, typedBackground, taskLimits); + const softBudgetUsd = config.budgetUsd === undefined ? undefined : config.budgetUsd * wrapUpShare; + const deadline = + config.timeoutSeconds === undefined + ? undefined + : startedAt + config.timeoutSeconds * 1000 * wrapUpShare; + const trajectory = join(outDir, "trajectory.jsonl"); + const log = (event: Event | TaskEvent) => + appendFileSync( + trajectory, + `${JSON.stringify({ at: new Date(now()).toISOString(), ...event })}\n`, + ); + log({ + type: "task", + task, + model: config.model, + effort: config.effort, + limits: taskLimits, + promptSha256: prompt.sha256, + system: prompt.system, + tools: prompt.tools, + }); + const shell = new BashSession({ + cwd: run.shellCwd, + env: run.shellEnv, + timeoutMs: taskLimits.commandTimeoutSeconds * 1000, + outputChars: taskLimits.outputChars, + }); + let outcome: Outcome; + try { + outcome = await investigate({ + client, + prompt, + task, + shell, + limits: taskLimits, + log, + now, + settings: { + model: config.model, + effort: config.effort, + prices: config.prices, + maxTokens, + ...(softBudgetUsd === undefined ? {} : { softBudgetUsd }), + ...(deadline === undefined ? {} : { deadline }), + }, + }); + } finally { + shell.restart(); + } + if (outcome.causes.length > 0) { + await writeFile(join(outDir, "prediction.json"), formatPrediction(outcome.causes)); + } + const endedAt = now(); + const { causes: _, costUsd, ...rest } = outcome; + const summary: Summary = { + ...rest, + model: config.model, + effort: config.effort, + promptSha256: prompt.sha256, + seed: config.seed ?? null, + estimatedCostUsd: costUsd, + budgetUsd: config.budgetUsd ?? null, + softBudgetUsd: softBudgetUsd ?? null, + maxSteps: taskLimits.maxSteps, + startedAt: new Date(startedAt).toISOString(), + endedAt: new Date(endedAt).toISOString(), + seconds: (endedAt - startedAt) / 1000, + }; + await writeFile(join(outDir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`); + return summary; +} + +const typedBackground: Background = background; + +type TaskEvent = { + readonly type: "task"; + readonly task: Task; + readonly model: string; + readonly effort: Effort; + readonly limits: Limits; + readonly promptSha256: string; + readonly system: string; + readonly tools: Prompt["tools"]; +}; + +/** + * The environment of the bash session: the agent's own, without the API key, + * the proxy's address, and the agent's settings. + */ +export function shellEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(env).filter(([name]) => !/^(ANTHROPIC|EVORCA)_/.test(name)), + ); +} + +if (import.meta.main) { + try { + const config = readConfig(process.env); + const task = parseTask(await readFile("/task.json", "utf8")); + const summary = await runTask({ + task, + config, + client: new Anthropic({ maxRetries: 5 }), + outDir: "/out", + shellCwd: "/tmp", + shellEnv: shellEnvironment(process.env), + }); + console.log( + `${summary.stopCause}, ${summary.answer} answer, ${summary.steps} steps, $${summary.estimatedCostUsd.toFixed(4)}`, + ); + process.exit(0); + } catch (error) { + console.error(error); + await writeFile( + "/out/summary.json", + `${JSON.stringify({ stopCause: "crashed", error: String(error) }, null, 2)}\n`, + ).catch(() => {}); + process.exit(1); + } +} diff --git a/agents/react/src/usage.ts b/agents/react/src/usage.ts new file mode 100644 index 0000000..6c9a121 --- /dev/null +++ b/agents/react/src/usage.ts @@ -0,0 +1,61 @@ +/** + * Token usage summed over a task's model responses, and its estimated cost. + * + * The cost uses the list prices per million tokens: base input, output, cache + * reads, and five-minute cache writes, which cost 1.25 times base input. The + * agent caches only with the five-minute TTL. + * + * @module + */ +import type Anthropic from "@anthropic-ai/sdk"; + +/** The token counts of one response, or of all responses of a task. */ +export type Usage = { + readonly input_tokens: number; + readonly output_tokens: number; + readonly cache_creation_input_tokens: number; + readonly cache_read_input_tokens: number; +}; + +/** Prices in US dollars per million tokens. */ +export type Prices = { + readonly input: number; + readonly output: number; + readonly cacheWrite: number; + readonly cacheRead: number; +}; + +/** The prices of each model that the agent can run, by model ID. */ +export const prices: Readonly> = { + "claude-sonnet-5": { input: 2, output: 10, cacheWrite: 2.5, cacheRead: 0.2 }, +}; + +/** Usage of no tokens. */ +export const noUsage: Usage = { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, +}; + +/** Adds the usage of a response to `total`. */ +export function addUsage(total: Usage, usage: Anthropic.Usage): Usage { + return { + input_tokens: total.input_tokens + usage.input_tokens, + output_tokens: total.output_tokens + usage.output_tokens, + cache_creation_input_tokens: + total.cache_creation_input_tokens + (usage.cache_creation_input_tokens ?? 0), + cache_read_input_tokens: total.cache_read_input_tokens + (usage.cache_read_input_tokens ?? 0), + }; +} + +/** The cost of `usage` in US dollars. */ +export function costUsd(usage: Usage, price: Prices): number { + return ( + (usage.input_tokens * price.input + + usage.output_tokens * price.output + + usage.cache_creation_input_tokens * price.cacheWrite + + usage.cache_read_input_tokens * price.cacheRead) / + 1_000_000 + ); +} diff --git a/agents/react/test/loop.test.ts b/agents/react/test/loop.test.ts new file mode 100644 index 0000000..74bfb1e --- /dev/null +++ b/agents/react/test/loop.test.ts @@ -0,0 +1,475 @@ +import { existsSync } from "node:fs"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Anthropic from "@anthropic-ai/sdk"; +import { afterEach, describe, expect, it } from "vitest"; +import { parsePrediction } from "../../../packages/openrca/src/prediction.ts"; +import { runTask, shellEnvironment, type Config } from "../src/main.ts"; +import type { Limits } from "../src/prompt.ts"; +import type { Task } from "../src/task.ts"; +import { prices } from "../src/usage.ts"; +import { + apiError, + bashCall, + message, + startMockApi, + submitCall, + thought, + toolUse, + type MockApi, + type Reply, +} from "./mock-api.ts"; + +const task: Task = { + scenario: "telecom", + row: 4, + taskIndex: "task_7", + instruction: "On April 11, 2020, between 00:00 and 00:30, a failure occurred.", + candidates: { components: ["os_001", "docker_003"], reasons: ["CPU fault", "network delay"] }, +}; +const sonnet = prices["claude-sonnet-5"] ?? { input: 0, output: 0, cacheWrite: 0, cacheRead: 0 }; +const config: Config = { model: "claude-sonnet-5", prices: sonnet, effort: "high", seed: "7" }; +const limits: Limits = { maxSteps: 50, commandTimeoutSeconds: 10, outputChars: 2_000 }; +const answer = { + occurrence_datetime: "2020-04-11 00:05:00", + component: "docker_003", + reason: "CPU fault", +}; +const expectedPrediction = + '{"1":{"root cause occurrence datetime":"2020-04-11 00:05:00","root cause component":"docker_003","root cause reason":"CPU fault"}}'; + +type Request = { + readonly model: string; + readonly max_tokens: number; + readonly system: unknown; + readonly tools: unknown; + readonly messages: readonly { readonly role: string; readonly content: unknown }[]; + readonly [key: string]: unknown; +}; + +const cleanups: (() => Promise)[] = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup(); +}); + +async function run( + replies: readonly Reply[], + options: { + config?: Partial; + limits?: Partial; + now?: (mock: MockApi) => number; + } = {}, +) { + const mock = await startMockApi(replies); + const dir = await mkdtemp(join(tmpdir(), "evorca-react-")); + cleanups.push(mock.close, () => rm(dir, { recursive: true, force: true })); + const outDir = join(dir, "out"); + const shellCwd = join(dir, "tmp"); + await Promise.all( + [outDir, shellCwd].map((path) => import("node:fs/promises").then((fs) => fs.mkdir(path))), + ); + const summary = await runTask({ + task, + config: { ...config, ...options.config }, + client: new Anthropic({ apiKey: "task-token", baseURL: mock.url, maxRetries: 0 }), + outDir, + shellCwd, + shellEnv: shellEnvironment({ ...process.env, ANTHROPIC_API_KEY: "secret", EVORCA_MODEL: "x" }), + limits: { ...limits, ...options.limits }, + ...(options.now === undefined ? {} : { now: () => options.now?.(mock) ?? 0 }), + }); + const read = (file: string) => readFile(join(outDir, file), "utf8"); + const prediction = existsSync(join(outDir, "prediction.json")) + ? await read("prediction.json") + : undefined; + const trajectory = (await read("trajectory.jsonl")) + .trim() + .split("\n") + .map((line): Record => JSON.parse(line)); + const requests = mock.requests.map((body): Request => JSON.parse(body)); + return { + summary, + savedSummary: JSON.parse(await read("summary.json")), + prediction, + trajectory, + requests, + rawRequests: mock.requests, + shellCwd, + }; +} + +const lastUserContent = (request: Request | undefined) => request?.messages.at(-1)?.content; + +describe("the ReAct loop", () => { + it("runs commands until the model submits, and writes the three files", async () => { + const result = await run([ + message([...thought("Let me look."), bashCall("echo hello > note.txt; cat note.txt")], { + usage: { input_tokens: 3000, output_tokens: 200, cache_creation_input_tokens: 5000 }, + }), + message( + [bashCall('cat note.txt; echo "key=$ANTHROPIC_API_KEY model=${EVORCA_MODEL:-unset}"')], + { + usage: { input_tokens: 50, output_tokens: 100, cache_read_input_tokens: 8000 }, + }, + ), + message([...thought("Found it."), submitCall([answer])], { + usage: { input_tokens: 50, output_tokens: 300, cache_read_input_tokens: 8200 }, + }), + ]); + + expect(result.prediction).toBe(expectedPrediction); + expect(parsePrediction(result.prediction ?? "")).toEqual([ + { + occurredAt: Date.UTC(2020, 3, 10, 16, 5) / 1000, + component: "docker_003", + reason: "CPU fault", + }, + ]); + expect(lastUserContent(result.requests[1])).toEqual([ + expect.objectContaining({ type: "tool_result", content: "hello\n", is_error: false }), + ]); + expect(lastUserContent(result.requests[2])).toEqual([ + expect.objectContaining({ content: "hello\nkey= model=unset\n" }), + ]); + const usage = { + input_tokens: 3100, + output_tokens: 600, + cache_creation_input_tokens: 5000, + cache_read_input_tokens: 16200, + }; + expect(result.summary).toMatchObject({ + stopCause: "submitted", + answer: "submitted", + wrapUp: null, + steps: 3, + turns: 3, + usage, + model: "claude-sonnet-5", + effort: "high", + seed: "7", + maxSteps: 50, + budgetUsd: null, + }); + expect(result.summary.estimatedCostUsd).toBeCloseTo( + (3100 * 2 + 600 * 10 + 5000 * 2.5 + 16200 * 0.2) / 1e6, + 10, + ); + expect(result.savedSummary).toEqual(JSON.parse(JSON.stringify(result.summary))); + expect(result.trajectory.map((event) => event.type)).toEqual([ + "task", + "user", + "response", + "tool", + "response", + "tool", + "response", + "tool", + ]); + expect(result.trajectory[0]).toMatchObject({ promptSha256: result.summary.promptSha256 }); + expect(result.trajectory[3]).toMatchObject({ + name: "bash", + step: 1, + input: { command: "echo hello > note.txt; cat note.txt" }, + output: "hello\n", + isError: false, + end: { kind: "finished", exitCode: 0 }, + }); + expect(result.trajectory[2]).toMatchObject({ + turn: 1, + maxTokens: 16000, + message: { stop_reason: "tool_use", usage: { input_tokens: 3000 } }, + }); + }); + + it("sends the same system prompt and tools every turn and only appends to the history", async () => { + const result = await run([ + message([bashCall("echo one")]), + message([...thought("Next."), bashCall("echo two"), bashCall("echo three")]), + message([submitCall([answer])]), + ]); + + expect(result.requests).toHaveLength(3); + const prefix = (body: string) => body.slice(0, body.indexOf('"messages":')); + for (const body of result.rawRequests) { + expect(JSON.stringify(JSON.parse(body).system)).toBe( + JSON.stringify(result.requests[0]?.system), + ); + expect(JSON.stringify(JSON.parse(body).tools)).toBe( + JSON.stringify(result.requests[0]?.tools), + ); + } + expect(new Set(result.rawRequests.map(prefix)).size).toBe(1); + for (const [i, request] of result.requests.entries()) { + const next = result.requests[i + 1]; + if (next !== undefined) + expect(next.messages.slice(0, request.messages.length)).toEqual(request.messages); + expect(request).toMatchObject({ + model: "claude-sonnet-5", + max_tokens: 16000, + thinking: { type: "adaptive", display: "summarized" }, + output_config: { effort: "high" }, + cache_control: { type: "ephemeral" }, + system: [{ type: "text", cache_control: { type: "ephemeral" } }], + }); + for (const key of ["temperature", "top_p", "top_k", "tool_choice", "stream"]) { + expect(request).not.toHaveProperty(key); + } + } + expect(result.requests[0]?.messages).toEqual([ + { + role: "user", + content: `The issue you are going to solve is:\n\n${task.instruction}\n\nSolve the issue step-by-step.`, + }, + ]); + expect(lastUserContent(result.requests[2])).toEqual([ + expect.objectContaining({ type: "tool_result", content: "two\n" }), + expect.objectContaining({ type: "tool_result", content: "three\n" }), + ]); + }); + + it("wraps up at the step cap and refuses further commands", async () => { + const result = await run( + [ + message([bashCall("echo 1 >> runs")]), + message([bashCall("echo 2 >> runs"), bashCall("echo 3 >> runs")]), + message([bashCall("echo 4 >> runs")]), + message([submitCall([answer])]), + ], + { limits: { maxSteps: 2 } }, + ); + + expect(await readFile(join(result.shellCwd, "runs"), "utf8")).toBe("1\n2\n"); + const wrapUp = { + type: "text", + text: "You have made all 2 tool calls. Call submit_answer now with your best answer from what you have found so far. You cannot run more commands.", + }; + expect(lastUserContent(result.requests[2])).toEqual([ + expect.objectContaining({ content: "(no output)", is_error: false }), + expect.objectContaining({ + content: "Not run: you cannot run more commands. Call submit_answer now.", + is_error: true, + }), + wrapUp, + ]); + expect(lastUserContent(result.requests[3])).toEqual([ + expect.objectContaining({ is_error: true }), + wrapUp, + ]); + expect(result.summary).toMatchObject({ + stopCause: "submitted", + wrapUp: "steps", + steps: 3, + turns: 4, + }); + expect(result.prediction).toBe(expectedPrediction); + }); + + it("gives up after two responses without an answer once it wraps up", async () => { + const result = await run( + [ + message([bashCall("true")]), + message(thought("I think..."), { stopReason: "end_turn" }), + message(thought("Still thinking."), { stopReason: "end_turn" }), + ], + { limits: { maxSteps: 1 } }, + ); + + expect(result.summary).toMatchObject({ + stopCause: "no-answer", + answer: "none", + wrapUp: "steps", + turns: 3, + }); + expect(result.prediction).toBeUndefined(); + }); + + it("wraps up at the soft budget before running the turn's commands", async () => { + const result = await run( + [ + message([bashCall("echo ran > spent")], { usage: { output_tokens: 85_000 } }), + message([submitCall([answer])]), + ], + { config: { budgetUsd: 1 } }, + ); + + expect(existsSync(join(result.shellCwd, "spent"))).toBe(false); + expect(lastUserContent(result.requests[1])).toEqual([ + expect.objectContaining({ is_error: true }), + { + type: "text", + text: "Your budget for this issue is almost spent. Call submit_answer now with your best answer from what you have found so far. You cannot run more commands.", + }, + ]); + expect(result.summary).toMatchObject({ + stopCause: "submitted", + wrapUp: "budget", + budgetUsd: 1, + softBudgetUsd: 0.8, + steps: 1, + }); + }); + + it("wraps up at the soft deadline", async () => { + const result = await run( + [message([bashCall("echo late > late")]), message([submitCall([answer])])], + { + config: { timeoutSeconds: 100 }, + now: (mock) => (mock.requests.length === 0 ? 0 : 90_000), + }, + ); + + expect(existsSync(join(result.shellCwd, "late"))).toBe(false); + expect(result.summary).toMatchObject({ stopCause: "submitted", wrapUp: "time" }); + expect(JSON.stringify(lastUserContent(result.requests[1]))).toContain( + "Your time for this issue is almost up.", + ); + }); + + it("keeps the valid fields of a rejected answer when the budget runs out", async () => { + const result = await run([ + message([submitCall([{ ...answer, occurrence_datetime: "2020-04-11T00:05" }])]), + apiError(403, "evorca_budget_exhausted", "The task's budget is spent."), + ]); + + expect(result.summary).toMatchObject({ + stopCause: "budget-exhausted", + answer: "partial", + error: { status: 403, errorType: "evorca_budget_exhausted" }, + }); + expect(result.prediction).toBe( + '{"1":{"root cause component":"docker_003","root cause reason":"CPU fault"}}', + ); + expect(result.trajectory.at(-1)).toMatchObject({ type: "api-error", status: 403 }); + }); + + it("writes no prediction when the budget runs out before any answer", async () => { + const result = await run([ + message([bashCall("true")]), + apiError(403, "evorca_budget_exhausted"), + ]); + + expect(result.summary).toMatchObject({ + stopCause: "budget-exhausted", + answer: "none", + steps: 1, + }); + expect(result.prediction).toBeUndefined(); + }); + + it("ends on another API error", async () => { + const result = await run([apiError(400, "invalid_request_error", "messages: bad")]); + + expect(result.summary).toMatchObject({ + stopCause: "api-error", + error: { status: 400, errorType: "invalid_request_error" }, + turns: 0, + }); + }); + + it("asks again with a larger output limit after max_tokens, without running the cut-off call", async () => { + const result = await run([ + message([...thought("A long thought"), toolUse("bash", { command: "echo cut > cut" })], { + stopReason: "max_tokens", + }), + message([submitCall([answer])]), + ]); + + expect(existsSync(join(result.shellCwd, "cut"))).toBe(false); + expect(result.requests.map((request) => request.max_tokens)).toEqual([16000, 64000]); + expect(result.requests[1]?.messages).toEqual(result.requests[0]?.messages); + expect(result.summary).toMatchObject({ stopCause: "submitted", turns: 2 }); + }); + + it("stops when the largest output limit is not enough", async () => { + const cut = message(thought("Endless"), { stopReason: "max_tokens" }); + const result = await run([cut, cut]); + + expect(result.summary).toMatchObject({ stopCause: "max-tokens", answer: "none", turns: 2 }); + }); + + it("asks once for an answer when the model stops without one, then gives up", async () => { + const result = await run([ + message(thought("The root cause is docker_003."), { stopReason: "end_turn" }), + message(thought("As I said, docker_003."), { stopReason: "end_turn" }), + ]); + + expect(lastUserContent(result.requests[1])).toEqual([ + { + type: "text", + text: "You ended your turn without calling submit_answer. Call submit_answer now with your answer, or continue the investigation with bash if you need more evidence.", + }, + ]); + expect(result.summary).toMatchObject({ stopCause: "no-answer", answer: "none" }); + expect(result.prediction).toBeUndefined(); + }); + + it("sends no empty assistant message after an empty response", async () => { + const result = await run([ + message([], { stopReason: "end_turn" }), + message([submitCall([answer])]), + ]); + + expect(result.requests[1]?.messages.map((entry) => entry.role)).toEqual(["user", "user"]); + expect(result.summary).toMatchObject({ stopCause: "submitted" }); + }); + + it("accepts an answer after the nudge", async () => { + const result = await run([ + message(thought("Done."), { stopReason: "end_turn" }), + message([submitCall([answer])]), + ]); + + expect(result.summary).toMatchObject({ stopCause: "submitted" }); + expect(result.prediction).toBe(expectedPrediction); + }); + + it("returns a malformed answer to the model and accepts the corrected one", async () => { + const result = await run([ + message([submitCall([{ ...answer, reason: null }])]), + message([submitCall([answer])]), + ]); + + expect(lastUserContent(result.requests[1])).toEqual([ + expect.objectContaining({ + type: "tool_result", + is_error: true, + content: + "Your answer was not accepted:\n- Root cause 1 has no reason, which the issue asks for.\nCall submit_answer again with a corrected answer.", + }), + ]); + expect(result.summary).toMatchObject({ stopCause: "submitted", steps: 2 }); + expect(result.prediction).toBe(expectedPrediction); + }); + + it("records a refusal and stops", async () => { + const result = await run([ + message([], { + stopReason: "refusal", + stopDetails: { type: "refusal", category: null, explanation: null }, + }), + ]); + + expect(result.summary).toMatchObject({ + stopCause: "refusal", + answer: "none", + refusal: { type: "refusal" }, + }); + }); + + it("restarts the bash session on request", async () => { + const result = await run([ + message([bashCall("export MARK=set")]), + message([toolUse("bash", { restart: true })]), + message([bashCall('echo "mark=$MARK"')]), + message([submitCall([answer])]), + ]); + + expect(lastUserContent(result.requests[2])).toEqual([ + expect.objectContaining({ content: "The bash session was restarted." }), + ]); + expect(lastUserContent(result.requests[3])).toEqual([ + expect.objectContaining({ content: "mark=\n" }), + ]); + }); +}); diff --git a/agents/react/test/mock-api.ts b/agents/react/test/mock-api.ts new file mode 100644 index 0000000..e9c0e07 --- /dev/null +++ b/agents/react/test/mock-api.ts @@ -0,0 +1,140 @@ +/** + * A scripted stand-in for the Messages API. It answers each `POST /v1/messages` + * with the next scripted reply and records the raw request bodies, so a test + * can point the real SDK at it and check what the agent sent. + * + * It uses only Node's built-in modules, so `node test/mock-api.ts + * ` also serves the replies in a JSON file from a container. + * + * @module + */ +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; + +/** One scripted reply: an HTTP status and a JSON body. */ +export type Reply = { readonly status: number; readonly body: unknown }; + +/** A running mock. `requests` holds the raw body of each request, in order. */ +export type MockApi = { + readonly url: string; + readonly requests: string[]; + close(): Promise; +}; + +/** Starts a mock that serves `replies` in order, and a 500 once they run out. */ +export async function startMockApi( + replies: readonly Reply[], + { host = "127.0.0.1", port = 0 } = {}, +): Promise { + const requests: string[] = []; + const server = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + if (request.method !== "POST" || request.url !== "/v1/messages") { + response.writeHead(404).end(); + return; + } + requests.push(body); + const reply = replies[requests.length - 1] ?? { + status: 500, + body: { type: "error", error: { type: "api_error", message: "No scripted reply left" } }, + }; + response.writeHead(reply.status, { "content-type": "application/json" }); + response.end(JSON.stringify(reply.body)); + }); + await new Promise((resolve) => server.listen(port, host, resolve)); + const address: AddressInfo | string | null = server.address(); + if (address === null || typeof address === "string") throw new Error("The mock has no port"); + return { + url: `http://${host}:${address.port}`, + requests, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +type Usage = { + readonly input_tokens?: number; + readonly output_tokens?: number; + readonly cache_creation_input_tokens?: number; + readonly cache_read_input_tokens?: number; +}; + +let counter = 0; + +/** A 200 reply with a model message of `content`. */ +export function message( + content: readonly unknown[], + options: { + readonly stopReason?: string; + readonly usage?: Usage; + readonly stopDetails?: unknown; + } = {}, +): Reply { + const { stopReason = "tool_use", usage = {}, stopDetails = null } = options; + counter += 1; + return { + status: 200, + body: { + id: `msg_${counter}`, + type: "message", + role: "assistant", + model: "claude-sonnet-5", + content, + stop_reason: stopReason, + stop_sequence: null, + stop_details: stopDetails, + usage: { + input_tokens: 10, + output_tokens: 100, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + ...usage, + }, + }, + }; +} + +/** A tool call. */ +export function toolUse(name: string, input: unknown): unknown { + counter += 1; + return { type: "tool_use", id: `toolu_${counter}`, name, input }; +} + +/** A call of the bash tool. */ +export function bashCall(command: string): unknown { + return toolUse("bash", { command }); +} + +/** A call of `submit_answer` with one root cause per entry. */ +export function submitCall( + rootCauses: readonly { + readonly occurrence_datetime: string | null; + readonly component: string | null; + readonly reason: string | null; + }[], +): unknown { + return toolUse("submit_answer", { root_causes: rootCauses }); +} + +/** A summarized thinking block and a text block. */ +export function thought(text: string): unknown[] { + return [ + { type: "thinking", thinking: text, signature: "c2lnbmF0dXJl" }, + { type: "text", text }, + ]; +} + +/** An API error reply. */ +export function apiError(status: number, type: string, text = type): Reply { + return { status, body: { type: "error", error: { type, message: text } } }; +} + +if (import.meta.main) { + const [path, port] = process.argv.slice(2); + if (path === undefined || port === undefined) + throw new Error("Usage: mock-api.ts "); + const replies: Reply[] = JSON.parse(await readFile(path, "utf8")); + const mock = await startMockApi(replies, { host: "0.0.0.0", port: Number(port) }); + console.log(`Mock Messages API on ${mock.url} with ${replies.length} replies`); +} From d60264aaea44e6054efd5060adb441d3eed7988a Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:42:26 +0200 Subject: [PATCH 5/6] feat(agent-react): add the image and a container smoke test The image adds Python 3.13 with pandas, pytz, and DuckDB from hashed pins. The smoke test runs one task in the image with the runner's restrictions against a mock Messages API on an internal network. --- agents/react/Dockerfile | 14 ++ agents/react/requirements.in | 3 + agents/react/requirements.txt | 159 +++++++++++++++++++++++ agents/react/scripts/smoke.ts | 234 ++++++++++++++++++++++++++++++++++ 4 files changed, 410 insertions(+) create mode 100644 agents/react/Dockerfile create mode 100644 agents/react/requirements.in create mode 100644 agents/react/requirements.txt create mode 100644 agents/react/scripts/smoke.ts diff --git a/agents/react/Dockerfile b/agents/react/Dockerfile new file mode 100644 index 0000000..9b3b141 --- /dev/null +++ b/agents/react/Dockerfile @@ -0,0 +1,14 @@ +FROM node:26-trixie-slim@sha256:e6bedef217677b276c591795e169840edd9096dbf13f29df8ef8d1856b8820bc +RUN apt-get update \ + && apt-get install -y --no-install-recommends python3 python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt /opt/agent/requirements.txt +RUN python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir --require-hashes -r /opt/agent/requirements.txt +ENV PATH=/opt/venv/bin:$PATH HOME=/tmp PYTHONDONTWRITEBYTECODE=1 +WORKDIR /agent +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund && npm cache clean --force +COPY background.json ./ +COPY src/ ./src/ +ENTRYPOINT ["node", "src/main.ts"] diff --git a/agents/react/requirements.in b/agents/react/requirements.in new file mode 100644 index 0000000..bacc50c --- /dev/null +++ b/agents/react/requirements.in @@ -0,0 +1,3 @@ +duckdb +pandas +pytz diff --git a/agents/react/requirements.txt b/agents/react/requirements.txt new file mode 100644 index 0000000..5e975cc --- /dev/null +++ b/agents/react/requirements.txt @@ -0,0 +1,159 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile requirements.in --generate-hashes --python-version 3.13 --python-platform linux --exclude-newer 2026-11-03 -o requirements.txt +duckdb==2.0.0 \ + --hash=sha256:09de511ef0dc76a5213e8d3d87a8b89bb4d3e8a6fe73a9f60dc001269946b44f \ + --hash=sha256:46c7a299ff5f45be9068cfc0ceecb3fd574af9535de8cb6908ae8b52d2846cbd \ + --hash=sha256:5a3601121c7cc50d0b70412995634682fafcb58a0156d5d663dfd0b3bbe30a33 \ + --hash=sha256:61a244faae9fd787e30f4c76888769ec49e79736c434f698e8178a00af8348f0 \ + --hash=sha256:62bea9a1786be98d2906197c67a4089cc8b34d66d19e87e5543d8201e81323ad \ + --hash=sha256:9bcc9921be0e1b2404b96a7fb447a8b80fe2f3a919cdc976ca6c39951cc8f2f5 \ + --hash=sha256:a387d4907b535a4bd82ba8aede6809d0bf023dfbb21362832f3c9848aea7743c \ + --hash=sha256:a3b38d8cdbef10007c64b65efc4087f799c4dc606785aa5f2a425ed13658ca01 \ + --hash=sha256:ab51ec57d329d6f3df37c0287b55d71c3fc07bb54b61d1afed09104f953b1431 \ + --hash=sha256:b5d758b53751c930e09d371230241005ea5203c8edc65a1e1fafca83fcc906ef \ + --hash=sha256:bcdf10dfbc2c1a6bdc5b2a94bba5b15481e552c5745c2af1994617b0eb6bc750 \ + --hash=sha256:c5e6708efaa8647e0914a16f1914354419e529a1925929950e7d610418e45873 \ + --hash=sha256:cb1cc30318397c3bcffa6b683196293698052588de3c1ba1e144ba0c72560ce9 \ + --hash=sha256:e38daf0acbad9de7ce892d402b68409519291ee16af3c2cba85ace3b085c7eaa \ + --hash=sha256:f61c96547ed432f54dae42a717fd88563eda55cb47549715fac9f61f770c205a + # via -r requirements.in +numpy==2.5.4 \ + --hash=sha256:067374eb538c34c745436365cf7b0112595c1d326f21ce4ff340f61230239fbb \ + --hash=sha256:0b4724a19de67bea8cfc4970798efa78bcbbe2ac2613cfac16721a42d44de2a5 \ + --hash=sha256:0f02a46e49cfb6c73bdb7aea1c0d3461dbae9aba613542b65f657cd3d17b9fab \ + --hash=sha256:1c2e71b04c6cad90026e544501bbe0ab9290fa8a4d845e7e8c0d124fb429c988 \ + --hash=sha256:1ef3aa6d7e29bb13677323114280b05acc57607fa2300e66432d665d5418a162 \ + --hash=sha256:2132418bf8dd124a427ca9e6a1daf9ee1a87185344c95119ceae868b99466da1 \ + --hash=sha256:2199ed071f460487c8db2c0e5c0b564494190edb4772fe80f9aad88b2604def5 \ + --hash=sha256:2377da2dd3ba2c1200956acbab2a358c83b8e1f8531191672d1cd6ad83250d53 \ + --hash=sha256:298eca75243f2cbbfdb460560b9fb2a1792a33cf2ab4286efd43d92e8d3df508 \ + --hash=sha256:2c2c4afffdeb7920e445028dd71eb932cac3e704792e964bc2a232426d4f1255 \ + --hash=sha256:2ca144f15135b6212a5c47b1e2aeca6e412f102f95a2d5d88d8aec77eb255de3 \ + --hash=sha256:2fa3328f784fc8277fc48026f6cad516f5c561c5d8e2e39b3c9e0c8f23223b34 \ + --hash=sha256:325518d4245b9e331387702aa58c2ce1dc4cdcbb41dfb4ccd5dcbc7e08db1266 \ + --hash=sha256:332f3378fe077dd850e677ec01bdcc4f22368fb5d50ef10b2c79230b1bf5a592 \ + --hash=sha256:3573cd22564692a5b899ec344e5d5b9cc4576f2985b96f22af3564ed54f2710f \ + --hash=sha256:381a7a3d2e65e64c0ec302795ab9dc12bb1e73f150904699c153716177eebdaf \ + --hash=sha256:38f47be9f74ab870d2633b5456ae519c43758a8d1fd05342f0ce4ecc034396ee \ + --hash=sha256:4054173604cd8658796053f1f3bc0befb68ec1c0762c57fdad61e199256a8617 \ + --hash=sha256:468397ba3c64427474706e5c9123fe266395496714dc684294eac75cd4930d1e \ + --hash=sha256:4e263278bfb5ee6409db8aedbc4cc32973b1b82bc1e8d3c668551d04d83a7e37 \ + --hash=sha256:5258bc06526964be5face2fc6f756857a3f24f21ec3e72ca131337a75b165d6c \ + --hash=sha256:56733449d2544178beaa4545cee357370440cf056c197f9c7bfb19dbfdd0e86d \ + --hash=sha256:5ec3753760c1a6d8bb91200666e545c3a9728e6269dfb5d6ce02340996698aa3 \ + --hash=sha256:5fbf7141bbfd63aea22f435c9062a032b9ea0082fe9845dad7f021d3f1234e71 \ + --hash=sha256:64d1c8ac28a4077cf987e0a71a7a0ef7e2df70722f07f0baa42dbb7eb6938647 \ + --hash=sha256:64f9c9878c1938476365e11ccfb6b770f3b9e5f045ccddc514235041e6959365 \ + --hash=sha256:6c109eac9cd439193678f69d70733c1108487546ca8eafc107b510ae10c1aecd \ + --hash=sha256:6d6a71b9d9a97c03633aa12565ef2825ffa036cc1d99cfd50dacf0f128af4fe2 \ + --hash=sha256:6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0 \ + --hash=sha256:7415db95818b39ec475a5eea54d9e3b6bc83e3912158e46da3438cdce399804d \ + --hash=sha256:77045a4b175bbf5316ec08003880804336c78f92281a1b72222b274ea85ec5ac \ + --hash=sha256:7a14a461d9340f1b46b8648578aed9cdb8b3b018a8fac6c1dde2c9192a01a87f \ + --hash=sha256:80d6ef6e8620eb2c2b4c4caad50b5935d6db3cde2d51581b55dcc79e14016d1d \ + --hash=sha256:81e3420b27048b65eb14c3acf0c174a8cb0e023277716110347d2dcb26026dad \ + --hash=sha256:823874a507a84af050493b622affde94b6f7c3a0dc22cb2801381bc03b871c00 \ + --hash=sha256:8b4d2fd2d34e5f8c9235ee787de5631a37a28402b15cb80814df973d2be54129 \ + --hash=sha256:8dddfbee2e68d26d0d7d7d9cb247b1fd4409241cce32d815a11d97ec2cfde179 \ + --hash=sha256:950ea81d57ef070665581b6e1b5f6a029306423cd1739c5b95fe78aa30db6b9d \ + --hash=sha256:956555e0603a4d38019ae6925711cb9dc43195c076a928accf7ea5d50bddfe53 \ + --hash=sha256:98b053943e5a0474ec0da309d2cb9d3f18ea57f8a2067c2ab7b5f763d1068380 \ + --hash=sha256:9968ab7e49b93ac6e1c3b2239732183152c9150f16308d30b66a372cffe3483c \ + --hash=sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a \ + --hash=sha256:9cb18a327b49c5c337f972b03682f6a49855525faaf3c0d3e9c96cd0fd8880a8 \ + --hash=sha256:a7b1b6353e36a7e50de2973a38d705c88ee93adcf120673cee7f45a4a3fa223a \ + --hash=sha256:a813ed7719bf45463c51779e6a98d0385fe905e48447526938a4b8337333d551 \ + --hash=sha256:aa1cce2ff3f8d953de38b76bf44602caeb69f101430208f64a10067f7cb4b1d3 \ + --hash=sha256:ad62a416ddcf863bf44bba76fbf6b53366ab0692e294f51cae4b5fbe0d246788 \ + --hash=sha256:aec3fc4b32ff82421274f5d205c559c51c840c8df66a78efd7f3612dd005a26a \ + --hash=sha256:b1185012870173de7ae33d370bd45b1cf5baee747ea4b97036b65f4e93016877 \ + --hash=sha256:b11e8fda06a7d69f15ebf542660b74466c2e51094800c1fb794f47ad4faeef17 \ + --hash=sha256:b64a85f40e154983960a4167d4c1d57a50c7f109b3d3264a3a984154e90a8454 \ + --hash=sha256:b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b \ + --hash=sha256:b89d0aaae2fe498c648f4c4795c084db535af5bd98ef942b2a3681fb74ce8645 \ + --hash=sha256:bc39ac66a7a9a3fbd6134fda43136b60ffde99c8f4501e64e0d2b24da137babf \ + --hash=sha256:c05ede731b03fb1b7591faca9389ade3267d2bddf1ad8882bb3f2cc5e101694f \ + --hash=sha256:c6342f54c67093cae5c0227eb0eb772fdb79f2a2c37a6eb278b9909ee06aa356 \ + --hash=sha256:c668b2f0d651605b58892644b0e302c7157f7159544227758c896982ef384b18 \ + --hash=sha256:c9b80cdf5cedba0e90d93fa5f9a333c4d65bd545cd669b71bb97ce2b703c9d73 \ + --hash=sha256:cfd73180400042a7c532d30c5e287bdd03c59ff9ee1b4c0316af0539e29dfe23 \ + --hash=sha256:d4cccbbc78717966f764cd3af4fb70276fa01fc7a2688af11c78901fa5c04f05 \ + --hash=sha256:d549420b8858885cea8838a727842249218b9c1da24dd517e25c9c7a948310a3 \ + --hash=sha256:d8200f16437b289a5bb927c6e184eccc3e8389bc0070fea4cd5b9e13c1757959 \ + --hash=sha256:e94aef2c639da4a960ad0db8e06471208d8589974953d78b61d345b4eb99e394 \ + --hash=sha256:fbde6962867ee75b48b0ee29b2b9372ec5d617799dbaf38e82dc0596f2f7738a \ + --hash=sha256:fe4d21ab149f15e4e6043dfb0de87e6e5f34ac176cde83060e9802981fca2ac2 \ + --hash=sha256:ffa6ce09a1c6a08e9667dd9c97aa0b14184e8d18f2a14b78b2a2328c9147f076 + # via pandas +pandas==3.1.0 \ + --hash=sha256:04ca88727dfa86837030c493d7512bb1c008aa4f5f14149c5f8d3ff446791340 \ + --hash=sha256:07807035ad66de0d833d68f9bee05801dbf08b88d56a233b0e9fd189372c3261 \ + --hash=sha256:0a28602573f50453b57802d282f9f8cb7663aa10201127ecbbe95ed4a41c64ce \ + --hash=sha256:0d2ca3b380f0f6cc788780430eb23762619730c11d53989f3982ec96835bfb9e \ + --hash=sha256:1340f823a1d189fae8365de314c4d3c2ea0173c61bca66cc76a91385c64d9d3d \ + --hash=sha256:224a9267e5fa70c16bf4f8a317435eef98387940ce6c8c611f1f68f2c3415e10 \ + --hash=sha256:2a8dc7780720b9e2182799ee86279d4cc7d1fd30b230ea1b202da5b6f7e14c3a \ + --hash=sha256:2bb3f8cc42fae0c0de5186778f4a4957656b9b3955aca48383abfdac44a83598 \ + --hash=sha256:37f88576f3449bfd86c1052f6b42274b61d7250c8d9dcc55f9dc4de7fe9883ae \ + --hash=sha256:388fad0c683ad061baabe23a76f43bf43c1d2c2fd46572b687082cfa195b7275 \ + --hash=sha256:39c67ed9e56ad3ac0727dc2ae6a1eef766988c762d5ce2e2a162613d81c8990b \ + --hash=sha256:3cba20fd269808e224d6212273008173978033a1dee0f3f217bfdbd58775a72c \ + --hash=sha256:3fe234bfb9999552dc54c45851752e56e163218d7663897dca950b4a7c32af39 \ + --hash=sha256:461b79db058639e446ddf90dd7f4804fae385acf0669f6e31321333e14cb6bc5 \ + --hash=sha256:4b5ce18612ca955e7a95d0bacc20120aedf97e2fbbebbe4de029fc24479e79b8 \ + --hash=sha256:54fed4a50b8321ff8d413b46b7e31a023cecefd06c7210c02aa980a9836428b2 \ + --hash=sha256:5611344f9f0ed51619ac7acdbf72b86f5043801a66e3f8730c8971257082e8ad \ + --hash=sha256:684ef3add5a92abb99c1622f2bd04b54b8843075a266e8240106ca94ce1d1237 \ + --hash=sha256:6c51273b89784c9d54020ed2f60e20524169aef349bddf6cba9f5e83b46077a7 \ + --hash=sha256:73b6cea22090b7123c78ee662a5108ff36616b0bec4fe47026f86fb04a49bce7 \ + --hash=sha256:743554ed96755ad1963397ccadbe9965d92db8777ca2ae8c1fe58bd9549d7f28 \ + --hash=sha256:756fd67995f3efb6b25f8eaea95c0e893f84e0068e2c53c5fdbb65275041d4db \ + --hash=sha256:84377fa82d0b40165379a0058d8a1a267ceb09e65160de6fcc75475d4d595884 \ + --hash=sha256:849d5019dead65944875ce21fbf71a2ca18e982164cd7367623cbda8567c3122 \ + --hash=sha256:84d73e0814a38711e4f2a24b272f748a0b4335169d2f9752407ae26bde22990f \ + --hash=sha256:89b66823678cdd53cf97187e7cd6861402bdd147d11a383706093136a4e83f45 \ + --hash=sha256:8e4de3fe8017e3445f1174707cd977e4e1154a704af2c060a1f648862f81acb9 \ + --hash=sha256:9433b5dc2f26ef9d9b56daaf4d41d078bf4372c5d88c42ed5b397a6eea80de16 \ + --hash=sha256:9b4e696ff70ed06e986feeec09356c491ce878d0df69a36c8e591b6a38afa29b \ + --hash=sha256:9ea2983fe096a3d997c8c4b9f26775fa51e7fb251b10634bd7f87e6fe3542d52 \ + --hash=sha256:a13199463b129363701a540c7b79586581b246b10b6a7ad682c41b268e7991d5 \ + --hash=sha256:a1d61e0eae5219cd024fcbc21304de759964b5642ddcdbb28d9a8ba442ab6901 \ + --hash=sha256:a7704e3d7367cfae71d3f8c95ee81e178c7a90bef8d6eac30df9815010722259 \ + --hash=sha256:a8c44d40cbb6d00128bf2d8f8bb66c8d561744b6d44686b5a1a01f3b3fd1245f \ + --hash=sha256:ac2f833557874a79fc0b5fac4c86e402ea6498beba8753dc29ca23d08481689a \ + --hash=sha256:b8c75dbb27c7d170b25c13dd35515da4bb92d40fe8bb3bd2c69a95de2218889b \ + --hash=sha256:bbb121205af7bc5e587b94a07b5f67b5ba687a918665b9e8748f21388a88b044 \ + --hash=sha256:bbfb087db83eb79ca348047961d367bdd4bf9820d90a5945f3652c7060da7363 \ + --hash=sha256:bfac077874a396b7ae61a307b27aa6f0256cf31c143b7f96ac622765754cc300 \ + --hash=sha256:c0bfec4d188fb0c05bba47b9f9def87624b8c545913aa3585a28f19f27ce847e \ + --hash=sha256:c1009f5b6819b81319cf177b4f01a2823ed54c81db8bd11cdf2375f42d064c02 \ + --hash=sha256:c1e084760f3b275eed5863407d025b933d25f3247515fcc84bf664405e8bb3d2 \ + --hash=sha256:c261c7a808982557c2917d40ffd27916769f26aacb4e6775c01e4467bcd34c3b \ + --hash=sha256:c4d9252b57bded194d09cd5ebf18d9b4edac238b16f56984ea8281d2ecfe39cd \ + --hash=sha256:cc5e4e12d287ea77bbf4c4d4bf47ed152d82a990d3a6794dee7c350f887532b5 \ + --hash=sha256:d3920d9b4de89fbef76ba6b863d58a330b5bfb720c9f8f58eb9b7da9b711d034 \ + --hash=sha256:d3e660af30a6683fa57a5a2969555df117e873eaaeb418ad3b2e59249405ee70 \ + --hash=sha256:d53387f8bbff4e7aa567619d4034ffdb1836db1e768a482851b904bbeeed8ef9 \ + --hash=sha256:dde9a55ecd3c4f9100ac50ab15e852e4c54fcf372d800e4495e0b2159a0122ab \ + --hash=sha256:de6d0fc96e8551b83f400bf212611a0b3a2b389594d20a2540dacc61b21e7bce \ + --hash=sha256:e4455cdcc03e42c17233184e8e4ea65c3ed2925d8b1d9d7dd8e6fced3f6e43c1 \ + --hash=sha256:e6f83e44846f9386e3593965bac017a54a5b5c611ea7b836ceab7f465bb4cb24 \ + --hash=sha256:e77112cfcabb3bef7281a72170670db9a5b31409a1cd3199939abb1fba67a7ad \ + --hash=sha256:ebbc0e652844e52e06d1525e5084ec1609158c086c8646aa6ccb34802102e533 \ + --hash=sha256:ecb9b2d8a817b1e1ba1fdb90c0abe9eab31b4fdf2562b7a56b86ac66ace17e49 \ + --hash=sha256:eee72bba8a72f0bf9522132e5c87cd10fce9d77440b50f65326b093ec9fb8d43 \ + --hash=sha256:f1a93c24175129bfc9922ef748ddae1d8137c743ed835e2615be329f606fed4d \ + --hash=sha256:f4772320b7735d96dbbef437129f840dec4175f8a18a2391bd1222ed8943825c + # via -r requirements.in +python-dateutil==4.0.0 \ + --hash=sha256:a5e246f79eb80475e691bf93862cead98decd8a498cf53ba0516fcdc0f50dc1c \ + --hash=sha256:d7295ff14fdc0f92f662100bfb6c551d160ca090f2939f1ec5b9794265c03f06 + # via pandas +pytz==2026.6 \ + --hash=sha256:3b0fc2727707663d6e517936f444fd8a8b240f1fc32f4bf7ee03274245945284 \ + --hash=sha256:7fed3fee8fd6bd4a2b7b75a8d56021d0624127bf69bd5691f115db929ac570b3 + # via -r requirements.in +tzdata==2026.6 \ + --hash=sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec \ + --hash=sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d + # via python-dateutil diff --git a/agents/react/scripts/smoke.ts b/agents/react/scripts/smoke.ts new file mode 100644 index 0000000..0f490fe --- /dev/null +++ b/agents/react/scripts/smoke.ts @@ -0,0 +1,234 @@ +/** + * `pnpm --filter @evorca/agent-react smoke` runs the agent's image on one + * task, as the runner would, against a scripted Messages API, and fails unless + * the agent writes all three files with the expected answer. + * + * The image runs with the runner's restrictions: a read-only root, the user + * `nobody`, a tmpfs at `/tmp`, no capabilities, and limits on memory, CPU, and + * processes. Its only network is an internal Docker network that reaches the + * mock and nothing else. The telemetry is a few megabytes of synthetic + * Telecom metrics in the schema that RCA-Agent's prompt describes. + * + * The scripted model runs three commands: one that reports the user, the + * environment, and the read-only root, a DuckDB query over the metrics, and a + * pandas script saved to `/tmp`. Then it submits an answer. + * + * @module + */ +import { execFile } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { parsePrediction } from "../../../packages/openrca/src/prediction.ts"; +import { bashCall, message, submitCall } from "../test/mock-api.ts"; + +const run = promisify(execFile); +const docker = (...args: string[]) => run("docker", args, { maxBuffer: 64 * 1024 * 1024 }); + +const agentDir = fileURLToPath(new URL("..", import.meta.url)); +const image = "evorca-agent-react:smoke"; +const network = `evorca-smoke-${process.pid}`; +const mockName = `${network}-mock`; +const agentName = `${network}-agent`; +const day = Date.UTC(2020, 3, 10, 16); +const faultStart = day + 5 * 60_000; + +const commands = { + sandbox: + 'id -u; echo "HOME=$HOME"; env | grep -c -E "^(ANTHROPIC|EVORCA)_"; cat /proc/self/oom_score_adj; touch /probe', + duckdb: `python3 -c "import duckdb; duckdb.sql(\\"SELECT cmdb_id, max(value) AS peak, arg_max(timestamp, value) AS at FROM read_csv('/data/2020_04_11/metric/metric_container.csv') WHERE name = 'container_cpu_used' GROUP BY cmdb_id ORDER BY peak DESC LIMIT 1\\").show()"`, + pandas: [ + "cat > first.py <<'EOF'", + "import pandas as pd, pytz", + "df = pd.read_csv('/data/2020_04_11/metric/metric_container.csv')", + "cpu = df[(df.name == 'container_cpu_used') & (df.cmdb_id == 'docker_003') & (df.value > 50)]", + "first = pd.to_datetime(cpu.timestamp.min(), unit='ms', utc=True).tz_convert(pytz.timezone('Asia/Shanghai'))", + "print(first.strftime('%Y-%m-%d %H:%M:%S'))", + "EOF", + "python3 first.py", + ].join("\n"), +}; + +const replies = [ + message([bashCall(commands.sandbox)]), + message([bashCall(commands.duckdb)]), + message([bashCall(commands.pandas)]), + message([ + submitCall([ + { occurrence_datetime: "2020-04-11 00:05:00", component: "docker_003", reason: "CPU fault" }, + ]), + ]), +]; + +const task = { + scenario: "telecom", + row: 0, + taskIndex: "task_7", + instruction: + "On April 11, 2020, within the time range of 00:00 to 00:30, there was one failure. Identify the root cause component, the root cause reason, and the root cause occurrence datetime.", + candidates: { + components: ["docker_001", "docker_002", "docker_003", "os_001"], + reasons: ["CPU fault", "network delay", "db connection limit"], + }, +}; + +function telemetry(): string { + const rows = ["itemid,name,bomc_id,timestamp,value,cmdb_id"]; + for (let minute = 0; minute < 24 * 60; minute += 1) { + const timestamp = day + minute * 60_000; + for (const container of ["docker_001", "docker_002", "docker_003"]) { + for (const [item, kpi] of [ + [999999996381330, "container_cpu_used"], + [999999996381331, "container_mem_used"], + [999999996381332, "container_fgc"], + [999999996381333, "container_thread_running"], + ] as const) { + const faulty = + container === "docker_003" && + kpi === "container_cpu_used" && + timestamp >= faultStart && + timestamp < faultStart + 10 * 60_000; + const value = faulty ? 95 + (minute % 3) : 20 + ((minute * 7 + kpi.length) % 13); + for (let second = 0; second < 60; second += 15) { + rows.push( + `${item},${kpi},ZJ-004-060,${timestamp + second * 1000},${value.toFixed(6)},${container}`, + ); + } + } + } + } + return `${rows.join("\n")}\n`; +} + +const work = await mkdtemp(join(tmpdir(), "evorca-smoke-")); +try { + console.log(`Building ${image}`); + await docker("build", "--quiet", "--tag", image, agentDir); + const [{ stdout: size }] = await Promise.all([ + docker("image", "ls", "--format", "{{.Size}}", image), + mkdir(join(work, "data", "2020_04_11", "metric"), { recursive: true }), + mkdir(join(work, "out")), + mkdir(join(work, "mock")), + ]); + console.log(`Image size: ${size.trim()}`); + const csv = telemetry(); + console.log(`Telemetry: ${(csv.length / 1e6).toFixed(1)} MB`); + await Promise.all([ + writeFile(join(work, "data", "2020_04_11", "metric", "metric_container.csv"), csv), + writeFile(join(work, "task.json"), JSON.stringify(task)), + writeFile(join(work, "mock", "replies.json"), JSON.stringify(replies)), + readFile(join(agentDir, "test", "mock-api.ts")).then((source) => + writeFile(join(work, "mock", "mock-api.ts"), source), + ), + ]); + const nodeImage = await baseImage(); + + await docker("network", "create", "--internal", network); + await docker( + "run", + "--detach", + "--name", + mockName, + "--network", + network, + "--read-only", + "--user", + "65534:65534", + "--mount", + `type=bind,src=${join(work, "mock")},dst=/mock,readonly`, + "--entrypoint", + "node", + nodeImage, + "/mock/mock-api.ts", + "/mock/replies.json", + "8080", + ); + + const started = Date.now(); + const agent = await docker( + "run", + "--rm", + "--init", + "--name", + agentName, + "--network", + network, + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=1g", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--user", + "65534:65534", + "--memory", + "3g", + "--cpus", + "1", + "--pids-limit", + "128", + "--mount", + `type=bind,src=${join(work, "task.json")},dst=/task.json,readonly`, + "--mount", + `type=bind,src=${join(work, "out")},dst=/out`, + "--mount", + `type=bind,src=${join(work, "data")},dst=/data,readonly`, + "--env", + `ANTHROPIC_BASE_URL=http://${mockName}:8080`, + "--env", + "ANTHROPIC_API_KEY=smoke-token", + "--env", + "EVORCA_SEED=1", + "--env", + "EVORCA_TASK_BUDGET_USD=1", + image, + ); + console.log(`Agent: ${agent.stdout.trim()} (${((Date.now() - started) / 1000).toFixed(1)} s)`); + + const out = (file: string) => readFile(join(work, "out", file), "utf8"); + const prediction = parsePrediction(await out("prediction.json")); + const summary = JSON.parse(await out("summary.json")); + const tools = (await out("trajectory.jsonl")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .filter((event) => event.type === "tool"); + const outputs = tools.map((event) => String(event.output)); + + for (const [i, output] of outputs.entries()) console.log(`\n--- tool ${i + 1}\n${output}`); + check("prediction.json is a valid prediction", prediction?.length === 1); + check( + "summary.json reports the submitted answer", + summary.stopCause === "submitted" && summary.steps === 4, + ); + check( + "the commands ran as nobody", + outputs[0]?.startsWith("65534\nHOME=/tmp\n0\n1000\n") === true, + ); + check("the root is read-only", outputs[0]?.includes("Read-only file system") === true); + check( + "DuckDB read the telemetry", + outputs[1]?.includes("docker_003") === true && outputs[1]?.includes("97.0") === true, + ); + check("pandas found the fault in UTC+8", outputs[2] === "2020-04-11 00:05:00\n"); + console.log("\nSmoke test passed"); +} finally { + await docker("rm", "--force", mockName, agentName).catch(() => {}); + await docker("network", "rm", network).catch(() => {}); + await rm(work, { recursive: true, force: true }); +} + +function check(claim: string, ok: boolean): void { + if (!ok) throw new Error(`Smoke test failed: ${claim}`); + console.log(`ok: ${claim}`); +} + +async function baseImage(): Promise { + const dockerfile = await readFile(join(agentDir, "Dockerfile"), "utf8"); + const from = /^FROM (\S+)/m.exec(dockerfile)?.[1]; + if (from === undefined) throw new Error("The Dockerfile has no FROM line"); + return from; +} From 4823de8d34b67e0c1847c4bf08655674853f2b69 Mon Sep 17 00:00:00 2001 From: Alexander Konstantinov Date: Tue, 17 Nov 2026 17:42:31 +0200 Subject: [PATCH 6/6] docs(agent-react): document the react agent --- .changeset/agent-react.md | 5 +++ README.md | 1 + agents/react/README.md | 81 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) create mode 100644 .changeset/agent-react.md create mode 100644 agents/react/README.md diff --git a/.changeset/agent-react.md b/.changeset/agent-react.md new file mode 100644 index 0000000..ced78c7 --- /dev/null +++ b/.changeset/agent-react.md @@ -0,0 +1,5 @@ +--- +"@evorca/agent-react": minor +--- + +The `react` agent investigates an OpenRCA task with Claude in a ReAct loop, with a bash tool over the telemetry and the domain knowledge of RCA-Agent's prompt. It writes its answer, its trajectory, and a summary of its steps and cost. diff --git a/README.md b/README.md index 23faacc..1444c58 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ pnpm check | Package | Purpose | | ------------------------------------------------- | --------------------------------------------------------------------------------- | | [`@evorca/agent-random`](agents/random/README.md) | The `random` agent, the chance-level floor of an evaluation. | +| [`@evorca/agent-react`](agents/react/README.md) | The `react` agent, the baseline without self-improvement. | | [`@evorca/cli`](packages/cli/README.md) | The `evorca` command, which `pnpm evorca` runs from the repository root. | | [`@evorca/core`](packages/core/README.md) | The domain types that every other package shares. | | [`@evorca/openrca`](packages/openrca/README.md) | Connects evorca to the [OpenRCA](https://github.com/microsoft/OpenRCA) benchmark. | diff --git a/agents/react/README.md b/agents/react/README.md new file mode 100644 index 0000000..462169c --- /dev/null +++ b/agents/react/README.md @@ -0,0 +1,81 @@ +# @evorca/agent-react + +The `react` agent, the baseline of an evorca evaluation. Claude investigates the telemetry in a ReAct loop with a bash tool, and reports the root causes with a `submit_answer` tool. The agent keeps no memory between tasks and does not improve itself, so it measures what the same model and tools do without evorca. + +## Scope + +- Answering an OpenRCA task from the telemetry. The model runs shell commands, Python, pandas, and DuckDB over `/data`, and submits one root cause for each failure. +- Giving the model the same domain knowledge that RCA-Agent, OpenRCA's published baseline, gives its model. `background.json` holds it, and a script makes it from RCA-Agent's prompts at the pinned OpenRCA commit. +- Recording each task for analysis. The agent writes every model response and every tool call as it happens, and a summary of the steps, the tokens, and the cost. + +## Non-goals + +- Domain hints beyond RCA-Agent's prompt. The prompt is not tuned on the answer keys, and the agent never reads `query.csv` or `record.csv`. +- Streaming, context compaction, or memory between tasks. +- Enforcing the budget. The model proxy refuses requests past the budget. The agent only wraps up before that point. + +## Public API + +The image runs `node src/main.ts`, as the [container contract](../../packages/cli/README.md#run-an-agent-in-a-container) describes. Node 26 runs the TypeScript files as they are, so the image has no build step. + +The container needs: + +- `/task.json`, read-only. The agent uses the `instruction`, the `taskIndex`, and, if present, the `candidates`. +- `/data`, read-only, the scenario's telemetry. +- `/out`, writable. +- `/tmp`, writable, where the bash session starts. Files there count toward the container's memory. +- A network that reaches the Messages API at `ANTHROPIC_BASE_URL`, with the key in `ANTHROPIC_API_KEY`. + +The environment: + +| Variable | Meaning | +| ----------------------------- | ------------------------------------------------------------------------------------- | +| `EVORCA_MODEL` | The model, `claude-sonnet-5` by default. The agent knows the prices of this model. | +| `EVORCA_EFFORT` | `low`, `medium`, `high`, `xhigh`, or `max`. The default is `high`. | +| `EVORCA_TASK_BUDGET_USD` | Optional. At 80% of this cost, the agent asks the model for its answer. | +| `EVORCA_TASK_TIMEOUT_SECONDS` | Optional. At 80% of this time, the agent asks the model for its answer. | +| `EVORCA_SEED` | Recorded in the summary. The model takes no seed, so two runs can answer differently. | + +The bash session gets the same environment without the `ANTHROPIC_` and `EVORCA_` variables. + +The agent writes three files to `/out`: + +- `prediction.json`, the answer as OpenRCA prediction JSON. The agent writes it when the model submits a valid answer, or, when the task ends without one, the valid fields of the last answer that the agent rejected. Without either, the file does not exist. +- `trajectory.jsonl`, one JSON line per event: the task with the system prompt and the tools, each message to the model, each response with its usage and stop reason, each tool call with its input and output, and API errors. +- `summary.json`, how the task ended (`stopCause`), whether the answer was submitted, the steps, the token usage, the estimated cost, the model, the effort, and the SHA-256 of the system prompt and the tools. + +The process exits with 0 whenever the loop ends, with or without an answer. It exits with 1 only if the agent itself fails. + +## The loop + +Each request sends the same system prompt and tools, and appends to the same history, so the prompt cache serves every earlier turn. Thinking is adaptive, and its summaries go to the trajectory. + +| Limit | Value | Reason | +| -------------------------- | ------------------- | ---------------------------------------------------------------------------------------- | +| Tool calls | 50 | RCA-Agent runs at most 25 controller steps, and each step can run code twice. | +| Time per command | 120 s | RCA-Agent stops a cell after 60 s on two CPUs. The container has one. | +| Output per command | 16,000 characters | Longer output keeps its start and its end. | +| Output tokens per response | 16,000, then 64,000 | A response cut off at the limit is discarded and asked again once with the larger limit. | + +After the last tool call, at 80% of the budget, or at 80% of the time, the agent refuses commands and asks for the answer. A model that ends its turn without an answer is asked once to submit. A refusal, a full context window, or an API error ends the task. The proxy's `evorca_budget_exhausted` 403 is one such error. + +## Update the background + +```sh +pnpm --filter @evorca/agent-react extract-background +``` + +The script checks out OpenRCA at the pinned commit in `.cache/` and writes `background.json`. It fails if RCA-Agent's prompts changed where it edits them, or if their candidates differ from `@evorca/openrca`. Its module comment lists each edit. + +## Dependencies + +The image installs pinned versions from two lockfiles, because it builds without the workspace: + +- `package-lock.json`, for `npm ci`. Make it again with `npm install --package-lock-only` in a copy of the folder outside the workspace. +- `requirements.txt`, Python packages with hashes. Make it again from `requirements.in` with the `uv pip compile` command at its top. + +## Tests + +`pnpm test` runs the loop against a local mock of the Messages API, through the real SDK. + +`pnpm --filter @evorca/agent-react smoke` builds the image and runs one task in a container with the runner's restrictions, against the mock on an internal Docker network, over synthetic telemetry. It needs Docker.