diff --git a/plugins/tracing/src/parse.ts b/plugins/tracing/src/parse.ts index 5c70645..91f9457 100644 --- a/plugins/tracing/src/parse.ts +++ b/plugins/tracing/src/parse.ts @@ -381,6 +381,20 @@ export function parseSession(lines: RolloutLine[]): { if (p.item.kind === "started" && typeof p.item.agent_thread_id === "string") { recordSubagentThread(p.item.agent_thread_id, p.item.id); } + } else if (et === "item_completed" && p.item?.type === "CommandExecution") { + const item = p.item; + if ( + item.source !== "user_shell" && + typeof item.id === "string" && + (item.status === "failed" || item.status === "declined") + ) { + // Nested commands have their own IDs; do not attribute their failures to the wrapper. + const tc = toolCallsById.get(item.id); + if (tc) + tc.error ??= + item.aggregated_output || + (typeof item.exit_code === "number" ? `Exit code: ${item.exit_code}` : item.status); + } } else if (et === "agent_message" && typeof p.message === "string") { turn!.lastAgentMessage = p.message; } else if (et === "token_count") { diff --git a/plugins/tracing/src/types.ts b/plugins/tracing/src/types.ts index 298bcc8..eb70ead 100644 --- a/plugins/tracing/src/types.ts +++ b/plugins/tracing/src/types.ts @@ -133,6 +133,10 @@ export type EventMsgPayload = { kind?: string; agent_thread_id?: string; content?: MessageContentPart[]; + source?: string; + status?: string; + aggregated_output?: string | null; + exit_code?: number | null; } | null; new_thread_id?: string | null; event_id?: string; diff --git a/plugins/tracing/test/fixtures/sessions/2026/06/03/rollout-command-failed.jsonl b/plugins/tracing/test/fixtures/sessions/2026/06/03/rollout-command-failed.jsonl new file mode 100644 index 0000000..4460119 --- /dev/null +++ b/plugins/tracing/test/fixtures/sessions/2026/06/03/rollout-command-failed.jsonl @@ -0,0 +1,9 @@ +{"timestamp":"2026-06-03T14:00:00.000Z","type":"session_meta","payload":{"id":"sess-command-failed","cli_version":"0.149.0","model_provider":"openai"}} +{"timestamp":"2026-06-03T14:00:01.000Z","type":"event_msg","payload":{"type":"task_started","turn_id":"turn-command-failed"}} +{"timestamp":"2026-06-03T14:00:01.100Z","type":"event_msg","payload":{"type":"user_message","message":"Run the check"}} +{"timestamp":"2026-06-03T14:00:02.000Z","type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"call-exec","arguments":"{\"cmd\":\"check\"}"}} +{"timestamp":"2026-06-03T14:00:02.100Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-exec","output":"Process running with session ID 123"}} +{"timestamp":"2026-06-03T14:00:03.000Z","type":"response_item","payload":{"type":"function_call","name":"write_stdin","call_id":"call-stdin","arguments":"{\"session_id\":123,\"chars\":\"\"}"}} +{"timestamp":"2026-06-03T14:00:03.100Z","type":"response_item","payload":{"type":"function_call_output","call_id":"call-stdin","output":"Process exited with code 1"}} +{"timestamp":"2026-06-03T14:00:03.200Z","type":"event_msg","payload":{"type":"item_completed","turn_id":"turn-command-failed","item":{"type":"CommandExecution","id":"call-exec","command":["check"],"cwd":"/repo","source":"unified_exec_startup","status":"failed","aggregated_output":"check failed","exit_code":1}}} +{"timestamp":"2026-06-03T14:00:04.000Z","type":"event_msg","payload":{"type":"task_complete","turn_id":"turn-command-failed"}} diff --git a/plugins/tracing/test/parse.test.ts b/plugins/tracing/test/parse.test.ts index 38b50ea..65a5f63 100644 --- a/plugins/tracing/test/parse.test.ts +++ b/plugins/tracing/test/parse.test.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { parseSession } from "../src/parse.js"; -import type { RolloutLine } from "../src/types.js"; +import type { EventMsgPayload, RolloutLine } from "../src/types.js"; const fixturesDir = path.join( path.dirname(fileURLToPath(import.meta.url)), @@ -67,6 +67,81 @@ describe("parseSession", () => { expect(step2.toolCalls).toHaveLength(0); }); + it.each([ + { status: "failed", aggregated_output: "check failed", exit_code: 1, error: "check failed" }, + { status: "failed", aggregated_output: "", exit_code: 2, error: "Exit code: 2" }, + { status: "failed", aggregated_output: "", exit_code: null, error: "failed" }, + { status: "declined", aggregated_output: "", exit_code: null, error: "declined" }, + ])("records CommandExecution $status with error '$error'", ({ error, ...details }) => { + const lines = loadFixture("rollout-command-failed.jsonl"); + const completion = lines.find((line) => line.payload.type === "item_completed")!; + Object.assign((completion.payload as EventMsgPayload).item!, details); + const { turns } = parseSession(lines); + const tools = turns[0].steps.flatMap((step) => step.toolCalls); + expect(tools).toHaveLength(2); + expect(tools[0]).toMatchObject({ + callId: "call-exec", + error, + output: "Process running with session ID 123", + endTime: Date.parse("2026-06-03T14:00:02.100Z"), + }); + expect(tools[1].error).toBeUndefined(); + }); + + it.each([ + { source: "user_shell" }, + { id: "unknown-call" }, + { status: "completed" }, + { status: "in_progress" }, + ])("ignores unrelated or non-failing command completions: %j", (details) => { + const lines = loadFixture("rollout-command-failed.jsonl"); + const completion = lines.find((line) => line.payload.type === "item_completed")!; + Object.assign((completion.payload as EventMsgPayload).item!, details); + const tools = parseSession(lines).turns[0].steps.flatMap((step) => step.toolCalls); + expect(tools).toHaveLength(2); + expect(tools.every((tool) => tool.error === undefined)).toBe(true); + }); + + it("preserves an existing error when a command completion arrives", () => { + const lines = loadFixture("rollout-command-failed.jsonl"); + lines.splice(7, 0, { + timestamp: "2026-06-03T14:00:03.150Z", + type: "event_msg", + payload: { + type: "exec_command_end", + call_id: "call-exec", + status: "failed", + error: "original error", + }, + }); + const tools = parseSession(lines).turns[0].steps.flatMap((step) => step.toolCalls); + expect(tools[0].error).toBe("original error"); + }); + + it("does not attach a previous turn's command failure to the current turn", () => { + const lines = loadFixture("rollout-command-failed.jsonl"); + lines.splice( + 5, + 0, + { + timestamp: "2026-06-03T14:00:02.200Z", + type: "event_msg", + payload: { type: "task_complete", turn_id: "turn-command-failed" }, + }, + { + timestamp: "2026-06-03T14:00:02.300Z", + type: "event_msg", + payload: { type: "task_started", turn_id: "turn-next" }, + }, + ); + (lines[lines.length - 1].payload as EventMsgPayload).turn_id = "turn-next"; + const { turns } = parseSession(lines); + expect(turns).toHaveLength(2); + const tools = turns.flatMap((turn) => turn.steps.flatMap((step) => step.toolCalls)); + expect(tools.map((tool) => tool.callId)).toEqual(["call-exec", "call-stdin"]); + expect(tools.every((tool) => tool.error === undefined)).toBe(true); + }); + it("captures an explicitly invoked skill without letting it leak into the prompt", () => { const { turns } = parseSession(loadFixture("rollout-skills-main.jsonl")); diff --git a/plugins/tracing/test/trace.test.ts b/plugins/tracing/test/trace.test.ts index fd8c1be..8ad0ee4 100644 --- a/plugins/tracing/test/trace.test.ts +++ b/plugins/tracing/test/trace.test.ts @@ -152,6 +152,20 @@ describe("convertRollout", () => { expect(attr(generations[0], "langfuse.observation.output")).toContain("call-1"); }); + it("marks the matching command as ERROR from item_completed without legacy end events", async () => { + const dir = stageFixtures(); + await convertRollout(path.join(dir, "rollout-command-failed.jsonl"), { config: baseConfig }); + const tools = exporter + .getFinishedSpans() + .filter((s) => obsType(s) === "tool") + .sort((a, b) => startMs(a) - startMs(b)); + expect(tools).toHaveLength(2); + expect(attr(tools[0], "langfuse.observation.metadata.codex.call_id")).toBe("call-exec"); + expect(attr(tools[0], "langfuse.observation.level")).toBe("ERROR"); + expect(attr(tools[0], "langfuse.observation.status_message")).toBe("check failed"); + expect(attr(tools[1], "langfuse.observation.level")).not.toBe("ERROR"); + }); + it("ends a generation at the tool call it emitted, not at the end of the step", async () => { const dir = stageFixtures(); await convertRollout(path.join(dir, "rollout-basic-main.jsonl"), { config: baseConfig });