Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions plugins/tracing/src/trace.ts
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ function assistantMessage(step: ModelStep): ChatMlMessage {
id: tc.callId,
type: "function" as const,
function: {
name: toolObservationName(tc),
name: tc.name || "tool",
...(tc.args !== undefined ? { arguments: toText(tc.args) } : {}),
},
})),
Expand All @@ -350,7 +350,7 @@ function toolMessages(step: ModelStep): ChatMlMessage[] {
return step.toolCalls.map((tc) => ({
role: "tool" as const,
tool_call_id: tc.callId,
name: toolObservationName(tc),
name: tc.name || "tool",
content: tc.output != null ? toText(tc.output) : (tc.error ?? ""),
...(tc.error ? { is_error: true as const } : {}),
}));
Expand Down
108 changes: 107 additions & 1 deletion plugins/tracing/test/trace.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,40 @@ describe("convertRollout", () => {
expect(meta("changed_this_turn")).toBe("true");
});

it.each(["", undefined])("keeps the history fallback for tool name %j", async (name) => {
const dir = stageFixtures();
const file = path.join(dir, "rollout-basic-main.jsonl");
const lines = fs
.readFileSync(file, "utf8")
.trim()
.split("\n")
.map((line) => JSON.parse(line));
// Malformed rollouts still need usable names in both halves of the tool exchange.
lines.find((line) => line.payload.type === "function_call").payload.name = name;
fs.writeFileSync(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n");
await convertRollout(file, { config: baseConfig });

const generations = exporter
.getFinishedSpans()
.filter((s) => obsType(s) === "generation")
.sort((a, b) => startMs(a) - startMs(b));
const history = JSON.parse(attr(generations[1], "langfuse.observation.input"));
expect(history).toContainEqual(
expect.objectContaining({
role: "assistant",
tool_calls: [
expect.objectContaining({
id: "call-1",
function: expect.objectContaining({ name: "tool" }),
}),
],
}),
);
expect(history).toContainEqual(
expect.objectContaining({ role: "tool", tool_call_id: "call-1", name: "tool" }),
);
});

it("carries earlier turns of the thread into a later turn's generation input", async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lf-codex-hist-"));
const day = path.join(dir, "sessions", "2026", "06", "03");
Expand Down Expand Up @@ -783,7 +817,23 @@ describe("convertRollout", () => {

it("captures web search, local shell, and MCP tool calls with specific names", async () => {
const dir = stageFixtures();
await convertRollout(path.join(dir, "rollout-tools-main.jsonl"), { config: baseConfig });
const file = path.join(dir, "rollout-tools-main.jsonl");
const lines = fs
.readFileSync(file, "utf-8")
.trim()
.split("\n")
.map((line) => JSON.parse(line));
lines.splice(
lines.findIndex((line) => line.payload.role === "assistant"),
0,
{
timestamp: "2026-06-03T12:00:04.700Z",
type: "event_msg",
payload: { type: "token_count" },
},
);
fs.writeFileSync(file, lines.map((line) => JSON.stringify(line)).join("\n") + "\n");
await convertRollout(file, { config: baseConfig });

const spans = exporter.getFinishedSpans();
const toolNames = spans
Expand All @@ -798,6 +848,30 @@ describe("convertRollout", () => {

const shell = spans.find((s) => s.name === "local_shell")!;
expect(attr(shell, "langfuse.observation.output")).toContain("clean");

const generations = spans
.filter((s) => obsType(s) === "generation")
.sort((a, b) => startMs(a) - startMs(b));
expect(generations).toHaveLength(2);
const history = JSON.parse(attr(generations[1], "langfuse.observation.input"));
expect(history).toContainEqual(
expect.objectContaining({
role: "assistant",
tool_calls: expect.arrayContaining([
expect.objectContaining({
id: "call-mcp",
function: expect.objectContaining({ name: "linear__create_issue" }),
}),
]),
}),
);
expect(history).toContainEqual(
expect.objectContaining({
role: "tool",
tool_call_id: "call-mcp",
name: "linear__create_issue",
}),
);
});

it("does not re-export a settings event between turns on every invocation", async () => {
Expand Down Expand Up @@ -975,6 +1049,38 @@ describe("deterministic trace ids (trace_seed)", () => {
});

describe("skill observations", () => {
it.each([true, false])(
"keeps real tool names in generation history with skill_tags=%s",
async (skill_tags) => {
const dir = stageFixtures();
await convertRollout(path.join(dir, "rollout-skills-main.jsonl"), {
config: { ...baseConfig, skill_tags },
});

const spans = exporter.getFinishedSpans();
const generations = spans
.filter((s) => obsType(s) === "generation")
.sort((a, b) => startMs(a) - startMs(b));
expect(generations).toHaveLength(2);
const history = JSON.parse(attr(generations[1], "langfuse.observation.input"));
expect(history).toContainEqual(
expect.objectContaining({
role: "assistant",
tool_calls: expect.arrayContaining([
expect.objectContaining({
id: "call-doc",
function: expect.objectContaining({ name: "exec_command" }),
}),
]),
}),
);
expect(history).toContainEqual(
expect.objectContaining({ role: "tool", tool_call_id: "call-doc", name: "exec_command" }),
);
expect(spans.find((s) => s.name === "skill:bug-mentor")).toBeDefined();
},
);

it("names a skill-loading command after the skill, keeping its own timing", async () => {
const dir = stageFixtures();
await convertRollout(path.join(dir, "rollout-skills-main.jsonl"), { config: baseConfig });
Expand Down