diff --git a/tests/providers/claude-cli-adapter.test.ts b/tests/providers/claude-cli-adapter.test.ts index e26c39db92f..2e62ec72565 100644 --- a/tests/providers/claude-cli-adapter.test.ts +++ b/tests/providers/claude-cli-adapter.test.ts @@ -12,6 +12,7 @@ import { type SpawnFn, } from "../../src/adapters/claude-cli/adapter"; import { CLAUDE_REPLAY_SYSTEM_PROMPT } from "../../src/adapters/claude-cli/stable-replay"; +import { satisfiesOpenAiStrictSchema } from "../../src/adapters/anthropic-output-schema"; import { baseScopedEnv } from "../../src/adapters/coding-agent/turn"; import { CLAUDE_CLI_PROFILE, clearClaudeCliBinaryCache } from "../../src/adapters/claude-cli/profiles"; import { effectiveAdapterContract, getAdapterDefinition } from "../../src/adapters/registry"; @@ -1047,6 +1048,11 @@ const textDelta = (text: string): string => describe("Issue #6469 — offline fixtures", () => { const systemInit = enc.encode('{"type":"system","subtype":"init","mcp_servers":[{"name":"opencodex","status":"connected"}]}\n'); + const schema = { + type: "object", + properties: { ocx_schema_probe_marker: { type: "string" } }, + required: ["ocx_schema_probe_marker"], + }; test("1. tool-use round trip: tool request, matching result, plain JSON final output", async () => { const firstChild = fakeChild([ @@ -1106,11 +1112,11 @@ describe("Issue #6469 — offline fixtures", () => { test("3. refusal: model refusal surfaces as an error, not fabricated output", async () => { const events = await run(adapterWith([ - enc.encode('{"type":"result","subtype":"success","is_error":true,"message":"refusal","usage":{"input_tokens":2,"output_tokens":0}}\n'), + enc.encode('{"type":"result","subtype":"success","is_error":true,"result":"refusal","usage":{"input_tokens":2,"output_tokens":0}}\n'), ]), parsed()); const err = events.find(e => e.type === "error"); expect(err).toBeDefined(); - expect(err).toMatchObject({ type: "error" }); + expect(err).toMatchObject({ type: "error", message: "refusal" }); expect(events.filter(e => e.type === "text_delta")).toEqual([]); }); @@ -1293,6 +1299,229 @@ describe("Issue #6469 — offline fixtures", () => { expect(promptFile).not.toBe(""); expect(existsSync(promptFile)).toBe(false); }); + + test("8a. valid Core JSON instance surfaced as plain text, not validated against schema", async () => { + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema } } }); + const validInstance = JSON.stringify({ ocx_schema_probe_marker: "present" }); + const stdout = [ + systemInit, + enc.encode(textDelta(validInstance)), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe(validInstance); + expect(JSON.parse(text)).toEqual({ ocx_schema_probe_marker: "present" }); + expect(events.some(e => e.type === "error")).toBe(false); + expect(events.at(-1)).toMatchObject({ type: "done", stopReason: "stop" }); + }); + + test("9. enum/const constraints ignored — output violating them still passed as plain text", async () => { + const enumSchema = { + type: "object", + properties: { + status: { type: "string", enum: ["ok", "fail"] }, + code: { type: "string", const: "exact" }, + }, + required: ["status", "code"], + }; + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: enumSchema } } }); + const violating = JSON.stringify({ status: "not-allowed", code: "wrong" }); + const stdout = [ + systemInit, + enc.encode(textDelta(violating)), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe(violating); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("10. additionalProperties:false ignored — extra properties still passed as plain text", async () => { + const strictSchema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + additionalProperties: false, + }; + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: strictSchema } } }); + const withExtra = JSON.stringify({ answer: "42", extra: "should-not-matter" }); + const stdout = [ + systemInit, + enc.encode(textDelta(withExtra)), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe(withExtra); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("11. missing schema field — textFormat.json_schema without schema treated as plain text", async () => { + const noSchema = parsed({ options: { textFormat: { type: "json_schema", name: "probe" } } }); + const stdout = [ + systemInit, + enc.encode(textDelta("plain output")), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), noSchema); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe("plain output"); + expect(events.some(e => e.type === "error")).toBe(false); + expect(events.at(-1)).toMatchObject({ type: "done", stopReason: "stop" }); + }); + + test("12. metadata/title-only schema does not certify unconstrained shape — still plain text", async () => { + const titleOnly = { + type: "object", + title: "Probe", + description: "Only metadata, no constraints", + }; + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: titleOnly } } }); + const unconstrained = JSON.stringify("goes"); + const stdout = [ + systemInit, + enc.encode(textDelta(unconstrained)), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe(unconstrained); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("13. unsupported/out-of-subset keywords ignored — schema not applied", async () => { + const weirdSchema = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + if: { properties: { answer: { const: "a" } } }, + then: { properties: { answer: { const: "b" } } }, + }; + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: weirdSchema } } }); + const stdout = [ + systemInit, + enc.encode(textDelta('{"answer":"a"}')), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe('{"answer":"a"}'); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("14. strict:true on uncertifiable schema ignored — no enforcement", async () => { + const uncertifiable = { + type: "object", + properties: { answer: { type: "string" } }, + required: ["answer"], + }; + expect(satisfiesOpenAiStrictSchema(uncertifiable)).toBe(false); + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema: uncertifiable, strict: true } } }); + const stdout = [ + systemInit, + enc.encode(textDelta('{"answer":42}')), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe('{"answer":42}'); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("15. oversized output surfaced as plain text without truncation", async () => { + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema } } }); + const big = "x".repeat(50_000); + const stdout = [ + systemInit, + enc.encode(textDelta(big)), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe(big); + expect(text.length).toBe(50_000); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("16. deeply nested JSON output surfaced as plain text without depth limits", async () => { + const schemaRequest = parsed({ options: { textFormat: { type: "json_schema", name: "probe", schema } } }); + let nested: any = { value: "leaf" }; + for (let i = 0; i < 200; i++) nested = { wrapper: nested }; + const deep = JSON.stringify(nested); + const stdout = [ + systemInit, + enc.encode(textDelta(deep)), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":2}}\n'), + ]; + const events = await run(adapterWith(stdout), schemaRequest); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe(deep); + expect(events.some(e => e.type === "error")).toBe(false); + }); + + test("17. tool turn with no final text — done with stopReason tool_use, no text_delta", async () => { + const stdout = [ + systemInit, + enc.encode(claudeCliToolFrame(0, "tu_1", CLAUDE_CLI_MCP_TOOL, '{"path":"a"}')), + enc.encode('{"type":"stream_event","event":{"type":"message_stop"}}\n'), + enc.encode('{"type":"result","subtype":"success","is_error":false,"usage":{"input_tokens":4,"output_tokens":0}}\n'), + ]; + const events = await run(adapterWith(stdout), claudeCliToolRequest()); + const starts = events.filter(e => e.type === "tool_call_start"); + expect(starts).toHaveLength(1); + expect(starts[0]).toMatchObject({ type: "tool_call_start", id: "tu_1", name: "read_file" }); + expect(events.filter(e => e.type === "text_delta")).toEqual([]); + expect(events.some(e => e.type === "error")).toBe(false); + expect(events.at(-1)).toMatchObject({ type: "done", stopReason: "tool_use" }); + }); + + test("18. terminal failure before complete output — is_error result ends turn with error", async () => { + const stdout = [ + systemInit, + enc.encode(textDelta("partial")), + enc.encode('{"type":"result","subtype":"success","is_error":true,"result":"model refused","usage":{"input_tokens":4,"output_tokens":1}}\n'), + ]; + const events = await run(adapterWith(stdout), parsed()); + const text = events.filter(e => e.type === "text_delta").map(e => (e as { text: string }).text).join(""); + expect(text).toBe("partial"); + const err = events.find(e => e.type === "error"); + expect(err).toBeDefined(); + expect(err).toMatchObject({ message: "model refused" }); + expect(events.some(e => e.type === "done")).toBe(false); + }); + + test("19. failure order — protocol error after tool calls does not replay or erase exposed calls", async () => { + let spawnCount = 0; + const stdout = [ + systemInit, + enc.encode(claudeCliToolFrame(0, "tu_1", CLAUDE_CLI_MCP_TOOL, '{"path":"a"}')), + enc.encode(claudeCliToolFrame(1, "tu_2", CLAUDE_CLI_MCP_TOOL, '{"path":"b"}')), + enc.encode("not-json\n"), + ]; + const events = await run(createClaudeCliAdapter(provider(), { + spawn: () => { spawnCount++; return fakeChild(stdout, { exitCode: 0 }) as unknown as ChildProcess; }, + which: () => "/opt/homebrew/bin/claude", + killGraceMs: 20, + }), claudeCliToolRequest()); + const starts = events.filter(e => e.type === "tool_call_start"); + expect(starts).toHaveLength(2); + expect(starts.map(e => (e as { id: string }).id)).toEqual(["tu_1", "tu_2"]); + expect(events.filter(e => e.type === "tool_call_delta").map(e => (e as { arguments: string }).arguments)) + .toEqual(['{"path":"a"}', '{"path":"b"}']); + expect(events.filter(e => e.type === "tool_call_end")).toHaveLength(2); + expect(events.filter(e => e.type === "tool_call_start" || e.type === "tool_call_delta" + || e.type === "tool_call_end" || e.type === "error").map(e => e.type)).toEqual([ + "tool_call_start", "tool_call_delta", "tool_call_end", + "tool_call_start", "tool_call_delta", "tool_call_end", "error", + ]); + const err = events.find(e => e.type === "error"); + expect(err).toBeDefined(); + expect(err).toMatchObject({ code: "protocol_error" }); + expect(events.some(e => e.type === "done")).toBe(false); + expect(spawnCount).toBe(1); + }); }); function adapterWith(stdout: Uint8Array[], exitCode = 0): ReturnType {