diff --git a/src/translator/openai-to-anthropic.test.ts b/src/translator/openai-to-anthropic.test.ts index 6d0b4cc..accd4c2 100644 --- a/src/translator/openai-to-anthropic.test.ts +++ b/src/translator/openai-to-anthropic.test.ts @@ -7,6 +7,7 @@ import { translateResponseAnthropicToOpenAI, anthropicUsageToOpenAI, } from "./openai-to-anthropic.js"; +import { responsesToChatCompletions } from "./responses-to-chat.js"; import { translateRequestAnthropicToOpenAI, translateResponseOpenAIToAnthropic, @@ -304,6 +305,37 @@ describe("translateRequestOpenAIToAnthropic", () => { expect(blocks[0].content).toBe("62°F and sunny"); }); + it("end-to-end: Responses tool output with input_image reaches Anthropic as a tool_result image block", () => { + // A client sending a read_file image result to /v1/responses goes through + // the full chain Responses→Chat→Anthropic, and every hop must preserve + // the image (previously dropped at the Responses→Chat hop). + const dataUrl = "data:image/png;base64,aGVsbG8="; + const responsesReq = { + model: "glm-4.6", + input: [ + { type: "message", role: "user", content: "look at the screenshot" }, + { type: "function_call", call_id: "call_img", name: "read_file", arguments: '{"path":"s.png"}' }, + { type: "function_call_output", call_id: "call_img", output: [ + { type: "input_text", text: "Read image file: s.png" }, + { type: "input_image", image_url: dataUrl }, + ] }, + ], + } as Parameters[0]; + const chatReq = responsesToChatCompletions(responsesReq).chatRequest; + const anthropicReq = translateRequestOpenAIToAnthropic(chatReq); + const toolResultMsg = anthropicReq.messages[2]; + expect(toolResultMsg.role).toBe("user"); + const blocks = toolResultMsg.content as any[]; + expect(blocks).toHaveLength(1); + expect(blocks[0].type).toBe("tool_result"); + expect(blocks[0].tool_use_id).toBe("call_img"); + const inner = blocks[0].content as any[]; + expect(inner).toEqual([ + { type: "text", text: "Read image file: s.png" }, + { type: "image", source: { type: "base64", media_type: "image/png", data: "aGVsbG8=" } }, + ]); + }); + it("coalesces consecutive role:'tool' messages into a single user message with multiple tool_result blocks", () => { const req: OpenAIChatRequest = { model: "glm-4.6", diff --git a/src/translator/responses-to-chat.test.ts b/src/translator/responses-to-chat.test.ts index 62bd8aa..8282881 100644 --- a/src/translator/responses-to-chat.test.ts +++ b/src/translator/responses-to-chat.test.ts @@ -158,6 +158,44 @@ describe("responsesToChatCompletions", () => { expect(msgs[2]).toEqual({ role: "tool", tool_call_id: "call_1", content: "sunny" }); }); + it("maps function_call_output content-part arrays (input_image) onto Chat tool-message image parts", () => { + // The Responses spec allows `output` to be a content-part array (OpenAI's + // own docs show {type:"input_image", image_url} inside a tool result so + // the model can look at a screenshot the tool returned) — stringly-typed + // handling dropped the image. + const dataUrl = "data:image/png;base64,aGVsbG8="; + const r = responsesToChatCompletions(baseReq({ + input: [ + { type: "message", role: "user", content: "read the png" }, + { type: "function_call", call_id: "call_img", name: "read_file", arguments: '{"path":"a.png"}' }, + { type: "function_call_output", call_id: "call_img", output: [ + { type: "input_text", text: "Read image file: a.png" }, + { type: "input_image", image_url: dataUrl }, + ] }, + ], + })); + const toolMsg = r.chatRequest.messages[2]; + expect(toolMsg.role).toBe("tool"); + expect(toolMsg.content).toEqual([ + { type: "text", text: "Read image file: a.png" }, + { type: "image_url", image_url: { url: dataUrl } }, + ]); + }); + + it("stringifies structured (non-array, non-string) function_call_output output", () => { + const r = responsesToChatCompletions(baseReq({ + input: [ + { type: "function_call", call_id: "c1", name: "get_data", arguments: "{}" }, + { type: "function_call_output", call_id: "c1", output: { rows: 3, ok: true } }, + ], + })); + expect(r.chatRequest.messages[1]).toEqual({ + role: "tool", + tool_call_id: "c1", + content: '{"rows":3,"ok":true}', + }); + }); + it("translates custom_tool_call + output with {input:...} argument shape", () => { const r = responsesToChatCompletions(baseReq({ input: [ diff --git a/src/translator/responses-to-chat.ts b/src/translator/responses-to-chat.ts index 4d86d2d..9fd8465 100644 --- a/src/translator/responses-to-chat.ts +++ b/src/translator/responses-to-chat.ts @@ -221,11 +221,11 @@ function buildMessagesFromItems(items: ResponsesInputItem[]): OpenAIMessage[] { } case "function_call_output": case "custom_tool_call_output": { - const fco = item as { call_id: string; output?: string }; + const fco = item as { call_id: string; output?: unknown }; out.push({ role: "tool", tool_call_id: fco.call_id, - content: fco.output ?? "", + content: toolOutputToChat(fco.output), }); pendingReasoning = ""; continue; @@ -302,6 +302,78 @@ function extractReasoningText(item: ResponsesInputItem): string { .join("\n"); } +/** + * Normalize a `function_call_output` / `custom_tool_call_output` `output` into + * a Chat tool-message `content` value. + * + * The Responses spec allows the output to be a content-part array (OpenAI's + * own docs show `{type:"input_image", image_url}` inside a tool result so the + * model can look at a screenshot the tool returned); upstream Chat dialects + * carry tool images as `image_url` parts on `role:"tool"` — and the downstream + * Chat→Anthropic translator maps those onto Anthropic `tool_result` image + * blocks. Clients sending read_file image results in the array form had every + * image silently dropped when only strings were accepted here. + * + * Non-array, non-string outputs (structured JSON objects) stringify — matches + * OpenAI Chat, where tool-message content is stringly typed. + */ +function toolOutputToChat(output: unknown): string | OpenAIContentPart[] { + if (typeof output === "string") return output; + if (Array.isArray(output)) { + const chatParts = outputPartsToChat(output); + if (chatParts.every((p) => p.type === "text")) return chatParts.map((p) => p.text ?? "").join(""); + return chatParts; + } + if (output === undefined || output === null) return ""; + try { + return JSON.stringify(output); + } catch { + return String(output); + } +} + +/** + * Convert content-part objects appearing inside a tool output array to Chat + * content parts. Accepts the Responses dialect (`input_image` with + * `image_url: string | {url}`, `input_text`, `input_video`, `input_file`) + * and the already-Chat-shaped parts some clients emit verbatim. Anything + * unknown degrades to a text part carrying its JSON — never dropped, so the + * model can still see what the tool returned. + */ +function outputPartsToChat(parts: unknown[]): OpenAIContentPart[] { + const out: OpenAIContentPart[] = []; + for (const part of parts) { + if (part === null || typeof part !== "object") { + out.push({ type: "text", text: String(part) }); + continue; + } + const p = part as ResponsesContentPart; + if ((p.type === "input_text" || p.type === "output_text" || p.type === "text") && typeof p.text === "string") { + out.push({ type: "text", text: p.text }); + } else if (p.type === "input_image" || p.type === "image_url") { + const url = typeof p.image_url === "string" ? p.image_url : p.image_url?.url; + if (!url) continue; + out.push({ type: "image_url", image_url: { url } }); + } else if (p.type === "input_video" || p.type === "video_url") { + const url = typeof p.video_url === "string" + ? p.video_url + : p.video_url?.url ?? (typeof p.data === "string" ? p.data : undefined); + if (!url) continue; + out.push({ type: "video_url", video_url: { url } }); + } else if (p.type === "input_file" || p.type === "file") { + const fileData = p.file_data ?? p.file?.file_data; + if (!fileData) continue; + out.push({ + type: "file", + file: { filename: p.filename ?? p.file?.filename, file_data: fileData }, + }); + } else { + out.push({ type: "text", text: JSON.stringify(p) }); + } + } + return out; +} + /** Convert Responses content parts to a Chat `content` value (string or parts[]). */ function contentPartsToChat( content: ResponsesContentPart[] | string | undefined,