fix(translator): accept content-part arrays in function_call_output so tool images reach the model

src/translator/openai-to-anthropic.test.ts

@@ -7,6 +7,7 @@ import {
   translateResponseAnthropicToOpenAI,
   anthropicUsageToOpenAI,
 } from "./openai-to-anthropic.js";
+import { responsesToChatCompletions } from "./responses-to-chat.js";
 import {
   translateRequestAnthropicToOpenAI,
   translateResponseOpenAIToAnthropic,
@@ -304,6 +305,37 @@ describe("translateRequestOpenAIToAnthropic", () => {
     expect(blocks[0].content).toBe("62°F and sunny");
   });
 
+  it("end-to-end: Responses tool output with input_image reaches Anthropic as a tool_result image block", () => {
+    // A client sending a read_file image result to /v1/responses goes through
+    // the full chain Responses→Chat→Anthropic, and every hop must preserve
+    // the image (previously dropped at the Responses→Chat hop).
+    const dataUrl = "data:image/png;base64,aGVsbG8=";
+    const responsesReq = {
+      model: "glm-4.6",
+      input: [
+        { type: "message", role: "user", content: "look at the screenshot" },
+        { type: "function_call", call_id: "call_img", name: "read_file", arguments: '{"path":"s.png"}' },
+        { type: "function_call_output", call_id: "call_img", output: [
+          { type: "input_text", text: "Read image file: s.png" },
+          { type: "input_image", image_url: dataUrl },
+        ] },
+      ],
+    } as Parameters<typeof responsesToChatCompletions>[0];
+    const chatReq = responsesToChatCompletions(responsesReq).chatRequest;
+    const anthropicReq = translateRequestOpenAIToAnthropic(chatReq);
+    const toolResultMsg = anthropicReq.messages[2];
+    expect(toolResultMsg.role).toBe("user");
+    const blocks = toolResultMsg.content as any[];
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0].type).toBe("tool_result");
+    expect(blocks[0].tool_use_id).toBe("call_img");
+    const inner = blocks[0].content as any[];
+    expect(inner).toEqual([
+      { type: "text", text: "Read image file: s.png" },
+      { type: "image", source: { type: "base64", media_type: "image/png", data: "aGVsbG8=" } },
+    ]);
+  });
+
   it("coalesces consecutive role:'tool' messages into a single user message with multiple tool_result blocks", () => {
     const req: OpenAIChatRequest = {
       model: "glm-4.6",

src/translator/responses-to-chat.test.ts

@@ -158,6 +158,44 @@ describe("responsesToChatCompletions", () => {
     expect(msgs[2]).toEqual({ role: "tool", tool_call_id: "call_1", content: "sunny" });
   });
 
+  it("maps function_call_output content-part arrays (input_image) onto Chat tool-message image parts", () => {
+    // The Responses spec allows `output` to be a content-part array (OpenAI's
+    // own docs show {type:"input_image", image_url} inside a tool result so
+    // the model can look at a screenshot the tool returned) — stringly-typed
+    // handling dropped the image.
+    const dataUrl = "data:image/png;base64,aGVsbG8=";
+    const r = responsesToChatCompletions(baseReq({
+      input: [
+        { type: "message", role: "user", content: "read the png" },
+        { type: "function_call", call_id: "call_img", name: "read_file", arguments: '{"path":"a.png"}' },
+        { type: "function_call_output", call_id: "call_img", output: [
+          { type: "input_text", text: "Read image file: a.png" },
+          { type: "input_image", image_url: dataUrl },
+        ] },
+      ],
+    }));
+    const toolMsg = r.chatRequest.messages[2];
+    expect(toolMsg.role).toBe("tool");
+    expect(toolMsg.content).toEqual([
+      { type: "text", text: "Read image file: a.png" },
+      { type: "image_url", image_url: { url: dataUrl } },
+    ]);
+  });
+
+  it("stringifies structured (non-array, non-string) function_call_output output", () => {
+    const r = responsesToChatCompletions(baseReq({
+      input: [
+        { type: "function_call", call_id: "c1", name: "get_data", arguments: "{}" },
+        { type: "function_call_output", call_id: "c1", output: { rows: 3, ok: true } },
+      ],
+    }));
+    expect(r.chatRequest.messages[1]).toEqual({
+      role: "tool",
+      tool_call_id: "c1",
+      content: '{"rows":3,"ok":true}',
+    });
+  });
+
   it("translates custom_tool_call + output with {input:...} argument shape", () => {
     const r = responsesToChatCompletions(baseReq({
       input: [

src/translator/responses-to-chat.ts

@@ -221,11 +221,11 @@ function buildMessagesFromItems(items: ResponsesInputItem[]): OpenAIMessage[] {
       }
       case "function_call_output":
       case "custom_tool_call_output": {
-        const fco = item as { call_id: string; output?: string };
+        const fco = item as { call_id: string; output?: unknown };
         out.push({
           role: "tool",
           tool_call_id: fco.call_id,
-          content: fco.output ?? "",
+          content: toolOutputToChat(fco.output),
         });
         pendingReasoning = "";
         continue;
@@ -302,6 +302,78 @@ function extractReasoningText(item: ResponsesInputItem): string {
     .join("\n");
 }
 
+/**
+ * Normalize a `function_call_output` / `custom_tool_call_output` `output` into
+ * a Chat tool-message `content` value.
+ *
+ * The Responses spec allows the output to be a content-part array (OpenAI's
+ * own docs show `{type:"input_image", image_url}` inside a tool result so the
+ * model can look at a screenshot the tool returned); upstream Chat dialects
+ * carry tool images as `image_url` parts on `role:"tool"` — and the downstream
+ * Chat→Anthropic translator maps those onto Anthropic `tool_result` image
+ * blocks. Clients sending read_file image results in the array form had every
+ * image silently dropped when only strings were accepted here.
+ *
+ * Non-array, non-string outputs (structured JSON objects) stringify — matches
+ * OpenAI Chat, where tool-message content is stringly typed.
+ */
+function toolOutputToChat(output: unknown): string | OpenAIContentPart[] {
+  if (typeof output === "string") return output;
+  if (Array.isArray(output)) {
+    const chatParts = outputPartsToChat(output);
+    if (chatParts.every((p) => p.type === "text")) return chatParts.map((p) => p.text ?? "").join("");
+    return chatParts;
+  }
+  if (output === undefined || output === null) return "";
+  try {
+    return JSON.stringify(output);
+  } catch {
+    return String(output);
+  }
+}
+
+/**
+ * Convert content-part objects appearing inside a tool output array to Chat
+ * content parts. Accepts the Responses dialect (`input_image` with
+ * `image_url: string | {url}`, `input_text`, `input_video`, `input_file`)
+ * and the already-Chat-shaped parts some clients emit verbatim. Anything
+ * unknown degrades to a text part carrying its JSON — never dropped, so the
+ * model can still see what the tool returned.
+ */
+function outputPartsToChat(parts: unknown[]): OpenAIContentPart[] {
+  const out: OpenAIContentPart[] = [];
+  for (const part of parts) {
+    if (part === null || typeof part !== "object") {
+      out.push({ type: "text", text: String(part) });
+      continue;
+    }
+    const p = part as ResponsesContentPart;
+    if ((p.type === "input_text" || p.type === "output_text" || p.type === "text") && typeof p.text === "string") {
+      out.push({ type: "text", text: p.text });
+    } else if (p.type === "input_image" || p.type === "image_url") {
+      const url = typeof p.image_url === "string" ? p.image_url : p.image_url?.url;
+      if (!url) continue;
+      out.push({ type: "image_url", image_url: { url } });
+    } else if (p.type === "input_video" || p.type === "video_url") {
+      const url = typeof p.video_url === "string"
+        ? p.video_url
+        : p.video_url?.url ?? (typeof p.data === "string" ? p.data : undefined);
+      if (!url) continue;
+      out.push({ type: "video_url", video_url: { url } });
+    } else if (p.type === "input_file" || p.type === "file") {
+      const fileData = p.file_data ?? p.file?.file_data;
+      if (!fileData) continue;
+      out.push({
+        type: "file",
+        file: { filename: p.filename ?? p.file?.filename, file_data: fileData },
+      });
+    } else {
+      out.push({ type: "text", text: JSON.stringify(p) });
+    }
+  }
+  return out;
+}
+
 /** Convert Responses content parts to a Chat `content` value (string or parts[]). */
 function contentPartsToChat(
   content: ResponsesContentPart[] | string | undefined,