From 55ea14fdd8c0727f4e17da7377c48b5bc8be47f0 Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:18:46 +0800 Subject: [PATCH 1/2] fix: preserve Responses message output text --- js/oai.test.ts | 113 +++++++++++++++++++++- js/oai.ts | 8 +- py/autoevals/oai.py | 6 +- py/autoevals/test_responses_conversion.py | 90 +++++++++++++++++ 4 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 py/autoevals/test_responses_conversion.py diff --git a/js/oai.test.ts b/js/oai.test.ts index 5adf9ba7..19c7ef30 100644 --- a/js/oai.test.ts +++ b/js/oai.test.ts @@ -10,7 +10,12 @@ import { test, vi, } from "vitest"; -import { buildOpenAIClient, init, getDefaultModel } from "./oai"; +import { + buildOpenAIClient, + cachedChatCompletion, + init, + getDefaultModel, +} from "./oai"; import { setupServer } from "msw/node"; @@ -74,7 +79,113 @@ const MOCK_OPENAI_COMPLETION_RESPONSE = { }, }; +const textMessage = (...texts: string[]) => ({ + id: "msg-test", + type: "message", + role: "assistant", + status: "completed", + content: texts.map((text) => ({ + type: "output_text", + text, + annotations: [], + })), +}); + +const responseToolCalls = [ + { + type: "function_call", + id: "fc-test", + call_id: "call-0", + name: "select_choice", + arguments: '{"choice":"A"}', + }, +]; +const requestTools: OpenAI.Chat.Completions.ChatCompletionTool[] = [ + { + type: "function", + function: { + name: "select_choice", + parameters: { + type: "object", + properties: { choice: { type: "string" } }, + }, + }, + }, +]; +const chatToolCalls = [ + { + id: "call-0", + type: "function", + function: { name: "select_choice", arguments: '{"choice":"A"}' }, + }, +]; + describe("OAI", () => { + test.each([ + { + name: "message text", + output: [textMessage("first ", "second"), textMessage(" third")], + content: "first second third", + toolCalls: [], + }, + { + name: "text with tools", + output: [textMessage("text with tools"), ...responseToolCalls], + content: "text with tools", + toolCalls: chatToolCalls, + }, + { + name: "tools only", + output: responseToolCalls, + content: null, + toolCalls: chatToolCalls, + }, + { + name: "top-level output_text", + output: [{ type: "output_text", text: "legacy text" }], + content: "legacy text", + toolCalls: [], + }, + { + name: "top-level text", + output: [{ type: "text", content: "legacy text" }], + content: "legacy text", + toolCalls: [], + }, + ])("converts Responses $name", async ({ output, content, toolCalls }) => { + const respond = vi.fn(() => + HttpResponse.json({ + id: "resp-test", + object: "response", + created_at: 1234567890, + model: "gpt-5-mini", + status: "completed", + output, + }), + ); + server.use(http.post("https://api.openai.com/v1/responses", respond)); + + const response = await cachedChatCompletion( + { + model: "gpt-5-mini", + messages: [{ role: "user", content: "test" }], + ...(toolCalls.length > 0 ? { tools: requestTools } : {}), + }, + { + client: new OpenAI({ + apiKey: "test-api-key", + baseURL: "https://api.openai.com/v1", + maxRetries: 0, + }), + }, + ); + + expect(respond).toHaveBeenCalledTimes(1); + const message = response.choices[0].message; + expect(message.content).toBe(content); + expect(message.tool_calls ?? []).toEqual(toolCalls); + }); + test("should use Azure OpenAI", async () => { server.use( http.post( diff --git a/js/oai.ts b/js/oai.ts index 6da1dee0..a614f1ef 100644 --- a/js/oai.ts +++ b/js/oai.ts @@ -409,7 +409,13 @@ export async function cachedChatCompletion( if (response.output && Array.isArray(response.output)) { for (const item of response.output) { - if (item.type === "output_text" || item.type === "text") { + if (item.type === "message") { + for (const part of item.content) { + if (part.type === "output_text") { + content = (content ?? "") + part.text; + } + } + } else if (item.type === "output_text" || item.type === "text") { content = item.content || item.text; } else if ( item.type === "function_call" || diff --git a/py/autoevals/oai.py b/py/autoevals/oai.py index 7a20e299..5384ea8e 100644 --- a/py/autoevals/oai.py +++ b/py/autoevals/oai.py @@ -247,7 +247,11 @@ def convert_responses_to_chat_completion(response: Any) -> dict[str, Any]: output_list = [] for item in output_list: item_type = item.get("type") - if item_type in ("output_text", "text"): + if item_type == "message": + for part in item["content"]: + if part["type"] == "output_text": + content = (content or "") + part["text"] + elif item_type in ("output_text", "text"): content = item.get("content") or item.get("text") elif item_type in ("function_call", "custom_tool_call"): # Convert Responses API tool call format to Chat Completions format diff --git a/py/autoevals/test_responses_conversion.py b/py/autoevals/test_responses_conversion.py new file mode 100644 index 00000000..d592f394 --- /dev/null +++ b/py/autoevals/test_responses_conversion.py @@ -0,0 +1,90 @@ +import pytest +import respx +from openai import AsyncOpenAI, OpenAI + +from autoevals.oai import LLMClient, arun_cached_request, run_cached_request + + +def text_message(*texts): + return { + "id": "msg-test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []} for text in texts], + } + + +TOOL_CALLS = [ + { + "type": "function_call", + "id": "fc-test", + "call_id": "call-0", + "name": "select_choice", + "arguments": '{"choice":"A"}', + }, +] +REQUEST_TOOLS = [ + { + "type": "function", + "function": { + "name": "select_choice", + "parameters": {"type": "object", "properties": {"choice": {"type": "string"}}}, + }, + }, +] +CHAT_TOOL_IDENTITIES = [("call-0", "function", "select_choice")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "output,content,tool_calls", + [ + pytest.param( + [text_message("first ", "second"), text_message(" third")], + "first second third", + [], + id="message-text", + ), + pytest.param( + [text_message("text with tools"), *TOOL_CALLS], "text with tools", CHAT_TOOL_IDENTITIES, id="mixed" + ), + pytest.param(TOOL_CALLS, None, CHAT_TOOL_IDENTITIES, id="tools-only"), + pytest.param([{"type": "output_text", "text": "legacy text"}], "legacy text", [], id="top-level-output-text"), + pytest.param([{"type": "text", "content": "legacy text"}], "legacy text", [], id="top-level-text"), + ], +) +@respx.mock +async def test_responses_conversion(is_async, output, content, tool_calls): + route = respx.post("https://api.openai.com/v1/responses").respond( + json={ + "id": "resp-test", + "object": "response", + "created_at": 1234567890, + "model": "gpt-5-mini", + "status": "completed", + "output": output, + } + ) + client_class = AsyncOpenAI if is_async else OpenAI + client = client_class(api_key="test-api-key", max_retries=0) + llm_client = LLMClient(openai=client, is_async=is_async) + params = {"client": llm_client, "model": "gpt-5-mini", "messages": [{"role": "user", "content": "test"}]} + if tool_calls: + params["tools"] = REQUEST_TOOLS + try: + response = await arun_cached_request(**params) if is_async else run_cached_request(**params) + finally: + if is_async: + await client.close() + else: + client.close() + + assert route.call_count == 1 + message = response["choices"][0]["message"] + assert message["content"] == content + converted_tools = message.get("tool_calls", []) + assert [(tool["id"], tool["type"], tool["function"]["name"]) for tool in converted_tools] == tool_calls + if tool_calls: + assert converted_tools[0]["function"]["arguments"] == '{"choice":"A"}' From ab1d14b66b04065b03e45d29ffca0d1364d15e38 Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:37:34 +0800 Subject: [PATCH 2/2] test: pin mocked Responses client endpoint --- py/autoevals/test_responses_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/autoevals/test_responses_conversion.py b/py/autoevals/test_responses_conversion.py index d592f394..38376c3f 100644 --- a/py/autoevals/test_responses_conversion.py +++ b/py/autoevals/test_responses_conversion.py @@ -68,7 +68,7 @@ async def test_responses_conversion(is_async, output, content, tool_calls): } ) client_class = AsyncOpenAI if is_async else OpenAI - client = client_class(api_key="test-api-key", max_retries=0) + client = client_class(api_key="test-api-key", base_url="https://api.openai.com/v1", max_retries=0) llm_client = LLMClient(openai=client, is_async=is_async) params = {"client": llm_client, "model": "gpt-5-mini", "messages": [{"role": "user", "content": "test"}]} if tool_calls: