Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 112 additions & 1 deletion js/oai.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@ import {
test,
vi,
} from "vitest";
import { buildOpenAIClient, init, getDefaultModel } from "./oai";
import {
buildOpenAIClient,
cachedChatCompletion,
init,
getDefaultModel,
} from "./oai";

import { setupServer } from "msw/node";

Expand Down Expand Up @@ -74,7 +79,113 @@ const MOCK_OPENAI_COMPLETION_RESPONSE = {
},
};

const textMessage = (...texts: string[]) => ({
id: "msg-test",
type: "message",
role: "assistant",
status: "completed",
content: texts.map((text) => ({
type: "output_text",
text,
annotations: [],
})),
});

const responseToolCalls = [
{
type: "function_call",
id: "fc-test",
call_id: "call-0",
name: "select_choice",
arguments: '{"choice":"A"}',
},
];
const requestTools: OpenAI.Chat.Completions.ChatCompletionTool[] = [
{
type: "function",
function: {
name: "select_choice",
parameters: {
type: "object",
properties: { choice: { type: "string" } },
},
},
},
];
const chatToolCalls = [
{
id: "call-0",
type: "function",
function: { name: "select_choice", arguments: '{"choice":"A"}' },
},
];

describe("OAI", () => {
test.each([
{
name: "message text",
output: [textMessage("first ", "second"), textMessage(" third")],
content: "first second third",
toolCalls: [],
},
{
name: "text with tools",
output: [textMessage("text with tools"), ...responseToolCalls],
content: "text with tools",
toolCalls: chatToolCalls,
},
{
name: "tools only",
output: responseToolCalls,
content: null,
toolCalls: chatToolCalls,
},
{
name: "top-level output_text",
output: [{ type: "output_text", text: "legacy text" }],
content: "legacy text",
toolCalls: [],
},
{
name: "top-level text",
output: [{ type: "text", content: "legacy text" }],
content: "legacy text",
toolCalls: [],
},
])("converts Responses $name", async ({ output, content, toolCalls }) => {
const respond = vi.fn(() =>
HttpResponse.json({
id: "resp-test",
object: "response",
created_at: 1234567890,
model: "gpt-5-mini",
status: "completed",
output,
}),
);
server.use(http.post("https://api.openai.com/v1/responses", respond));

const response = await cachedChatCompletion(
{
model: "gpt-5-mini",
messages: [{ role: "user", content: "test" }],
...(toolCalls.length > 0 ? { tools: requestTools } : {}),
},
{
client: new OpenAI({
apiKey: "test-api-key",
baseURL: "https://api.openai.com/v1",
maxRetries: 0,
}),
},
);

expect(respond).toHaveBeenCalledTimes(1);
const message = response.choices[0].message;
expect(message.content).toBe(content);
expect(message.tool_calls ?? []).toEqual(toolCalls);
});

test("should use Azure OpenAI", async () => {
server.use(
http.post(
Expand Down
8 changes: 7 additions & 1 deletion js/oai.ts
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,13 @@ export async function cachedChatCompletion(

if (response.output && Array.isArray(response.output)) {
for (const item of response.output) {
if (item.type === "output_text" || item.type === "text") {
if (item.type === "message") {
for (const part of item.content) {
if (part.type === "output_text") {
content = (content ?? "") + part.text;
}
}
} else if (item.type === "output_text" || item.type === "text") {
content = item.content || item.text;
} else if (
item.type === "function_call" ||
Expand Down
6 changes: 5 additions & 1 deletion py/autoevals/oai.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,11 @@ def convert_responses_to_chat_completion(response: Any) -> dict[str, Any]:
output_list = []
for item in output_list:
item_type = item.get("type")
if item_type in ("output_text", "text"):
if item_type == "message":
for part in item["content"]:
if part["type"] == "output_text":
content = (content or "") + part["text"]
elif item_type in ("output_text", "text"):
content = item.get("content") or item.get("text")
elif item_type in ("function_call", "custom_tool_call"):
# Convert Responses API tool call format to Chat Completions format
Expand Down
90 changes: 90 additions & 0 deletions py/autoevals/test_responses_conversion.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import pytest
import respx
from openai import AsyncOpenAI, OpenAI

from autoevals.oai import LLMClient, arun_cached_request, run_cached_request


def text_message(*texts):
return {
"id": "msg-test",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": text, "annotations": []} for text in texts],
}


TOOL_CALLS = [
{
"type": "function_call",
"id": "fc-test",
"call_id": "call-0",
"name": "select_choice",
"arguments": '{"choice":"A"}',
},
]
REQUEST_TOOLS = [
{
"type": "function",
"function": {
"name": "select_choice",
"parameters": {"type": "object", "properties": {"choice": {"type": "string"}}},
},
},
]
CHAT_TOOL_IDENTITIES = [("call-0", "function", "select_choice")]


@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize(
"output,content,tool_calls",
[
pytest.param(
[text_message("first ", "second"), text_message(" third")],
"first second third",
[],
id="message-text",
),
pytest.param(
[text_message("text with tools"), *TOOL_CALLS], "text with tools", CHAT_TOOL_IDENTITIES, id="mixed"
),
pytest.param(TOOL_CALLS, None, CHAT_TOOL_IDENTITIES, id="tools-only"),
pytest.param([{"type": "output_text", "text": "legacy text"}], "legacy text", [], id="top-level-output-text"),
pytest.param([{"type": "text", "content": "legacy text"}], "legacy text", [], id="top-level-text"),
],
)
@respx.mock
async def test_responses_conversion(is_async, output, content, tool_calls):
route = respx.post("https://api.openai.com/v1/responses").respond(
json={
"id": "resp-test",
"object": "response",
"created_at": 1234567890,
"model": "gpt-5-mini",
"status": "completed",
"output": output,
}
)
client_class = AsyncOpenAI if is_async else OpenAI
client = client_class(api_key="test-api-key", base_url="https://api.openai.com/v1", max_retries=0)
llm_client = LLMClient(openai=client, is_async=is_async)
params = {"client": llm_client, "model": "gpt-5-mini", "messages": [{"role": "user", "content": "test"}]}
if tool_calls:
params["tools"] = REQUEST_TOOLS
try:
response = await arun_cached_request(**params) if is_async else run_cached_request(**params)
finally:
if is_async:
await client.close()
else:
client.close()

assert route.call_count == 1
message = response["choices"][0]["message"]
assert message["content"] == content
converted_tools = message.get("tool_calls", [])
assert [(tool["id"], tool["type"], tool["function"]["name"]) for tool in converted_tools] == tool_calls
if tool_calls:
assert converted_tools[0]["function"]["arguments"] == '{"choice":"A"}'