Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions py/autoevals/oai.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,19 +353,29 @@ def complete_wrapper(**kwargs: Any) -> Any:
return convert_responses_to_chat_completion(response)
return chat_complete(**kwargs)

self.complete = complete_wrapper
self.embed = self.openai.embeddings.create
self.moderation = self.openai.moderations.create
self.RateLimitError = openai_module.RateLimitError
if self.complete is None:
self.complete = complete_wrapper
if self.embed is None:
self.embed = self.openai.embeddings.create
if self.moderation is None:
self.moderation = self.openai.moderations.create
if self.RateLimitError is None:
self.RateLimitError = openai_module.RateLimitError
else:
openai_module = cast(OpenAIV0Module, openai_module)
self.openai = cast(OpenAIV0Module, self.openai)

# v0
self.complete = self.openai.ChatCompletion.acreate if self.is_async else self.openai.ChatCompletion.create
self.embed = self.openai.Embedding.acreate if self.is_async else self.openai.Embedding.create
self.moderation = self.openai.Moderation.acreate if self.is_async else self.openai.Moderation.create
self.RateLimitError = openai_module.error.RateLimitError
if self.complete is None:
self.complete = (
self.openai.ChatCompletion.acreate if self.is_async else self.openai.ChatCompletion.create
)
if self.embed is None:
self.embed = self.openai.Embedding.acreate if self.is_async else self.openai.Embedding.create
if self.moderation is None:
self.moderation = self.openai.Moderation.acreate if self.is_async else self.openai.Moderation.create
if self.RateLimitError is None:
self.RateLimitError = openai_module.error.RateLimitError

@property
def is_wrapped(self) -> bool:
Expand Down
126 changes: 126 additions & 0 deletions py/autoevals/test_oai.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import sys
from typing import Any, cast
from unittest.mock import AsyncMock

import httpx
import openai
import pytest
from braintrust.oai import NamedWrapper, wrap_openai
Expand All @@ -13,9 +15,11 @@
OpenAIV1Module,
_named_wrapper, # type: ignore[import] # Accessing private members for testing
_wrap_openai, # type: ignore[import] # Accessing private members for testing
arun_cached_request,
get_default_model,
get_openai_wrappers,
prepare_openai,
run_cached_request,
)


Expand Down Expand Up @@ -60,6 +64,128 @@ def test_prepare_openai_uses_unwrapped_global_client():
assert prepared_client.openai.api_key == "api-key"


@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize(
"custom_methods", [("complete", "embed", "moderation"), ("complete",), ("embed",), ("moderation",)]
)
async def test_custom_v1_methods_are_used_and_other_methods_default(is_async, custom_methods, mocker):
request_args = {
"complete": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]},
"embed": {"model": "text-embedding-3-small", "input": "hello", "encoding_format": "float"},
"moderation": {"model": "omni-moderation-latest", "input": "hello"},
}
responses = {
"complete": {
"id": "chat-test",
"object": "chat.completion",
"created": 0,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}, "finish_reason": "stop"}],
},
"embed": {
"object": "list",
"model": "text-embedding-3-small",
"data": [{"object": "embedding", "index": 0, "embedding": [1.0]}],
"usage": {"prompt_tokens": 1, "total_tokens": 1},
},
"moderation": {"id": "mod-test", "model": "omni-moderation-latest", "results": []},
}
paths = {"/v1/chat/completions": "complete", "/v1/embeddings": "embed", "/v1/moderations": "moderation"}
sdk_calls = []
custom_calls = {method: [] for method in custom_methods}
sleep = mocker.patch("autoevals.oai.time.sleep")
async_sleep = mocker.patch("autoevals.oai.asyncio.sleep", new_callable=AsyncMock)

class CustomRateLimitError(Exception):
pass

def respond(request):
method = paths[request.url.path]
sdk_calls.append(method)
return httpx.Response(200, json=responses[method])

def make_callback(method):
def callback(**kwargs):
custom_calls[method].append(kwargs)
if len(custom_calls[method]) == 1:
raise CustomRateLimitError("try again")
return responses[method]

async def async_callback(**kwargs):
return callback(**kwargs)

return async_callback if is_async else callback

transport = httpx.MockTransport(respond)
http_client = httpx.AsyncClient(transport=transport) if is_async else httpx.Client(transport=transport)
sdk_class = openai.AsyncOpenAI if is_async else openai.OpenAI
sdk = sdk_class(api_key="test-key", base_url="http://test/v1", http_client=http_client)
try:
client = LLMClient(
openai=sdk,
is_async=is_async,
RateLimitError=CustomRateLimitError,
**{method: make_callback(method) for method in custom_methods},
)
init(client)
for method, kwargs in request_args.items():
if is_async:
result = await arun_cached_request(request_type=method, **kwargs)
else:
result = run_cached_request(request_type=method, **kwargs)
assert result["model"] == kwargs["model"]
assert custom_calls == {method: [request_args[method], request_args[method]] for method in custom_methods}
assert sdk_calls == [method for method in request_args if method not in custom_methods]
assert (async_sleep.await_count if is_async else sleep.call_count) == len(custom_methods)
finally:
if is_async:
await sdk.close()
else:
sdk.close()


@pytest.mark.asyncio
@pytest.mark.parametrize("is_async", [False, True])
@pytest.mark.parametrize(
"custom_methods", [("complete", "embed", "moderation"), ("complete",), ("embed",), ("moderation",)]
)
async def test_custom_v0_methods_are_used_and_other_methods_default(mock_openai_v0, is_async, custom_methods, mocker):
request_args = {
"complete": {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]},
"embed": {"model": "text-embedding-3-small", "input": "hello"},
"moderation": {"input": "hello"},
}

class CustomRateLimitError(Exception):
pass

mock_type = AsyncMock if is_async else mocker.Mock
defaults = {}
for method, resource in (("complete", "ChatCompletion"), ("embed", "Embedding"), ("moderation", "Moderation")):
defaults[method] = mock_type(return_value={"method": f"default-{method}"})
mocker.patch.object(getattr(mock_openai_v0, resource), "acreate" if is_async else "create", defaults[method])
callbacks = {
method: mock_type(side_effect=[CustomRateLimitError("try again"), {"method": f"custom-{method}"}])
for method in custom_methods
}
sleep = mocker.patch("autoevals.oai.time.sleep")
async_sleep = mocker.patch("autoevals.oai.asyncio.sleep", new_callable=AsyncMock)
client = LLMClient(openai=mock_openai_v0, is_async=is_async, RateLimitError=CustomRateLimitError, **callbacks)
for method, kwargs in request_args.items():
if is_async:
result = await arun_cached_request(client=client, request_type=method, **kwargs)
else:
result = run_cached_request(client=client, request_type=method, **kwargs)
assert result == {"method": f"{'custom' if method in custom_methods else 'default'}-{method}"}
if method in custom_methods:
assert callbacks[method].call_count == 2
defaults[method].assert_not_called()
else:
defaults[method].assert_called_once_with(**kwargs)
assert (async_sleep.await_count if is_async else sleep.call_count) == len(custom_methods)


def test_init_creates_llmclient_if_needed():
openai_obj = openai.OpenAI()
init(openai_obj)
Expand Down