Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,4 @@ Tests require:
- TypeScript tests use `vitest` with `msw` for HTTP mocking
- Tests that call real LLM APIs need valid API keys
- Test files are colocated: `test_*.py` (Python), `*.test.ts` (TypeScript)
- Python `JSONDiff.eval_async()` recursively awaits custom string and number scorers; keep its aggregation behavior aligned with `json_diff()` when changing either path.
36 changes: 36 additions & 0 deletions py/autoevals/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,42 @@ def __init__(self, string_scorer: Scorer = None, number_scorer: Scorer = None, p
def _run_eval_sync(self, output, expected=None, **kwargs):
return Score(name=self._name(), score=self.json_diff(output, expected))

async def _run_eval_async(self, output, expected=None, **kwargs):
return Score(name=self._name(), score=await self._json_diff_async(output, expected))

async def _json_diff_async(self, o1, o2):
if not self.preserve_strings:
if isinstance(o1, str) and self._valid_json.valid_json(o1) == 1:
o1 = json.loads(o1)
if isinstance(o2, str) and self._valid_json.valid_json(o2) == 1:
o2 = json.loads(o2)

if isinstance(o1, dict) and isinstance(o2, dict):
if len(o1) == 0 and len(o2) == 0:
return 1

all_keys = set(o1.keys()).union(set(o2.keys()))
base_scores = [await self._json_diff_async(o1.get(k), o2.get(k)) for k in all_keys]
base_scores = [s for s in base_scores if s is not None]
return sum(base_scores) / len(base_scores)
elif isinstance(o1, list) and isinstance(o2, list):
if len(o1) == 0 and len(o2) == 0:
return 1
base_scores = [await self._json_diff_async(e1, e2) for (e1, e2) in zip(o1, o2)]
base_scores = [s for s in base_scores if s is not None]
return sum(base_scores) / max(len(o1), len(o2))
elif isinstance(o1, str) and isinstance(o2, str):
return (await self.string_scorer.eval_async(o1, o2)).score
elif (isinstance(o1, int) or isinstance(o1, float)) and (isinstance(o2, int) or isinstance(o2, float)):
return (await self.number_scorer.eval_async(o1, o2)).score
elif o1 is None and o2 is None:
return 1
elif o1 is None or o2 is None:
return 0
else:
kwargs = {"separators": (",", ":"), "sort_keys": True}
return (await self.string_scorer.eval_async(json.dumps(o1, **kwargs), json.dumps(o2, **kwargs))).score

def json_diff(self, o1, o2):
if not self.preserve_strings:
if isinstance(o1, str) and self._valid_json.valid_json(o1) == 1:
Expand Down
94 changes: 94 additions & 0 deletions py/autoevals/test_json_async.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import json

import httpx
import pytest
from openai import AsyncOpenAI

from autoevals.json import JSONDiff
from autoevals.score import Score, Scorer
from autoevals.string import EmbeddingSimilarity


class AsyncOnlyScorer(Scorer):
def _run_eval_sync(self, output, expected=None, **kwargs):
raise AssertionError("Synchronous scorer called during async evaluation")

async def _run_eval_async(self, output, expected=None, **kwargs):
return Score(name=self._name(), score=0.5)


@pytest.mark.asyncio
@pytest.mark.parametrize(
"output,expected",
[
({"answer": ["Paris"]}, {"answer": ["City of Paris"]}),
('{"answer": ["Paris"]}', '{"answer": ["City of Paris"]}'),
({"answer": [1]}, {"answer": [2]}),
([{}], [[]]),
],
)
async def test_json_diff_awaits_custom_scorers(output, expected):
scorer = JSONDiff(string_scorer=AsyncOnlyScorer(), number_scorer=AsyncOnlyScorer())
assert (await scorer.eval_async(output, expected)).score == 0.5


@pytest.mark.asyncio
async def test_json_diff_partial_awaits_custom_scorer():
scorer = JSONDiff.partial(expected={"answer": "Paris"})(string_scorer=AsyncOnlyScorer())
assert (await scorer.eval_async({"answer": "City of Paris"})).score == 0.5


@pytest.mark.asyncio
@pytest.mark.parametrize(
"output,expected,preserve_strings",
[
({}, {}, False),
([], [], False),
(None, None, False),
({"answer": None}, {"answer": "Paris"}, False),
({"a": [1, "Paris"]}, {"a": [2, "Pari"], "extra": 1}, False),
([1, 2], [1], False),
('{"a": 1}', '{"a": 2}', False),
('{"a": 1}', '{"a": 2}', True),
(True, False, False),
],
)
async def test_json_diff_async_preserves_sync_scores(output, expected, preserve_strings):
scorer = JSONDiff(preserve_strings=preserve_strings)
assert (await scorer.eval_async(output, expected)).score == scorer.eval(output, expected).score


@pytest.mark.asyncio
async def test_json_diff_with_async_embedding_client():
requests = []

def handle(request):
body = json.loads(request.content)
requests.append((request.url.path, body["model"], body["input"]))
return httpx.Response(
200,
json={
"object": "list",
"data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
"model": "test-embedding",
"usage": {"prompt_tokens": 1, "total_tokens": 1},
},
)

with EmbeddingSimilarity._CACHE_LOCK:
EmbeddingSimilarity._CACHE.clear()
try:
async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as http_client:
async with AsyncOpenAI(
api_key="test-key", base_url="https://embeddings.example/v1", http_client=http_client
) as client:
scorer = JSONDiff(string_scorer=EmbeddingSimilarity(client=client, model="test-embedding"))
result = await scorer.eval_async({"answer": ["Paris"]}, {"answer": ["City of Paris"]})
assert result.score == 1
assert requests == [
("/v1/embeddings", "test-embedding", "Paris"),
("/v1/embeddings", "test-embedding", "City of Paris"),
]
finally:
with EmbeddingSimilarity._CACHE_LOCK:
EmbeddingSimilarity._CACHE.clear()