From d3c63837bb8ea98f22fb11c08a81e87db01fb2c1 Mon Sep 17 00:00:00 2001 From: bin <178128333+xlb3055@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:08:51 +0800 Subject: [PATCH] fix(python): await custom scorers in JSONDiff async evaluations --- AGENTS.md | 1 + py/autoevals/json.py | 36 +++++++++++++ py/autoevals/test_json_async.py | 94 +++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 py/autoevals/test_json_async.py diff --git a/AGENTS.md b/AGENTS.md index 85089885..a9406d7e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,3 +105,4 @@ Tests require: - TypeScript tests use `vitest` with `msw` for HTTP mocking - Tests that call real LLM APIs need valid API keys - Test files are colocated: `test_*.py` (Python), `*.test.ts` (TypeScript) +- Python `JSONDiff.eval_async()` recursively awaits custom string and number scorers; keep its aggregation behavior aligned with `json_diff()` when changing either path. diff --git a/py/autoevals/json.py b/py/autoevals/json.py index a299e29c..f9fba2f3 100644 --- a/py/autoevals/json.py +++ b/py/autoevals/json.py @@ -157,6 +157,42 @@ def __init__(self, string_scorer: Scorer = None, number_scorer: Scorer = None, p def _run_eval_sync(self, output, expected=None, **kwargs): return Score(name=self._name(), score=self.json_diff(output, expected)) + async def _run_eval_async(self, output, expected=None, **kwargs): + return Score(name=self._name(), score=await self._json_diff_async(output, expected)) + + async def _json_diff_async(self, o1, o2): + if not self.preserve_strings: + if isinstance(o1, str) and self._valid_json.valid_json(o1) == 1: + o1 = json.loads(o1) + if isinstance(o2, str) and self._valid_json.valid_json(o2) == 1: + o2 = json.loads(o2) + + if isinstance(o1, dict) and isinstance(o2, dict): + if len(o1) == 0 and len(o2) == 0: + return 1 + + all_keys = set(o1.keys()).union(set(o2.keys())) + base_scores = [await self._json_diff_async(o1.get(k), o2.get(k)) for k in all_keys] + base_scores = [s for s in base_scores if s is not None] + return sum(base_scores) / len(base_scores) + elif isinstance(o1, list) and isinstance(o2, list): + if len(o1) == 0 and len(o2) == 0: + return 1 + base_scores = [await self._json_diff_async(e1, e2) for (e1, e2) in zip(o1, o2)] + base_scores = [s for s in base_scores if s is not None] + return sum(base_scores) / max(len(o1), len(o2)) + elif isinstance(o1, str) and isinstance(o2, str): + return (await self.string_scorer.eval_async(o1, o2)).score + elif (isinstance(o1, int) or isinstance(o1, float)) and (isinstance(o2, int) or isinstance(o2, float)): + return (await self.number_scorer.eval_async(o1, o2)).score + elif o1 is None and o2 is None: + return 1 + elif o1 is None or o2 is None: + return 0 + else: + kwargs = {"separators": (",", ":"), "sort_keys": True} + return (await self.string_scorer.eval_async(json.dumps(o1, **kwargs), json.dumps(o2, **kwargs))).score + def json_diff(self, o1, o2): if not self.preserve_strings: if isinstance(o1, str) and self._valid_json.valid_json(o1) == 1: diff --git a/py/autoevals/test_json_async.py b/py/autoevals/test_json_async.py new file mode 100644 index 00000000..e182668c --- /dev/null +++ b/py/autoevals/test_json_async.py @@ -0,0 +1,94 @@ +import json + +import httpx +import pytest +from openai import AsyncOpenAI + +from autoevals.json import JSONDiff +from autoevals.score import Score, Scorer +from autoevals.string import EmbeddingSimilarity + + +class AsyncOnlyScorer(Scorer): + def _run_eval_sync(self, output, expected=None, **kwargs): + raise AssertionError("Synchronous scorer called during async evaluation") + + async def _run_eval_async(self, output, expected=None, **kwargs): + return Score(name=self._name(), score=0.5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output,expected", + [ + ({"answer": ["Paris"]}, {"answer": ["City of Paris"]}), + ('{"answer": ["Paris"]}', '{"answer": ["City of Paris"]}'), + ({"answer": [1]}, {"answer": [2]}), + ([{}], [[]]), + ], +) +async def test_json_diff_awaits_custom_scorers(output, expected): + scorer = JSONDiff(string_scorer=AsyncOnlyScorer(), number_scorer=AsyncOnlyScorer()) + assert (await scorer.eval_async(output, expected)).score == 0.5 + + +@pytest.mark.asyncio +async def test_json_diff_partial_awaits_custom_scorer(): + scorer = JSONDiff.partial(expected={"answer": "Paris"})(string_scorer=AsyncOnlyScorer()) + assert (await scorer.eval_async({"answer": "City of Paris"})).score == 0.5 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output,expected,preserve_strings", + [ + ({}, {}, False), + ([], [], False), + (None, None, False), + ({"answer": None}, {"answer": "Paris"}, False), + ({"a": [1, "Paris"]}, {"a": [2, "Pari"], "extra": 1}, False), + ([1, 2], [1], False), + ('{"a": 1}', '{"a": 2}', False), + ('{"a": 1}', '{"a": 2}', True), + (True, False, False), + ], +) +async def test_json_diff_async_preserves_sync_scores(output, expected, preserve_strings): + scorer = JSONDiff(preserve_strings=preserve_strings) + assert (await scorer.eval_async(output, expected)).score == scorer.eval(output, expected).score + + +@pytest.mark.asyncio +async def test_json_diff_with_async_embedding_client(): + requests = [] + + def handle(request): + body = json.loads(request.content) + requests.append((request.url.path, body["model"], body["input"])) + return httpx.Response( + 200, + json={ + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "model": "test-embedding", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + ) + + with EmbeddingSimilarity._CACHE_LOCK: + EmbeddingSimilarity._CACHE.clear() + try: + async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as http_client: + async with AsyncOpenAI( + api_key="test-key", base_url="https://embeddings.example/v1", http_client=http_client + ) as client: + scorer = JSONDiff(string_scorer=EmbeddingSimilarity(client=client, model="test-embedding")) + result = await scorer.eval_async({"answer": ["Paris"]}, {"answer": ["City of Paris"]}) + assert result.score == 1 + assert requests == [ + ("/v1/embeddings", "test-embedding", "Paris"), + ("/v1/embeddings", "test-embedding", "City of Paris"), + ] + finally: + with EmbeddingSimilarity._CACHE_LOCK: + EmbeddingSimilarity._CACHE.clear()