diff --git a/js/json.test.ts b/js/json.test.ts index c219c18..683c410 100644 --- a/js/json.test.ts +++ b/js/json.test.ts @@ -181,3 +181,49 @@ test("Semantic JSON Test", async () => { } } }); + +const missingMemberCases: [unknown, unknown, number][] = [ + [{}, { answer: null }, 0], + [{ answer: null }, {}, 0], + [{ answer: "ok" }, { answer: "ok", evidence: null }, 0.5], + [{ answer: "ok", evidence: null }, { answer: "ok" }, 0.5], + [{ record: {} }, { record: { source: null } }, 0], + [[{}], [{ answer: null }], 0], + ["{}", '{"answer":null}', 0], + [{ answer: null }, { answer: null }, 1], + [{}, {}, 1], + [{ answer: 0 }, {}, 0], + [{}, { answer: false }, 0], + [{ answer: 0 }, { answer: 0 }, 1], + [{ answer: false }, { answer: false }, 1], + [{}, JSON.parse('{"toString":null}'), 0], + [{}, JSON.parse('{"__proto__":null}'), 0], +]; + +test.each(missingMemberCases)( + "missing object members differ from null: %j / %j", + async (output, expected, score) => { + expect((await JSONDiff({ output, expected })).score).toBe(score); + }, +); + +test("missing members preserve the scorer for shared values", async () => { + expect( + ( + await JSONDiff({ + output: { answer: 1 }, + expected: { answer: 1, evidence: null }, + numberScorer: ExactMatch, + }) + ).score, + ).toBe(0.5); + expect( + ( + await JSONDiff({ + output: { answer: 1 }, + expected: { answer: 2, evidence: null }, + numberScorer: ExactMatch, + }) + ).score, + ).toBe(0); +}); diff --git a/js/json.ts b/js/json.ts index 50be17a..b321d28 100644 --- a/js/json.ts +++ b/js/json.ts @@ -241,7 +241,16 @@ async function jsonDiff( const baseScores = ( await Promise.all( allKeys.map((k) => - jsonDiff(o1[k], o2[k], stringScorer, numberScorer, preserveStrings), + Object.prototype.hasOwnProperty.call(o1, k) && + Object.prototype.hasOwnProperty.call(o2, k) + ? jsonDiff( + o1[k], + o2[k], + stringScorer, + numberScorer, + preserveStrings, + ) + : 0, ), ) ).filter((s) => s !== null) as number[]; diff --git a/py/autoevals/json.py b/py/autoevals/json.py index a299e29..e638175 100644 --- a/py/autoevals/json.py +++ b/py/autoevals/json.py @@ -169,7 +169,7 @@ def json_diff(self, o1, o2): return 1 all_keys = set(o1.keys()).union(set(o2.keys())) - base_scores = [self.json_diff(o1.get(k), o2.get(k)) for k in all_keys] + base_scores = [self.json_diff(o1[k], o2[k]) if k in o1 and k in o2 else 0 for k in all_keys] base_scores = [s for s in base_scores if s is not None] return sum(base_scores) / len(base_scores) elif isinstance(o1, list) and isinstance(o2, list): diff --git a/py/autoevals/test_json.py b/py/autoevals/test_json.py index a78d3ba..bad9847 100644 --- a/py/autoevals/test_json.py +++ b/py/autoevals/test_json.py @@ -1,3 +1,4 @@ +import pytest from pytest import approx from autoevals.json import JSONDiff, ValidJSON @@ -166,3 +167,34 @@ def test_semantic_json(): assert abs(score - expected) < 0.0001 else: assert round(score * 100) <= round(expected * 100) + + +@pytest.mark.parametrize( + "output,expected,score", + [ + ({}, {"answer": None}, 0), + ({"answer": None}, {}, 0), + ({"answer": "ok"}, {"answer": "ok", "evidence": None}, 0.5), + ({"answer": "ok", "evidence": None}, {"answer": "ok"}, 0.5), + ({"record": {}}, {"record": {"source": None}}, 0), + ([{}], [{"answer": None}], 0), + ("{}", '{"answer":null}', 0), + ({"answer": None}, {"answer": None}, 1), + ({}, {}, 1), + ({"answer": 0}, {}, 0), + ({}, {"answer": False}, 0), + ({"answer": 0}, {"answer": 0}, 1), + ({"answer": False}, {"answer": False}, 1), + ], +) +@pytest.mark.asyncio +async def test_missing_object_members_are_not_null(output, expected, score): + scorer = JSONDiff() + assert scorer.eval(output, expected).score == score + assert (await scorer.eval_async(output, expected)).score == score + + +def test_missing_member_preserves_custom_shared_scorer(): + scorer = JSONDiff(number_scorer=ExactMatch()) + assert scorer.eval({"answer": 1}, {"answer": 1, "evidence": None}).score == 0.5 + assert scorer.eval({"answer": 1}, {"answer": 2, "evidence": None}).score == 0