Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions js/json.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -181,3 +181,49 @@ test("Semantic JSON Test", async () => {
}
}
});

const missingMemberCases: [unknown, unknown, number][] = [
[{}, { answer: null }, 0],
[{ answer: null }, {}, 0],
[{ answer: "ok" }, { answer: "ok", evidence: null }, 0.5],
[{ answer: "ok", evidence: null }, { answer: "ok" }, 0.5],
[{ record: {} }, { record: { source: null } }, 0],
[[{}], [{ answer: null }], 0],
["{}", '{"answer":null}', 0],
[{ answer: null }, { answer: null }, 1],
[{}, {}, 1],
[{ answer: 0 }, {}, 0],
[{}, { answer: false }, 0],
[{ answer: 0 }, { answer: 0 }, 1],
[{ answer: false }, { answer: false }, 1],
[{}, JSON.parse('{"toString":null}'), 0],
[{}, JSON.parse('{"__proto__":null}'), 0],
];

test.each(missingMemberCases)(
"missing object members differ from null: %j / %j",
async (output, expected, score) => {
expect((await JSONDiff({ output, expected })).score).toBe(score);
},
);

test("missing members preserve the scorer for shared values", async () => {
expect(
(
await JSONDiff({
output: { answer: 1 },
expected: { answer: 1, evidence: null },
numberScorer: ExactMatch,
})
).score,
).toBe(0.5);
expect(
(
await JSONDiff({
output: { answer: 1 },
expected: { answer: 2, evidence: null },
numberScorer: ExactMatch,
})
).score,
).toBe(0);
});
11 changes: 10 additions & 1 deletion js/json.ts
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,16 @@ async function jsonDiff(
const baseScores = (
await Promise.all(
allKeys.map((k) =>
jsonDiff(o1[k], o2[k], stringScorer, numberScorer, preserveStrings),
Object.prototype.hasOwnProperty.call(o1, k) &&
Object.prototype.hasOwnProperty.call(o2, k)
? jsonDiff(
o1[k],
o2[k],
stringScorer,
numberScorer,
preserveStrings,
)
: 0,
),
)
).filter((s) => s !== null) as number[];
Expand Down
2 changes: 1 addition & 1 deletion py/autoevals/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ def json_diff(self, o1, o2):
return 1

all_keys = set(o1.keys()).union(set(o2.keys()))
base_scores = [self.json_diff(o1.get(k), o2.get(k)) for k in all_keys]
base_scores = [self.json_diff(o1[k], o2[k]) if k in o1 and k in o2 else 0 for k in all_keys]
base_scores = [s for s in base_scores if s is not None]
return sum(base_scores) / len(base_scores)
elif isinstance(o1, list) and isinstance(o2, list):
Expand Down
32 changes: 32 additions & 0 deletions py/autoevals/test_json.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import pytest
from pytest import approx

from autoevals.json import JSONDiff, ValidJSON
Expand Down Expand Up @@ -166,3 +167,34 @@ def test_semantic_json():
assert abs(score - expected) < 0.0001
else:
assert round(score * 100) <= round(expected * 100)


@pytest.mark.parametrize(
"output,expected,score",
[
({}, {"answer": None}, 0),
({"answer": None}, {}, 0),
({"answer": "ok"}, {"answer": "ok", "evidence": None}, 0.5),
({"answer": "ok", "evidence": None}, {"answer": "ok"}, 0.5),
({"record": {}}, {"record": {"source": None}}, 0),
([{}], [{"answer": None}], 0),
("{}", '{"answer":null}', 0),
({"answer": None}, {"answer": None}, 1),
({}, {}, 1),
({"answer": 0}, {}, 0),
({}, {"answer": False}, 0),
({"answer": 0}, {"answer": 0}, 1),
({"answer": False}, {"answer": False}, 1),
],
)
@pytest.mark.asyncio
async def test_missing_object_members_are_not_null(output, expected, score):
scorer = JSONDiff()
assert scorer.eval(output, expected).score == score
assert (await scorer.eval_async(output, expected)).score == score


def test_missing_member_preserves_custom_shared_scorer():
scorer = JSONDiff(number_scorer=ExactMatch())
assert scorer.eval({"answer": 1}, {"answer": 1, "evidence": None}).score == 0.5
assert scorer.eval({"answer": 1}, {"answer": 2, "evidence": None}).score == 0