From 72e928d1823a65f2e331cc2de1b37289cbb56bdf Mon Sep 17 00:00:00 2001 From: feiiiiii5 Date: Mon, 28 Sep 2026 15:59:22 +0800 Subject: [PATCH] fix(json): do not treat booleans as numbers in JSONDiff `bool` is a subclass of `int` in Python, so the isinstance check that selects the numeric scorer also matched booleans. `NumericDiff(True, 1)` is then `1 - |1 - 1| / (1 + 1) = 1.0`, so a JSON boolean compared against the integer 1 scored a perfect match: JSONDiff().eval({"flag": True}, {"flag": 1}).score # 1.0 The TypeScript implementation tests `typeof v === "number"`, which is false for booleans, so it never took that path and scored the same input 0.0. AGENTS.md states the two implementations share behavior; they did not for booleans. Excluding `bool` sends Python booleans to the same fallback the TypeScript version reaches, and the two now agree on all seven cases I checked against the published 0.3.0 packages, including the equal-boolean cases (1.0) and the 'true' vs true string case (0.6667). Test: `pytest py/autoevals/test_json.py -k booleans` fails on main with `AssertionError: {'flag': True} scored equal to {'flag': 1}` and passes here. The rest of the non-LLM suite passes; test_partial fails on main too, for a missing OPENAI_API_KEY. --- py/autoevals/json.py | 13 ++++++++++++- py/autoevals/test_json.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/py/autoevals/json.py b/py/autoevals/json.py index a299e29c..9f96836c 100644 --- a/py/autoevals/json.py +++ b/py/autoevals/json.py @@ -84,6 +84,7 @@ def _run_eval_sync(self, output, **kwargs): """ import json +from typing import Any from jsonschema import ValidationError, validate @@ -94,6 +95,16 @@ def _run_eval_sync(self, output, **kwargs): from .string import Levenshtein +def _is_number(value: Any) -> bool: + # `bool` is a subclass of `int`, so a bare isinstance check treats `True` as + # the number 1 and `True`/`1` as an exact match. The TypeScript + # implementation tests `typeof value === "number"`, which is false for + # booleans, so it compares them as the strings "true"/"false" instead. A + # boolean and an integer are different JSON values, so exclude them here to + # keep both implementations in agreement. + return isinstance(value, (int, float)) and not isinstance(value, bool) + + class JSONDiff(ScorerWithPartial): """Compare JSON objects for structural and content similarity. @@ -180,7 +191,7 @@ def json_diff(self, o1, o2): return sum(base_scores) / max(len(o1), len(o2)) elif isinstance(o1, str) and isinstance(o2, str): return self.string_scorer.eval(o1, o2).score - elif (isinstance(o1, int) or isinstance(o1, float)) and (isinstance(o2, int) or isinstance(o2, float)): + elif _is_number(o1) and _is_number(o2): return self.number_scorer.eval(o1, o2).score elif o1 is None and o2 is None: return 1 diff --git a/py/autoevals/test_json.py b/py/autoevals/test_json.py index a78d3bae..6b0ab19d 100644 --- a/py/autoevals/test_json.py +++ b/py/autoevals/test_json.py @@ -166,3 +166,40 @@ def test_semantic_json(): assert abs(score - expected) < 0.0001 else: assert round(score * 100) <= round(expected * 100) + + +def test_json_booleans_are_not_numbers(): + # `bool` is a subclass of `int` in Python, so a bare isinstance check sends + # booleans to the numeric scorer. The TypeScript implementation tests + # `typeof v === "number"`, which is false for booleans, so it compares them + # as the strings "true"/"false" instead. A boolean and an integer are + # different JSON values, so neither implementation should call them equal. + cases = [ + ({"flag": True}, {"flag": 1}), + ({"flag": 1}, {"flag": True}), + ({"flag": False}, {"flag": 0}), + ({"flag": 0}, {"flag": False}), + ([True], [1]), + (True, 1), + ] + + evaluator = JSONDiff() + for output, expected in cases: + print(f"[{output}]", f"[{expected}]") + assert evaluator(output, expected).score < 1, f"{output!r} scored equal to {expected!r}" + + +def test_json_equal_booleans_still_match(): + # The fix must not break the case that is genuinely equal. + cases = [ + ({"flag": True}, {"flag": True}), + ({"flag": False}, {"flag": False}), + (True, True), + (False, False), + ([True, False], [True, False]), + ] + + evaluator = JSONDiff() + for output, expected in cases: + print(f"[{output}]", f"[{expected}]") + assert evaluator(output, expected).score == approx(1, abs=1e-4)