diff --git a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md new file mode 100644 index 000000000..10471ad64 --- /dev/null +++ b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md @@ -0,0 +1,3 @@ +### Fixed + +- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. With an integer among the values that range check sums the magnitudes exactly too, so integers whose exact total fits the float range are taken (`fsum` would round each to a float first and overflow; `main` refused them when a float came first, as in `[0.0, m - 2**970 + 1, 2**970 - 1]` with `m` the largest float). The mixed `[m, 2**969, 2.0**969]`, whose exact total passes the range, is refused now, where `main` returned the mean of the rounded total. diff --git a/dokimi/dokimi/eval/finite_delta.py b/dokimi/dokimi/eval/finite_delta.py index e74e8fb12..0a3862a34 100644 --- a/dokimi/dokimi/eval/finite_delta.py +++ b/dokimi/dokimi/eval/finite_delta.py @@ -14,6 +14,7 @@ import math from collections.abc import Sequence +from fractions import Fraction def _shown(value: float) -> str: @@ -53,16 +54,35 @@ def check_sum_range(values: Sequence[float], error: type[Exception] = OverflowEr addition is safe for the sign patterns of a sign-flip test: they are refused. """ try: - refused = not math.isfinite(sum(abs(v) for v in values)) and all(math.isfinite(v) for v in values) - except OverflowError: # an integer sum too large for a float: ``isfinite`` cannot read it - refused = not _has_non_finite_float(*values) + if _has_non_finite_float(*values): + refused = False + elif all(isinstance(v, float) for v in values): + # fsum is exact: ``sum`` before 3.12 rounds small terms away and stays finite where the exact total does not + refused = not math.isfinite(math.fsum(abs(v) for v in values)) + else: + # an integer: ``fsum`` would round it to a float first and could overflow where the exact total fits + float(sum((Fraction(abs(v)) for v in values), Fraction(0))) + refused = False + except OverflowError: # an exact sum or an integer too large for a float + refused = True if refused: raise error(f"{where}{': ' if where else ''}the sum of the magnitudes of the values leaves the float range") def checked_mean(values: Sequence[float], error: type[Exception] = OverflowError, where: str = "") -> float: - """``sum(values) / len(values)`` after :func:`check_sum_range`.""" + """The mean of ``values`` after :func:`check_sum_range`, with an exactly rounded sum (``math.fsum``). + + The result does not depend on the Python version: ``sum`` of floats compensates from 3.12 on and + adds left to right before, so deltas that cancel left a rounding residue there. Integers only are + summed exactly as integers, and integers mixed with floats as exact fractions, since ``math.fsum`` + would first round each integer to a float. ``values`` are floats or integers; another number type + (``numpy.float32``) is outside the ``Sequence[float]`` contract and raises ``TypeError``. + """ check_sum_range(values, error, where) if _has_non_finite_float(*values): # they decide; finite values summed first could overflow and cancel a -inf return sum(v for v in values if isinstance(v, float) and not math.isfinite(v)) / len(values) - return sum(values) / len(values) + if all(isinstance(v, float) for v in values): + return math.fsum(values) / len(values) + if all(isinstance(v, int) for v in values): + return sum(values) / len(values) + return float(sum((Fraction(v if isinstance(v, int) else float(v)) for v in values), Fraction(0)) / len(values)) # mixed: exact, as fsum rounds an integer first diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 646756fcf..0f72bd3d0 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -80,12 +80,60 @@ def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy(): def test_checked_mean_is_the_plain_mean_where_it_is_finite(): - assert checked_mean([0.1, 0.2, 0.4]) == sum([0.1, 0.2, 0.4]) / 3 - assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == sum([1.0e307, -1.0e307, 1.0e307]) / 3 + assert checked_mean([0.1, 0.2, 0.4]) == math.fsum([0.1, 0.2, 0.4]) / 3 # fsum: sum() before 3.12 can differ in the last bit + assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == math.fsum([1.0e307, -1.0e307, 1.0e307]) / 3 assert checked_mean([math.inf, 1.0]) == math.inf assert math.isnan(checked_mean([math.nan, 1.0])) +def test_checked_mean_sums_exactly_whatever_the_python_version(): + assert checked_mean([1e307] * 6 + [-1e307] * 6) == 0.0 + assert checked_mean([1, 1e16, 1.0, -1e16]) == 0.5 # sum() starts on an int and loses its compensation: 0.25 + + +def test_checked_mean_refuses_a_total_that_only_the_exact_sum_sees_beyond_the_range(monkeypatch): + def left_to_right(values, start=0): # what sum() does before Python 3.12 + total = start + for v in values: + total += v + return total + + monkeypatch.setattr("dokimi.eval.finite_delta.sum", left_to_right, raising=False) + values = [float.fromhex("0x1.fffffffffffffp+1023"), 2.0**969, 2.0**969] + assert math.isfinite(left_to_right(values)) # the rounded total stays finite + with pytest.raises(CompareError, match=r"task 't': the sum of the magnitudes .* float range"): + checked_mean(values, CompareError, "task 't'") + + +def test_checked_mean_keeps_the_exact_mean_of_integers(): + assert checked_mean([2**60 + 1, 1, -(2**60)]) == 2 / 3 # fsum would round the first to a float: 1 / 3 + assert checked_mean([2**60 + 1, 2**60 + 1]) == float(2**60 + 1) + + +def test_checked_mean_keeps_integer_contributions_beside_floats(): + assert checked_mean([2**53 + 1, -(2**53), 0.0]) == 1 / 3 # fsum rounds the first to 2**53: 0.0 + assert checked_mean([2**60 + 1, -(2**60), 0.5]) == 0.5 + + +def test_checked_mean_refuses_a_mixed_total_that_only_the_exact_sum_sees_beyond_the_range(): + values = [float.fromhex("0x1.fffffffffffffp+1023"), 2**969, 2.0**969] # a left-to-right sum rounds back to the maximum + with pytest.raises(CompareError, match=r"task 't': the sum of the magnitudes .* float range"): + checked_mean(values, CompareError, "task 't'") + + +@pytest.mark.parametrize("order", ["float first", "float between"]) +def test_checked_mean_takes_integers_whose_exact_total_fits_the_float_range(order): + top = int(float.fromhex("0x1.fffffffffffffp+1023")) + low, high = top - 2**970 + 1, 2**970 - 1 + values = [0.0, low, high] if order == "float first" else [low, 0.0, high] # a float among the integers: sum() rounded the partial total and overflowed + assert checked_mean(values, CompareError) == top / 3 + + +def test_sign_flip_summary_mean_is_the_exact_mean(): + row = sign_flip_summary([_task(d) for d in (1, 1e16, 1.0, -1e16)]) + assert row["delta"]["mean"] == 0.5 + + def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error(): with pytest.raises(CompareError, match="float range"): checked_delta(10**309, 0.0, CompareError)