From 249f63e4d2e6cb1b52e12dec67683c1549d8cfed Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:38:24 +0000 Subject: [PATCH 1/5] fix(eval): checked_mean sums floats exactly, not by the version's sum() (#772) checked_mean added its values with sum(), which compensates float rounding only from Python 3.12: before that, deltas that cancel (`[1e307] * 6 + [-1e307] * 6`) left a residue of -2.079e290 in the repo compare summary mean while the interval used math.fsum. Even on 3.12+, sum() loses its compensation when the first value is an int: `checked_mean([1, 1e16, 1.0, -1e16])` was 0.25, the exact mean is 0.5. The finite branch now returns math.fsum(values) / len(values) after check_sum_range, so the mean is the same on every version. Integer-only inputs keep the exact integer sum, because math.fsum would first round each integer to a float (`[2**60 + 1, 1, -2**60]` has the mean 2/3, fsum gives 1/3). The non-finite branch and the refusals are unchanged. test_checked_mean_is_the_plain_mean_where_it_is_finite now compares with math.fsum(...) / n, since sum() can differ from it in the last bit before 3.12. New tests pin the cancelling case, the int-first case, the exact integer mean and the sign_flip_summary mean. Changelog fragment (Fixed). Signed-off-by: Anton Volkov --- .../20261116_130000_checked-mean-fsum.md | 3 +++ dokimi/dokimi/eval/finite_delta.py | 11 +++++++++-- dokimi/tests/test_repo_compare_overflow.py | 19 +++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 dokimi/changelog.d/20261116_130000_checked-mean-fsum.md diff --git a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md new file mode 100644 index 000000000..a13deccb6 --- /dev/null +++ b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md @@ -0,0 +1,3 @@ +### Fixed + +- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers. A mean that is not affected by rounding is unchanged. diff --git a/dokimi/dokimi/eval/finite_delta.py b/dokimi/dokimi/eval/finite_delta.py index e74e8fb12..22403e738 100644 --- a/dokimi/dokimi/eval/finite_delta.py +++ b/dokimi/dokimi/eval/finite_delta.py @@ -61,8 +61,15 @@ def check_sum_range(values: Sequence[float], error: type[Exception] = OverflowEr def checked_mean(values: Sequence[float], error: type[Exception] = OverflowError, where: str = "") -> float: - """``sum(values) / len(values)`` after :func:`check_sum_range`.""" + """The mean of ``values`` after :func:`check_sum_range`, with an exactly rounded sum (``math.fsum``). + + The result does not depend on the Python version: ``sum`` of floats compensates from 3.12 on and + adds left to right before, so deltas that cancel left a rounding residue there. Integers only are + summed exactly as integers, since ``math.fsum`` would first round each to a float. + """ check_sum_range(values, error, where) if _has_non_finite_float(*values): # they decide; finite values summed first could overflow and cancel a -inf return sum(v for v in values if isinstance(v, float) and not math.isfinite(v)) / len(values) - return sum(values) / len(values) + if all(isinstance(v, int) for v in values): + return sum(values) / len(values) + return math.fsum(values) / len(values) diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 646756fcf..655918341 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -80,12 +80,27 @@ def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy(): def test_checked_mean_is_the_plain_mean_where_it_is_finite(): - assert checked_mean([0.1, 0.2, 0.4]) == sum([0.1, 0.2, 0.4]) / 3 - assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == sum([1.0e307, -1.0e307, 1.0e307]) / 3 + assert checked_mean([0.1, 0.2, 0.4]) == math.fsum([0.1, 0.2, 0.4]) / 3 # fsum: sum() before 3.12 can differ in the last bit + assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == math.fsum([1.0e307, -1.0e307, 1.0e307]) / 3 assert checked_mean([math.inf, 1.0]) == math.inf assert math.isnan(checked_mean([math.nan, 1.0])) +def test_checked_mean_sums_exactly_whatever_the_python_version(): + assert checked_mean([1e307] * 6 + [-1e307] * 6) == 0.0 + assert checked_mean([1, 1e16, 1.0, -1e16]) == 0.5 # sum() starts on an int and loses its compensation: 0.25 + + +def test_checked_mean_keeps_the_exact_mean_of_integers(): + assert checked_mean([2**60 + 1, 1, -(2**60)]) == 2 / 3 # fsum would round the first to a float: 1 / 3 + assert checked_mean([2**60 + 1, 2**60 + 1]) == float(2**60 + 1) + + +def test_sign_flip_summary_mean_is_the_exact_mean(): + row = sign_flip_summary([_task(d) for d in (1, 1e16, 1.0, -1e16)]) + assert row["delta"]["mean"] == 0.5 + + def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error(): with pytest.raises(CompareError, match="float range"): checked_delta(10**309, 0.0, CompareError) From 66447eff5571bc190b4eca180bdac783ca45aa61 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 17:18:40 +0000 Subject: [PATCH 2/5] fix(eval): checked_mean keeps integer contributions beside floats Settles the review point that `checked_mean([2**53 + 1, -(2**53), 0.0])` returned 0.0: `math.fsum` rounds each integer to a float first. Floats only still use `fsum` and integers only still sum as integers; a mix is now summed as exact fractions, so the result is 1 / 3. The test pins that case. Signed-off-by: Anton Volkov --- dokimi/changelog.d/20261116_130000_checked-mean-fsum.md | 2 +- dokimi/dokimi/eval/finite_delta.py | 8 ++++++-- dokimi/tests/test_repo_compare_overflow.py | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md index a13deccb6..23f02ffde 100644 --- a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md +++ b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md @@ -1,3 +1,3 @@ ### Fixed -- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers. A mean that is not affected by rounding is unchanged. +- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. diff --git a/dokimi/dokimi/eval/finite_delta.py b/dokimi/dokimi/eval/finite_delta.py index 22403e738..d553f0f67 100644 --- a/dokimi/dokimi/eval/finite_delta.py +++ b/dokimi/dokimi/eval/finite_delta.py @@ -14,6 +14,7 @@ import math from collections.abc import Sequence +from fractions import Fraction def _shown(value: float) -> str: @@ -65,11 +66,14 @@ def checked_mean(values: Sequence[float], error: type[Exception] = OverflowError The result does not depend on the Python version: ``sum`` of floats compensates from 3.12 on and adds left to right before, so deltas that cancel left a rounding residue there. Integers only are - summed exactly as integers, since ``math.fsum`` would first round each to a float. + summed exactly as integers, and integers mixed with floats as exact fractions, since ``math.fsum`` + would first round each integer to a float. """ check_sum_range(values, error, where) if _has_non_finite_float(*values): # they decide; finite values summed first could overflow and cancel a -inf return sum(v for v in values if isinstance(v, float) and not math.isfinite(v)) / len(values) + if all(isinstance(v, float) for v in values): + return math.fsum(values) / len(values) if all(isinstance(v, int) for v in values): return sum(values) / len(values) - return math.fsum(values) / len(values) + return float(sum((Fraction(v if isinstance(v, int) else float(v)) for v in values), Fraction(0)) / len(values)) # mixed: exact, as fsum rounds an integer first diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 655918341..1742ace46 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -96,6 +96,11 @@ def test_checked_mean_keeps_the_exact_mean_of_integers(): assert checked_mean([2**60 + 1, 2**60 + 1]) == float(2**60 + 1) +def test_checked_mean_keeps_integer_contributions_beside_floats(): + assert checked_mean([2**53 + 1, -(2**53), 0.0]) == 1 / 3 # fsum rounds the first to 2**53: 0.0 + assert checked_mean([2**60 + 1, -(2**60), 0.5]) == 0.5 + + def test_sign_flip_summary_mean_is_the_exact_mean(): row = sign_flip_summary([_task(d) for d in (1, 1e16, 1.0, -1e16)]) assert row["delta"]["mean"] == 0.5 From 3ded3db7369458bc4887be05901e3bb1358a82c8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 18:08:11 +0000 Subject: [PATCH 3/5] fix(eval): checked_mean refuses a total only the exact sum sees beyond the float range Settles the review point that checked_mean could leak a raw OverflowError on Python 3.11: the magnitude precheck used sum(), which rounds small terms away left to right and stays finite, then math.fsum raised. check_sum_range now sums the magnitudes with math.fsum, so the precheck and the mean agree on every version and the requested error, with its context, is raised. The test emulates the pre-3.12 sum() and checks the CompareError and the context prefix. Signed-off-by: Anton Volkov --- .../20261116_130000_checked-mean-fsum.md | 2 +- dokimi/dokimi/eval/finite_delta.py | 5 +++-- dokimi/tests/test_repo_compare_overflow.py | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md index 23f02ffde..d33317abb 100644 --- a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md +++ b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md @@ -1,3 +1,3 @@ ### Fixed -- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. +- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. diff --git a/dokimi/dokimi/eval/finite_delta.py b/dokimi/dokimi/eval/finite_delta.py index d553f0f67..8128f3531 100644 --- a/dokimi/dokimi/eval/finite_delta.py +++ b/dokimi/dokimi/eval/finite_delta.py @@ -54,8 +54,9 @@ def check_sum_range(values: Sequence[float], error: type[Exception] = OverflowEr addition is safe for the sign patterns of a sign-flip test: they are refused. """ try: - refused = not math.isfinite(sum(abs(v) for v in values)) and all(math.isfinite(v) for v in values) - except OverflowError: # an integer sum too large for a float: ``isfinite`` cannot read it + # fsum is exact: ``sum`` before 3.12 rounds small terms away and stays finite where the exact total does not + refused = not math.isfinite(math.fsum(abs(v) for v in values)) and all(math.isfinite(v) for v in values) + except OverflowError: # an exact sum or an integer too large for a float: ``fsum`` or ``isfinite`` cannot read it refused = not _has_non_finite_float(*values) if refused: raise error(f"{where}{': ' if where else ''}the sum of the magnitudes of the values leaves the float range") diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 1742ace46..3bb1989da 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -91,6 +91,20 @@ def test_checked_mean_sums_exactly_whatever_the_python_version(): assert checked_mean([1, 1e16, 1.0, -1e16]) == 0.5 # sum() starts on an int and loses its compensation: 0.25 +def test_checked_mean_refuses_a_total_that_only_the_exact_sum_sees_beyond_the_range(monkeypatch): + def left_to_right(values, start=0): # what sum() does before Python 3.12 + total = start + for v in values: + total += v + return total + + monkeypatch.setattr("dokimi.eval.finite_delta.sum", left_to_right, raising=False) + values = [float.fromhex("0x1.fffffffffffffp+1023"), 2.0**969, 2.0**969] + assert math.isfinite(left_to_right(values)) # the rounded total stays finite + with pytest.raises(CompareError, match=r"task 't': the sum of the magnitudes .* float range"): + checked_mean(values, CompareError, "task 't'") + + def test_checked_mean_keeps_the_exact_mean_of_integers(): assert checked_mean([2**60 + 1, 1, -(2**60)]) == 2 / 3 # fsum would round the first to a float: 1 / 3 assert checked_mean([2**60 + 1, 2**60 + 1]) == float(2**60 + 1) From cbafa1328a11d999c50b012adf0cc1846dc8262d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 18:51:13 +0000 Subject: [PATCH 4/5] fix(eval): check_sum_range sums the magnitudes of integers exactly Settles the review point that the range check refused integer-only and mixed inputs whose exact total fits the float range: math.fsum rounds each integer to a float first, so [m - 2**970 + 1, 2**970 - 1] (m the largest float) raised although the exact total is m. With an integer among the values the check now sums the magnitudes as exact fractions and refuses only when that total does not fit a float; float-only inputs keep math.fsum. A test pins both cases from the review (with and without an appended 0.0). Signed-off-by: Anton Volkov --- .../20261116_130000_checked-mean-fsum.md | 2 +- dokimi/dokimi/eval/finite_delta.py | 15 +++++++++++---- dokimi/tests/test_repo_compare_overflow.py | 7 +++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md index d33317abb..63e7c8761 100644 --- a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md +++ b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md @@ -1,3 +1,3 @@ ### Fixed -- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. +- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. With an integer among the values that range check sums the magnitudes exactly too, so integers whose exact total fits the float range are taken (`fsum` would round each to a float first and overflow). diff --git a/dokimi/dokimi/eval/finite_delta.py b/dokimi/dokimi/eval/finite_delta.py index 8128f3531..cab31dc6f 100644 --- a/dokimi/dokimi/eval/finite_delta.py +++ b/dokimi/dokimi/eval/finite_delta.py @@ -54,10 +54,17 @@ def check_sum_range(values: Sequence[float], error: type[Exception] = OverflowEr addition is safe for the sign patterns of a sign-flip test: they are refused. """ try: - # fsum is exact: ``sum`` before 3.12 rounds small terms away and stays finite where the exact total does not - refused = not math.isfinite(math.fsum(abs(v) for v in values)) and all(math.isfinite(v) for v in values) - except OverflowError: # an exact sum or an integer too large for a float: ``fsum`` or ``isfinite`` cannot read it - refused = not _has_non_finite_float(*values) + if _has_non_finite_float(*values): + refused = False + elif all(isinstance(v, float) for v in values): + # fsum is exact: ``sum`` before 3.12 rounds small terms away and stays finite where the exact total does not + refused = not math.isfinite(math.fsum(abs(v) for v in values)) + else: + # an integer: ``fsum`` would round it to a float first and could overflow where the exact total fits + float(sum((Fraction(abs(v)) for v in values), Fraction(0))) + refused = False + except OverflowError: # an exact sum or an integer too large for a float + refused = True if refused: raise error(f"{where}{': ' if where else ''}the sum of the magnitudes of the values leaves the float range") diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 3bb1989da..9e818db3f 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -115,6 +115,13 @@ def test_checked_mean_keeps_integer_contributions_beside_floats(): assert checked_mean([2**60 + 1, -(2**60), 0.5]) == 0.5 +@pytest.mark.parametrize("extra", [[], [0.0]]) +def test_checked_mean_takes_integers_whose_exact_total_fits_the_float_range(extra): + top = int(float.fromhex("0x1.fffffffffffffp+1023")) + values = [top - 2**970 + 1, 2**970 - 1, *extra] # fsum would round each integer first and overflow + assert checked_mean(values, CompareError) == top / len(values) + + def test_sign_flip_summary_mean_is_the_exact_mean(): row = sign_flip_summary([_task(d) for d in (1, 1e16, 1.0, -1e16)]) assert row["delta"]["mean"] == 0.5 From 5dcaf030f8dc0a25c8caa3e111b7965a9cb3a02f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 22:59:29 +0000 Subject: [PATCH 5/5] fix(eval): pin the exact-total refusal and the integer case on inputs main takes or refuses Settles the review points on the checked_mean tests and wording. The integer test took an input main already accepted (the integers summed as integers first); it now puts a float ahead of them, which main refused, so it fails there. A new test pins the refusal of `[m, 2**969, 2.0**969]`, where main returned the mean of the rounded total, and the changelog fragment names that refusal. The docstring says that only floats and integers are accepted, since `numpy.float32` was never part of the `Sequence[float]` contract. Signed-off-by: Anton Volkov --- .../20261116_130000_checked-mean-fsum.md | 2 +- dokimi/dokimi/eval/finite_delta.py | 3 ++- dokimi/tests/test_repo_compare_overflow.py | 15 +++++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md index 63e7c8761..10471ad64 100644 --- a/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md +++ b/dokimi/changelog.d/20261116_130000_checked-mean-fsum.md @@ -1,3 +1,3 @@ ### Fixed -- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. With an integer among the values that range check sums the magnitudes exactly too, so integers whose exact total fits the float range are taken (`fsum` would round each to a float first and overflow). +- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. With an integer among the values that range check sums the magnitudes exactly too, so integers whose exact total fits the float range are taken (`fsum` would round each to a float first and overflow; `main` refused them when a float came first, as in `[0.0, m - 2**970 + 1, 2**970 - 1]` with `m` the largest float). The mixed `[m, 2**969, 2.0**969]`, whose exact total passes the range, is refused now, where `main` returned the mean of the rounded total. diff --git a/dokimi/dokimi/eval/finite_delta.py b/dokimi/dokimi/eval/finite_delta.py index cab31dc6f..0a3862a34 100644 --- a/dokimi/dokimi/eval/finite_delta.py +++ b/dokimi/dokimi/eval/finite_delta.py @@ -75,7 +75,8 @@ def checked_mean(values: Sequence[float], error: type[Exception] = OverflowError The result does not depend on the Python version: ``sum`` of floats compensates from 3.12 on and adds left to right before, so deltas that cancel left a rounding residue there. Integers only are summed exactly as integers, and integers mixed with floats as exact fractions, since ``math.fsum`` - would first round each integer to a float. + would first round each integer to a float. ``values`` are floats or integers; another number type + (``numpy.float32``) is outside the ``Sequence[float]`` contract and raises ``TypeError``. """ check_sum_range(values, error, where) if _has_non_finite_float(*values): # they decide; finite values summed first could overflow and cancel a -inf diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 9e818db3f..0f72bd3d0 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -115,11 +115,18 @@ def test_checked_mean_keeps_integer_contributions_beside_floats(): assert checked_mean([2**60 + 1, -(2**60), 0.5]) == 0.5 -@pytest.mark.parametrize("extra", [[], [0.0]]) -def test_checked_mean_takes_integers_whose_exact_total_fits_the_float_range(extra): +def test_checked_mean_refuses_a_mixed_total_that_only_the_exact_sum_sees_beyond_the_range(): + values = [float.fromhex("0x1.fffffffffffffp+1023"), 2**969, 2.0**969] # a left-to-right sum rounds back to the maximum + with pytest.raises(CompareError, match=r"task 't': the sum of the magnitudes .* float range"): + checked_mean(values, CompareError, "task 't'") + + +@pytest.mark.parametrize("order", ["float first", "float between"]) +def test_checked_mean_takes_integers_whose_exact_total_fits_the_float_range(order): top = int(float.fromhex("0x1.fffffffffffffp+1023")) - values = [top - 2**970 + 1, 2**970 - 1, *extra] # fsum would round each integer first and overflow - assert checked_mean(values, CompareError) == top / len(values) + low, high = top - 2**970 + 1, 2**970 - 1 + values = [0.0, low, high] if order == "float first" else [low, 0.0, high] # a float among the integers: sum() rounded the partial total and overflowed + assert checked_mean(values, CompareError) == top / 3 def test_sign_flip_summary_mean_is_the_exact_mean():