Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dokimi/changelog.d/20261116_130000_checked-mean-fsum.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- **Mean of float deltas no longer depends on the Python version** — `checked_mean`, which gives the `repo compare` summary mean (`sign_flip_summary`) and the per-question means of the model comparison, added floats with `sum()`, which compensates only from Python 3.12: `[1e307] * 6 + [-1e307] * 6` gave `-2.079e290` on 3.11 and `0.0` on 3.12. It now sums with `math.fsum` (exactly rounded), so the mean is the same on every version and agrees with the interval. Integer-only inputs are still summed as exact integers, and integers mixed with floats as exact fractions (`[2**53 + 1, -(2**53), 0.0]` gives `1 / 3`, not `0.0`). A mean that is not affected by rounding is unchanged. The range check on the sum of magnitudes is exact too, so values whose exact total passes the float range (`0x1.fffffffffffffp+1023, 2**969, 2**969`) raise the requested error with its context on Python 3.11, not a bare `OverflowError`. With an integer among the values that range check sums the magnitudes exactly too, so integers whose exact total fits the float range are taken (`fsum` would round each to a float first and overflow; `main` refused them when a float came first, as in `[0.0, m - 2**970 + 1, 2**970 - 1]` with `m` the largest float). The mixed `[m, 2**969, 2.0**969]`, whose exact total passes the range, is refused now, where `main` returned the mean of the rounded total.
30 changes: 25 additions & 5 deletions dokimi/dokimi/eval/finite_delta.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import math
from collections.abc import Sequence
from fractions import Fraction


def _shown(value: float) -> str:
Expand Down Expand Up @@ -53,16 +54,35 @@ def check_sum_range(values: Sequence[float], error: type[Exception] = OverflowEr
addition is safe for the sign patterns of a sign-flip test: they are refused.
"""
try:
refused = not math.isfinite(sum(abs(v) for v in values)) and all(math.isfinite(v) for v in values)
except OverflowError: # an integer sum too large for a float: ``isfinite`` cannot read it
refused = not _has_non_finite_float(*values)
if _has_non_finite_float(*values):
refused = False
elif all(isinstance(v, float) for v in values):
# fsum is exact: ``sum`` before 3.12 rounds small terms away and stays finite where the exact total does not
refused = not math.isfinite(math.fsum(abs(v) for v in values))
else:
# an integer: ``fsum`` would round it to a float first and could overflow where the exact total fits
float(sum((Fraction(abs(v)) for v in values), Fraction(0)))
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
refused = False
except OverflowError: # an exact sum or an integer too large for a float
refused = True
if refused:
raise error(f"{where}{': ' if where else ''}the sum of the magnitudes of the values leaves the float range")


def checked_mean(values: Sequence[float], error: type[Exception] = OverflowError, where: str = "") -> float:
"""``sum(values) / len(values)`` after :func:`check_sum_range`."""
"""The mean of ``values`` after :func:`check_sum_range`, with an exactly rounded sum (``math.fsum``).

The result does not depend on the Python version: ``sum`` of floats compensates from 3.12 on and
adds left to right before, so deltas that cancel left a rounding residue there. Integers only are
summed exactly as integers, and integers mixed with floats as exact fractions, since ``math.fsum``
would first round each integer to a float. ``values`` are floats or integers; another number type
(``numpy.float32``) is outside the ``Sequence[float]`` contract and raises ``TypeError``.
"""
check_sum_range(values, error, where)
if _has_non_finite_float(*values): # they decide; finite values summed first could overflow and cancel a -inf
return sum(v for v in values if isinstance(v, float) and not math.isfinite(v)) / len(values)
return sum(values) / len(values)
if all(isinstance(v, float) for v in values):
return math.fsum(values) / len(values)
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
if all(isinstance(v, int) for v in values):
return sum(values) / len(values)
return float(sum((Fraction(v if isinstance(v, int) else float(v)) for v in values), Fraction(0)) / len(values)) # mixed: exact, as fsum rounds an integer first
52 changes: 50 additions & 2 deletions dokimi/tests/test_repo_compare_overflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,12 +80,60 @@ def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy():


def test_checked_mean_is_the_plain_mean_where_it_is_finite():
assert checked_mean([0.1, 0.2, 0.4]) == sum([0.1, 0.2, 0.4]) / 3
assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == sum([1.0e307, -1.0e307, 1.0e307]) / 3
assert checked_mean([0.1, 0.2, 0.4]) == math.fsum([0.1, 0.2, 0.4]) / 3 # fsum: sum() before 3.12 can differ in the last bit
assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == math.fsum([1.0e307, -1.0e307, 1.0e307]) / 3
assert checked_mean([math.inf, 1.0]) == math.inf
assert math.isnan(checked_mean([math.nan, 1.0]))


def test_checked_mean_sums_exactly_whatever_the_python_version():
assert checked_mean([1e307] * 6 + [-1e307] * 6) == 0.0
assert checked_mean([1, 1e16, 1.0, -1e16]) == 0.5 # sum() starts on an int and loses its compensation: 0.25


def test_checked_mean_refuses_a_total_that_only_the_exact_sum_sees_beyond_the_range(monkeypatch):
def left_to_right(values, start=0): # what sum() does before Python 3.12
total = start
for v in values:
total += v
return total

monkeypatch.setattr("dokimi.eval.finite_delta.sum", left_to_right, raising=False)
values = [float.fromhex("0x1.fffffffffffffp+1023"), 2.0**969, 2.0**969]
assert math.isfinite(left_to_right(values)) # the rounded total stays finite
with pytest.raises(CompareError, match=r"task 't': the sum of the magnitudes .* float range"):
checked_mean(values, CompareError, "task 't'")


def test_checked_mean_keeps_the_exact_mean_of_integers():
assert checked_mean([2**60 + 1, 1, -(2**60)]) == 2 / 3 # fsum would round the first to a float: 1 / 3
assert checked_mean([2**60 + 1, 2**60 + 1]) == float(2**60 + 1)


def test_checked_mean_keeps_integer_contributions_beside_floats():
assert checked_mean([2**53 + 1, -(2**53), 0.0]) == 1 / 3 # fsum rounds the first to 2**53: 0.0
assert checked_mean([2**60 + 1, -(2**60), 0.5]) == 0.5


def test_checked_mean_refuses_a_mixed_total_that_only_the_exact_sum_sees_beyond_the_range():
values = [float.fromhex("0x1.fffffffffffffp+1023"), 2**969, 2.0**969] # a left-to-right sum rounds back to the maximum
with pytest.raises(CompareError, match=r"task 't': the sum of the magnitudes .* float range"):
checked_mean(values, CompareError, "task 't'")


@pytest.mark.parametrize("order", ["float first", "float between"])
def test_checked_mean_takes_integers_whose_exact_total_fits_the_float_range(order):
top = int(float.fromhex("0x1.fffffffffffffp+1023"))
low, high = top - 2**970 + 1, 2**970 - 1
values = [0.0, low, high] if order == "float first" else [low, 0.0, high] # a float among the integers: sum() rounded the partial total and overflowed
assert checked_mean(values, CompareError) == top / 3


def test_sign_flip_summary_mean_is_the_exact_mean():
row = sign_flip_summary([_task(d) for d in (1, 1e16, 1.0, -1e16)])
assert row["delta"]["mean"] == 0.5


def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error():
with pytest.raises(CompareError, match="float range"):
checked_delta(10**309, 0.0, CompareError)
Expand Down
Loading