diff --git a/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md b/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md new file mode 100644 index 000000000..ad8b0794f --- /dev/null +++ b/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md @@ -0,0 +1,3 @@ +### Fixed + +- **Sign-flip p of a delta that dwarfs the others** — `sign_flip_p` added the signed deltas as floats, so from a delta of about 1e16 on the small ones were absorbed by the large one and all 64 sign patterns of `[1e17, 1, 1, 1, 1, 1]` summed to the same 1e17: its p was 1.0 and is now 0.03125, as for `1e15`. The sums are now exact (each delta an integer multiple of one power of two, so the p no longer depends on the order or the Python that adds them), for the exact enumeration and the Monte Carlo draws alike; the rounding slack that lets a pattern tie the observed sum (0.1 + 0.2 against 0.3) is unchanged. Rows of ordinary scale give the same p; the `1e9 + 1/3` row, whose sums only tie in exact arithmetic, is 0.0625 on every Python (0.046875 before on Python 3.11, which sums left to right). The interval search of a row of up to 16 changed deltas that span more than 40 binary orders of magnitude (`[1e17, 1, 1, 1, 1, 1]`) tests each shift with the same exact sums, so its lower end is 1.0, not the 0.0001 the float sums gave; a row of more than 16 such deltas keeps the float search. diff --git a/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md b/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md new file mode 100644 index 000000000..ba2bb4e5c --- /dev/null +++ b/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md @@ -0,0 +1,3 @@ +### Fixed + +- **`paired_test` of finite deltas whose sum leaves the float range** — a row of deltas such as `[1e308 * (g / 9) for g in [2, 4, 3, 5, 6, 9, 7, 8]]` (each finite, their magnitudes summing to 4.9e308) was refused with an `OverflowError` for the sum of the magnitudes, though its mean (6.1e307), sd and sign-flip p all fit. `paired_test`, `arm_significance` and `interaction_test` now give the mean, sd, p and interval of such a row, and refuse a row only when its sd is beyond the float range (`[1.7e308, -1.7e308] * 3`), as before, naming the sd. `interaction_test` refuses a row whose two arm deltas add past the float range (its additive expectation would be `inf`). `model compare` sections keep refusing a sum past the range, naming the run. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 72c0776b5..8463794a8 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -368,7 +368,10 @@ records its `correction_family`, `family_size` (every member), and `family_teste Each row's `p` is one two-sided sign-flip test on the per-question deltas: under the null each delta's sign is a fair coin, and p is the share of sign patterns whose |sum| reaches the observed one (exact up to 16 changed pairs, Monte Carlo -with a fixed seed above). A delta within 1e-9 of 0 is a tie and carries no sign, +with a fixed seed above). The sums are exact, not float sums: a delta of 1e17 does not +hide five deltas of 1, and a pattern whose sum falls short of the observed one by no more +than the rounding slack of the deltas (at most 1e-9) still reaches it (0.1 + 0.2 against +0.3). A delta within 1e-9 of 0 is a tie and carries no sign, unless the values of its pair are below 1: float noise in a delta scales with the values subtracted, so a pair whose larger value is m below 1 ties within 1e-9 times m (a score of 0.3 against 0.1 + 0.2 is a tie; 1e-12 against 0 is a change, as 1e-3 against 0 @@ -446,7 +449,8 @@ heading, when no arm has a row); the significance of the other arms stays when t family test itself succeeds. A mean of finite scores or of finite paired differences whose sum would pass the float range is computed exactly (`statistics.mean`), so the per-arm `avg_aggregate` and `delta_vs_baseline` are refused only when a single -difference leaves the range. The console summary of `arms run` prints +difference leaves the range. A significance row computes the mean, the sd and the +sign-flip p of such differences too, and is refused only when its sd leaves the range; an interaction row is also refused when the sum of its two arm deltas (its additive expectation) does. The console summary of `arms run` prints `(delta not computed: )` for such an arm. Everything above assumes the judge's continuous 0–100 score. The diff --git a/dokimi/dokimi/eval/interactions.py b/dokimi/dokimi/eval/interactions.py index 63fb134d2..b34e57e6c 100644 --- a/dokimi/dokimi/eval/interactions.py +++ b/dokimi/dokimi/eval/interactions.py @@ -115,6 +115,8 @@ def _interaction_contrast( res = _describe_deltas(contrasts, confidence=confidence, seed=seed, magnitudes=magnitudes) delta_a = _arm_delta(diffs_a, confidence, seed, pair_magnitudes(col_a, bases)) delta_b = _arm_delta(diffs_b, confidence, seed, pair_magnitudes(col_b, bases)) + if math.isfinite(delta_a) and math.isfinite(delta_b) and not math.isfinite(delta_a + delta_b): # the row's additive expectation would print as inf next to a computed p + raise OverflowError("deltas too large: the sum of the two arm deltas leaves the float range") res.update( { "arm_a": arm_a, diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index 4b26af6ee..859158aed 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -8,7 +8,7 @@ :func:`min_sign_flip_p` is the floor it cannot go below over k changed deltas, ``2 / 2**k`` exact. ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as -:func:`sign_flip_p` computes it on them: a shifted delta within the tie width of its pair (:func:`tie_widths`, kept +:func:`sign_flip_p` computes it on them (that one adds the deltas as exact integers; this one, below, in floats, so its p can differ at the ends of a row whose deltas floats cannot resolve): a shifted delta within the tie width of its pair (:func:`tie_widths`, kept from the original deltas) of 0 is a tie, so a delta tied at 0 counts again once ``theta`` moves off 0. Exact, a sign pattern counts when its sum falls short of the observed one by at most the rounding error of adding the deltas summed (:func:`_sum_slack`; :data:`EPS` at most, and no more than the largest tie width among them), @@ -39,12 +39,12 @@ from __future__ import annotations import functools -import itertools import math import random import statistics import sys from collections.abc import Sequence +from fractions import Fraction from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -113,9 +113,12 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. ``None`` when every delta is a tie: there is no evidence either way, and 1.0 would read as a measured null. - Rounded to 6 places as printed, unless ``exact``. Deltas whose largest magnitude is above ``max float / (8 * n)`` - (or whose values are all below 2**-664) are scaled by a power of two (exact) first, as in - :func:`sign_flip_interval`, so no sum leaves the float range. + Rounded to 6 places as printed, unless ``exact``. The sums are exact: each delta is an integer multiple of one + power of two, so a delta of 1e300 does not hide five of 1, and the p does not depend on the order or the Python + that adds them. A pattern whose sum falls short of the observed one by at most the rounding slack of the deltas + (:func:`_sum_slack`, never more than :data:`EPS`) still counts, as sums of decimals do (0.1 + 0.2 against 0.3). Deltas whose largest magnitude is above + ``max float / (8 * n)`` (or whose values are all below 2**-664) are divided by a power of two (exact) first, as in + :func:`sign_flip_interval`, so the tie cap and the interval's scale agree. ``magnitudes`` holds the larger value of each pair (:func:`pair_magnitudes`), which sets each tie width (:func:`tie_widths`); without it a tie is within :data:`EPS` of 0. @@ -128,21 +131,42 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL # the slack on a sum follows the pairs the sum is of, not a tied pair's larger values unit = tie_scale(None if magnitudes is None else [magnitudes[i] for i in active]) k = _scale_exponent(d, unit) - if k: # near the float maximum the sums go to inf/nan: the same p on the deltas divided by a power of two + if k: # the same p on the deltas divided by a power of two, as the interval searches them d = [math.ldexp(x, -k) for x in d] - eps = _sum_slack(d, EPS * math.ldexp(unit, -k)) - observed = abs(sum(d)) - eps + cap_eps = EPS * math.ldexp(unit, -k) if len(d) <= EXACT_MAX: - hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed - for signs in itertools.product((1, -1), repeat=len(d))) - p = hits / 2 ** len(d) + p = _enumerated_p(d, cap_eps) else: + ints, cap = _as_integers(d, cap_eps) + observed = _observed(ints, cap) rng = random.Random(seed) - hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) + hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in ints)) >= observed for _ in range(n_resamples)) p = (hits + 1) / (n_resamples + 1) return p if exact else round(p, 6) +def _observed(ints: list[int], cap: int) -> int: + """The observed ``|sum|`` less the rounding slack of the deltas (:func:`_sum_slack`, in the units of ``ints``).""" + return abs(sum(ints)) - min(cap, (len(ints) * sum(map(abs, ints))) >> 52) + + +def _enumerated_p(d: list[float], cap_eps: float) -> float: + """The exact p over every sign pattern of ``d``, whose sums are integers: a small delta is not lost against a large one.""" + ints, cap = _as_integers(d, cap_eps) + observed = _observed(ints, cap) + sums = [0] + for x in ints: + sums = [s + x for s in sums] + [s - x for s in sums] + return sum(abs(s) >= observed for s in sums) / 2 ** len(d) + + +def _as_integers(values: Sequence[float], cap: float) -> tuple[list[int], int]: + """``values`` and ``cap`` as integer multiples of one power of two, so that sums of them are exact; ``cap`` rounded down.""" + ratios = [v.as_integer_ratio() for v in values] + denominator = max(den for _, den in ratios) + return [num * (denominator // den) for num, den in ratios], math.floor(Fraction(cap) * denominator) + + @functools.cache def _patterns(k: int) -> np.ndarray: """Every sign pattern of k deltas, one per row: ``2**k x k``, as floats for the product.""" @@ -173,8 +197,9 @@ class ShiftTest: """The sign-flip p of ``deltas - theta`` over every delta at any ``theta``, unrounded; at 0 it is :func:`sign_flip_p` of ``deltas``, Monte Carlo draws included, up to rounding. From deltas of about 1e6 on, a sign pattern whose sum ties the observed one in exact arithmetic (magnitudes that repeat, such as 1e9 + 0.1) - can round to one side of it in the product and to the other in ``sum()``, so ``p(0)`` can differ there: the - row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`).""" + can round to one side of it in the product and to the other in the exact sums of :func:`sign_flip_p`, so ``p(0)`` can differ there: the + row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`). Up to :data:`EXACT_MAX` shifted deltas that span more than + 40 binary orders of magnitude are summed exactly too, as floats lose the small ones there; more of them are not (#865).""" def __init__(self, deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, eps: float | Sequence[float] = EPS) -> None: @@ -197,6 +222,9 @@ def p(self, theta: float) -> float | None: moved = x[x != 0.0] if not moved.size: return None + spread = math.frexp(float(np.abs(moved).max()))[1] - math.frexp(float(np.abs(moved).min()))[1] # in binary orders + if moved.size <= EXACT_MAX and spread > 40: + return _enumerated_p(moved.tolist(), float(self.widths[~tied].max())) # floats lose the small deltas eps = _sum_slack(moved.tolist(), float(self.widths[~tied].max())) observed = abs(math.fsum(moved.tolist())) # exactly rounded: sum() rounds differently before 3.12 if moved.size <= EXACT_MAX: diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 6fa61bc04..61381f313 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -65,7 +65,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from dokimi.eval.arm_summary import mean_in_range -from dokimi.eval.finite_delta import check_sum_range, checked_delta +from dokimi.eval.finite_delta import checked_delta from dokimi.eval.sign_flip import ( ALPHA, METHOD, @@ -631,9 +631,8 @@ def _describe_deltas( d_sd = _sample_sd(d) cohens_d = _cohens_d(d_mean, d_sd) if all(math.isfinite(x) for x in d): - # The sign-flip sums would overflow and read as a measured zero effect. - # A finite sum of magnitudes bounds the spread (sd**2 <= sum(x**2) <= that sum squared), so it is finite too. - check_sum_range(d) + if not math.isfinite(d_sd): # the spread of finite deltas past the float range: refused, as before + raise OverflowError("the sd of the deltas leaves the float range") row = sign_flip_row(d, seed=seed, exact=True, magnitudes=magnitudes) p, lo, hi, shown_lo, shown_hi = row["p"], row["lo"], row["hi"], row["ci_lo"], row["ci_hi"] else: # a NaN delta is no evidence either way: no p, no verdict, open ends diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index 41b81faf5..acc3121da 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -941,12 +941,12 @@ def _adjusted_significance( if paired: with_paired, without_paired = zip(*paired, strict=True) try: + finite_delta.check_sum_range([w - wo for w, wo in paired]) # the section's averages are float sums: +inf sigs[run_id] = significance_test(list(with_paired), list(without_paired), rounded=False) - except OverflowError as exc: # finite deltas whose sum or spread leaves the float range + except OverflowError as exc: # finite deltas whose sum leaves the float range raise InputError(f"run '{run_id}': {exc}") from exc else: sigs[run_id] = None - # The arms' own correction, so the two verdicts cannot drift; a run with no p, or under the floor rule, stays out. family = [sig for rid in run_ids if (sig := sigs[rid])] arm_stats._apply_correction(family, correction=correction, label="runs") diff --git a/dokimi/tests/test_interactions_float_range.py b/dokimi/tests/test_interactions_float_range.py index 15ed1fb21..0bb37f658 100644 --- a/dokimi/tests/test_interactions_float_range.py +++ b/dokimi/tests/test_interactions_float_range.py @@ -84,6 +84,15 @@ def scores(ab, a, b, base): assert res["p"] is None +def test_two_arm_deltas_whose_sum_leaves_the_float_range_raise_instead_of_an_inf_additive_expectation(): + # every delta and the contrast fit (the sd is finite), but 1e308 + 1e308 is the row's additive expectation + evaluations = [_ev(f"q{i}", 1.5e308 + i * 1e306, 1e308, 1e308, 0.0) for i in range(8)] + with pytest.raises(OverflowError, match="the sum of the two arm deltas leaves the float range"): + _test(evaluations) + with pytest.raises(OverflowError, match="the sum of the two arm deltas leaves the float range"): + interaction_significance(evaluations, ARMS, "baseline") + + def test_a_near_range_contrast_that_stays_finite_returns_finite_numbers(): evaluations = [_ev("q1", BIG, BIG, BIG, BIG), _ev("q2", -BIG, -BIG, -BIG, -BIG), _ev("q3", 1.0, 0.0, 0.0, 0.0)] res = _test(evaluations) diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 0fa4f2c86..d563c0ada 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -5,10 +5,7 @@ from __future__ import annotations -import functools import json -import math -import operator import random import pytest @@ -59,34 +56,14 @@ def test_an_end_is_open_at_five_or_fewer_tasks_and_bounded_from_six(): assert sign_flip_interval([5.0] * 6, sign_flip_p([5.0] * 6)) == (5.0, 5.0) -def _left_to_right(xs, start=0.0): - """``sum`` of floats as Python 3.11 adds them: in order, uncompensated (3.12 made ``sum`` compensated).""" - return functools.reduce(operator.add, xs, start) - - -@pytest.mark.parametrize("summed", [sum, _left_to_right], ids=["builtin", "left_to_right"]) -def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(monkeypatch, summed): +def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(): # 1e16 + 1.0 == 1e16: a far bound 1 past the extreme tied every shifted delta and left the end open - monkeypatch.setattr(sign_flip, "sum", summed, raising=False) # 3.11's sum() put 2e16's end at 2e16 + 4 for v in (1e16, 1e150): # finite measures go up to SCORE_BOUND, 1e150 assert sign_flip_interval([v] * 6, sign_flip_p([v] * 6)) == (v, v) assert sign_flip_interval([-v] * 6, sign_flip_p([-v] * 6)) == (-v, -v) assert sign_flip_interval([1e16] * 5 + [2e16], sign_flip_p([1e16] * 5 + [2e16])) == (1e16, 2e16) -def test_the_ends_are_the_same_whether_python_sums_compensated_or_left_to_right(monkeypatch): - # sum() of floats is compensated from Python 3.12 on and not before: a row's ends must not depend on the Python - rng = random.Random(11) - rows = [[scale * rng.uniform(-0.6, 1.0) for _ in range(rng.randint(6, 24))] - for scale in (1e6, 1e13, 1e16, 1e100) for _ in range(10)] - ps = [sign_flip_p(d) for d in rows] - ends = {} - for name, summed in (("builtin", sum), ("compensated", math.fsum), ("left_to_right", _left_to_right)): - monkeypatch.setattr(sign_flip, "sum", summed, raising=False) - ends[name] = [sign_flip_interval(d, p, rounded=False) for d, p in zip(rows, ps, strict=True)] - assert ends["builtin"] == ends["compensated"] == ends["left_to_right"] - - def test_large_deltas_keep_the_ends_and_the_p_of_the_shifted_deltas(): # at 1e7 the product's rounding passed EPS: the observed pattern stopped counting and both ends moved d = [0.62, 0.18, 0.94, -0.37, 0.71, 0.29, 0.83, 0.56, -0.05] @@ -124,15 +101,12 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None -@pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)], - ids=["compensated", "left_to_right"]) -def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p, lo): - # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625; the - # printed p itself is sum()'s, 0.0625 compensated (3.12 on) and 0.047 left to right (3.11): 0 follows either - monkeypatch.setattr(sign_flip, "sum", summed, raising=False) +def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(): + # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625, which adds + # the deltas as integers (exact on every Python): 0 follows it a, b, c = 1e9 + 1 / 3, 1e9 + 0.1, 2e9 + 0.2 row = sign_flip_summary(_pairs([a, b, c, a, -b, a, b])) - assert row["p"] == p and row["delta"]["lo"] == lo and row["interval_p_disagree"] is False + assert row["p"] == 0.0625 and row["delta"]["lo"] == 0.0 and row["interval_p_disagree"] is False def test_the_interval_excludes_zero_exactly_when_p_is_below_005(): diff --git a/dokimi/tests/test_sign_flip_float_range.py b/dokimi/tests/test_sign_flip_float_range.py index 2350fad80..160ac6c6a 100644 --- a/dokimi/tests/test_sign_flip_float_range.py +++ b/dokimi/tests/test_sign_flip_float_range.py @@ -86,3 +86,28 @@ def test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled(ro assert (big_row[end] is None) == (want is None) if want is not None: assert big_row[end] == pytest.approx(want, rel=1e-12) + + +@pytest.mark.parametrize("m", [1e15, 1e16, 1e17, 1e20, 1e300]) +def test_a_delta_dwarfing_the_others_does_not_hide_them_in_the_exact_p(m): + # benchmarking-and-comparison.md: p is the share of sign patterns whose |sum| reaches the observed one. The five + # deltas of 1 share a sign and none ties, so only the all-same-sign pattern and its mirror reach it: 2 / 2**6 (#816) + assert sign_flip_p([m, 1, 1, 1, 1, 1], exact=True) == 2 / 2 ** 6 + + +@pytest.mark.parametrize("m", [1e16, 1e17, 1e300]) +def test_the_lower_end_of_a_row_with_a_dwarfing_delta_is_where_the_small_deltas_stop_rejecting(m): + # benchmarking-and-comparison.md: the interval is every shift the same test does not reject. The five deltas of 1 + # stay on one side until the shift reaches 1, so the end is 1, not the 0.0001 a float search read as measured (#867) + row = sign_flip_row([m, 1, 1, 1, 1, 1], exact=True) + assert row["p"] == 2 / 2 ** 6 and row["lo"] == pytest.approx(1.0, abs=1e-6) + + +def test_a_delta_dwarfing_the_others_does_not_hide_them_in_the_monte_carlo_p(): + # 18 changed deltas, above EXACT_MAX: the same draws as for 1e15, and p at its floor 1 / (N_RESAMPLES + 1) (#816) + assert sign_flip_p([1e17] + [1.0] * 17, exact=True) == sign_flip_p([1e15] + [1.0] * 17, exact=True) == 1 / 20_001 + + +def test_huge_deltas_that_cancel_leave_the_p_of_the_small_ones_to_the_scale_where_float_sums_are_exact(): + # +-1e17 cancel in the observed sum 6, as +-1e5 do; 1e5 + 1 is still a float sum without a rounding error (#816) + assert sign_flip_p([1e17, -1e17] + [1] * 6, exact=True) == sign_flip_p([1e5, -1e5] + [1] * 6, exact=True) == 33 / 64 diff --git a/dokimi/tests/test_stats.py b/dokimi/tests/test_stats.py index 9c88b88f2..95fd7f73a 100644 --- a/dokimi/tests/test_stats.py +++ b/dokimi/tests/test_stats.py @@ -780,11 +780,13 @@ def test_mean_ci_raises_when_the_interval_leaves_the_float_range(): def test_paired_test_raises_when_the_deltas_leave_the_float_range(): - """Sign-flip sums that overflow would read as a measured zero effect: an error instead.""" + """A spread past the float range would be an inf delta_std next to a finite mean: an error instead. + + Test-narrowed: 9e307 no longer raises. Its sd is 9.9e307, so the mean, sd and sign-flip sums are all computed (#832). + """ pytest.importorskip("scipy") - for big in (9e307, 1.7e308): - with pytest.raises(OverflowError): - stats.paired_test([big, -big] * 3, [0.0] * 6, rounded=False) + with pytest.raises(OverflowError, match="the sd of the deltas leaves the float range"): + stats.paired_test([1.7e308, -1.7e308] * 3, [0.0] * 6, rounded=False) res = stats.paired_test([1e300, -1e300] * 3, [0.0] * 6, rounded=False) assert res is not None and math.isfinite(res["delta_std"]) diff --git a/dokimi/tests/test_stats_float_range.py b/dokimi/tests/test_stats_float_range.py index f78e7e8bc..a611444f8 100644 --- a/dokimi/tests/test_stats_float_range.py +++ b/dokimi/tests/test_stats_float_range.py @@ -63,6 +63,20 @@ def test_paired_test_of_a_delta_just_inside_the_range_still_runs(): assert stats._deltas([BIG], [-9e306]) == [BIG + 9e306] +def test_paired_test_of_finite_deltas_whose_sum_leaves_the_range_runs_on_them(): + # benchmarking-and-comparison.md: a row is refused only where its sd leaves the float range, not its sum (#832). + # 1e308 * g / 9 would be inf already at g = 2, so the deltas are scaled first; their sum is 4.9e308 + deltas = [1e308 * (g / 9) for g in [2, 4, 3, 5, 6, 9, 7, 8]] + assert all(math.isfinite(d) for d in deltas) and not math.isfinite(sum(deltas)) + result = stats.paired_test(deltas, [0.0] * 8) + assert result["delta_mean"] == pytest.approx(6.111111111111111e307, rel=1e-12) + assert result["exact"]["delta_mean"] == pytest.approx(6.111111111111111e307, rel=1e-12) + assert result["delta_std"] == pytest.approx(2.721655269759087e307, rel=1e-12) + assert result["cohens_d"] == 2.245 + assert (result["p"], result["n_changed"], result["significant"]) == (0.0078, 8, True) + assert result["ci_lo"] <= result["delta_mean"] <= result["ci_hi"] and math.isfinite(result["ci_hi"]) + + def test_paired_test_refuses_integers_whose_exact_difference_leaves_the_range(): with pytest.raises(OverflowError, match="float range"): stats.paired_test([10 ** 308] * 4, [-(10 ** 308)] * 4)