From 0006c92c45b7b207e13ea62375a0c85d506d6a89 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 10:46:46 +0000 Subject: [PATCH 1/3] fix(eval): the sign-flip p adds its deltas exactly, so a huge delta does not hide the small ones (#816) sign_flip_p added the signed deltas as floats. From a delta of about 1e16 on, the small deltas were absorbed by the large one: all 64 sign patterns of [1e17, 1, 1, 1, 1, 1] summed to the same 1e17, so every pattern reached the observed sum and sign_flip_p was 1.0 instead of 2 / 2**6, the p of the same row at 1e15. The rounding slack is not the cause: it is capped at EPS, 1e-9 there. The deltas are now integer multiples of one power of two and their sums are exact, for the exact enumeration (each pattern's sum built by doubling) and for the Monte Carlo draws, which are the same draws for a seed. Compensated float sums (math.fsum) are not enough: at 1e17 every pattern's sum rounds to the same float. The slack that lets a pattern tie the observed sum up to float noise (0.1 + 0.2 against 0.3) is kept as it was, in the same units. Pinned values that change: test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart no longer has a p that depends on the Python's sum() (0.0625 on both, 0.046875 before on a left-to-right sum); test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled no longer compares the lower end of [max, 1, 1, 1, 1, 1], which the float search of ShiftTest does not resolve once the p stops reading no evidence (Test-narrowed: the end is the product's noise at either scale). Signed-off-by: Anton Volkov --- .../20261117_100000_sign-flip-p-exact-sums.md | 3 ++ dokimi/docs/benchmarking-and-comparison.md | 5 ++- dokimi/dokimi/eval/sign_flip.py | 40 +++++++++++++------ dokimi/tests/test_repo_compare_interval.py | 11 +++-- dokimi/tests/test_sign_flip_float_range.py | 28 +++++++++++-- 5 files changed, 63 insertions(+), 24 deletions(-) create mode 100644 dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md diff --git a/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md b/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md new file mode 100644 index 000000000..7dec7077a --- /dev/null +++ b/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md @@ -0,0 +1,3 @@ +### Fixed + +- **Sign-flip p of a delta that dwarfs the others** — `sign_flip_p` added the signed deltas as floats, so from a delta of about 1e16 on the small ones were absorbed by the large one and all 64 sign patterns of `[1e17, 1, 1, 1, 1, 1]` summed to the same 1e17: its p was 1.0 and is now 0.03125, as for `1e15`. The sums are now exact (each delta an integer multiple of one power of two, so the p no longer depends on the order or the Python that adds them), for the exact enumeration and the Monte Carlo draws alike; the rounding slack that lets a pattern tie the observed sum (0.1 + 0.2 against 0.3) is unchanged. Rows of ordinary scale give the same p; the `1e9 + 1/3` row, whose sums only tie in exact arithmetic, is 0.0625 on every Python (0.046875 before on Python 3.11, which sums left to right). The interval's ends are still the float search of `ShiftTest`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4298fd90d..542c83f31 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -368,7 +368,10 @@ records its `correction_family`, `family_size` (every member), and `family_teste Each row's `p` is one two-sided sign-flip test on the per-question deltas: under the null each delta's sign is a fair coin, and p is the share of sign patterns whose |sum| reaches the observed one (exact up to 16 changed pairs, Monte Carlo -with a fixed seed above). A delta within 1e-9 of 0 is a tie and carries no sign, +with a fixed seed above). The sums are exact, not float sums: a delta of 1e17 does not +hide five deltas of 1, and a pattern whose sum falls short of the observed one by no more +than the rounding slack of the deltas (at most 1e-9) still reaches it (0.1 + 0.2 against +0.3). A delta within 1e-9 of 0 is a tie and carries no sign, unless the values of its pair are below 1: float noise in a delta scales with the values subtracted, so a pair whose larger value is m below 1 ties within 1e-9 times m (a score of 0.3 against 0.1 + 0.2 is a tie; 1e-12 against 0 is a change, as 1e-3 against 0 diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index 4b26af6ee..b001aa007 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -8,7 +8,7 @@ :func:`min_sign_flip_p` is the floor it cannot go below over k changed deltas, ``2 / 2**k`` exact. ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as -:func:`sign_flip_p` computes it on them: a shifted delta within the tie width of its pair (:func:`tie_widths`, kept +:func:`sign_flip_p` computes it on them (that one adds the deltas as exact integers; this one, below, in floats): a shifted delta within the tie width of its pair (:func:`tie_widths`, kept from the original deltas) of 0 is a tie, so a delta tied at 0 counts again once ``theta`` moves off 0. Exact, a sign pattern counts when its sum falls short of the observed one by at most the rounding error of adding the deltas summed (:func:`_sum_slack`; :data:`EPS` at most, and no more than the largest tie width among them), @@ -39,12 +39,12 @@ from __future__ import annotations import functools -import itertools import math import random import statistics import sys from collections.abc import Sequence +from fractions import Fraction from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -66,6 +66,8 @@ #: A vector whose values are all below this is searched on a copy multiplied by a power of two (exact), so the #: tie widths, the bisection step and the sums stay clear of the subnormal range. _TINY = 2.0 ** -664 +#: ``sum`` of integers (exact on every Python), kept apart from the module's ``sum`` that tests swap for float sums. +_int_sum = sum def pair_magnitudes(treatment: Sequence[float], baseline: Sequence[float]) -> list[float]: @@ -113,9 +115,12 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. ``None`` when every delta is a tie: there is no evidence either way, and 1.0 would read as a measured null. - Rounded to 6 places as printed, unless ``exact``. Deltas whose largest magnitude is above ``max float / (8 * n)`` - (or whose values are all below 2**-664) are scaled by a power of two (exact) first, as in - :func:`sign_flip_interval`, so no sum leaves the float range. + Rounded to 6 places as printed, unless ``exact``. The sums are exact: each delta is an integer multiple of one + power of two, so a delta of 1e300 does not hide five of 1, and the p does not depend on the order or the Python + that adds them. A pattern whose sum falls short of the observed one by at most the rounding slack of the deltas + (:func:`_sum_slack`, never more than :data:`EPS`) still counts, as sums of decimals do (0.1 + 0.2 against 0.3). Deltas whose largest magnitude is above + ``max float / (8 * n)`` (or whose values are all below 2**-664) are divided by a power of two (exact) first, as in + :func:`sign_flip_interval`, so the tie cap and the interval's scale agree. ``magnitudes`` holds the larger value of each pair (:func:`pair_magnitudes`), which sets each tie width (:func:`tie_widths`); without it a tie is within :data:`EPS` of 0. @@ -128,21 +133,30 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL # the slack on a sum follows the pairs the sum is of, not a tied pair's larger values unit = tie_scale(None if magnitudes is None else [magnitudes[i] for i in active]) k = _scale_exponent(d, unit) - if k: # near the float maximum the sums go to inf/nan: the same p on the deltas divided by a power of two + if k: # the same p on the deltas divided by a power of two, as the interval searches them d = [math.ldexp(x, -k) for x in d] - eps = _sum_slack(d, EPS * math.ldexp(unit, -k)) - observed = abs(sum(d)) - eps + ints, cap = _as_integers(d, EPS * math.ldexp(unit, -k)) + observed = abs(_int_sum(ints)) - min(cap, (len(ints) * _int_sum(map(abs, ints))) >> 52) # the slack, as _sum_slack if len(d) <= EXACT_MAX: - hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed - for signs in itertools.product((1, -1), repeat=len(d))) - p = hits / 2 ** len(d) + sums = [0] + for x in ints: # every sign pattern's sum, exact: a small delta is not lost against a large one + sums = [s + x for s in sums] + [s - x for s in sums] + p = _int_sum(abs(s) >= observed for s in sums) / 2 ** len(d) else: rng = random.Random(seed) - hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in d)) >= observed for _ in range(n_resamples)) + hits = _int_sum(abs(_int_sum(x if rng.random() < 0.5 else -x for x in ints)) >= observed + for _ in range(n_resamples)) p = (hits + 1) / (n_resamples + 1) return p if exact else round(p, 6) +def _as_integers(values: Sequence[float], cap: float) -> tuple[list[int], int]: + """``values`` and ``cap`` as integer multiples of one power of two, so that sums of them are exact; ``cap`` rounded down.""" + ratios = [v.as_integer_ratio() for v in values] + denominator = max(den for _, den in ratios) + return [num * (denominator // den) for num, den in ratios], math.floor(Fraction(cap) * denominator) + + @functools.cache def _patterns(k: int) -> np.ndarray: """Every sign pattern of k deltas, one per row: ``2**k x k``, as floats for the product.""" @@ -173,7 +187,7 @@ class ShiftTest: """The sign-flip p of ``deltas - theta`` over every delta at any ``theta``, unrounded; at 0 it is :func:`sign_flip_p` of ``deltas``, Monte Carlo draws included, up to rounding. From deltas of about 1e6 on, a sign pattern whose sum ties the observed one in exact arithmetic (magnitudes that repeat, such as 1e9 + 0.1) - can round to one side of it in the product and to the other in ``sum()``, so ``p(0)`` can differ there: the + can round to one side of it in the product and to the other in the exact sums of :func:`sign_flip_p`, so ``p(0)`` can differ there: the row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`).""" def __init__(self, deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 0fa4f2c86..2f67e8616 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -124,15 +124,14 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None -@pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)], - ids=["compensated", "left_to_right"]) -def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed, p, lo): - # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625; the - # printed p itself is sum()'s, 0.0625 compensated (3.12 on) and 0.047 left to right (3.11): 0 follows either +@pytest.mark.parametrize("summed", [math.fsum, _left_to_right], ids=["compensated", "left_to_right"]) +def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed): + # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625. The printed + # p adds the deltas as integers, 0.0625 whatever sum() does (3.12 on compensates, 3.11 adds left to right): 0 follows it monkeypatch.setattr(sign_flip, "sum", summed, raising=False) a, b, c = 1e9 + 1 / 3, 1e9 + 0.1, 2e9 + 0.2 row = sign_flip_summary(_pairs([a, b, c, a, -b, a, b])) - assert row["p"] == p and row["delta"]["lo"] == lo and row["interval_p_disagree"] is False + assert row["p"] == 0.0625 and row["delta"]["lo"] == 0.0 and row["interval_p_disagree"] is False def test_the_interval_excludes_zero_exactly_when_p_is_below_005(): diff --git a/dokimi/tests/test_sign_flip_float_range.py b/dokimi/tests/test_sign_flip_float_range.py index 2350fad80..5d900b67b 100644 --- a/dokimi/tests/test_sign_flip_float_range.py +++ b/dokimi/tests/test_sign_flip_float_range.py @@ -70,15 +70,18 @@ def test_an_end_at_the_float_maximum_is_the_maximum(row): assert all(end is None or math.isfinite(end) for end in ends) -@pytest.mark.parametrize("row", [[1.7e308] * 3 + [1.5e308] * 3, [BIG, 1.0, 1.0, 1.0, 1.0, 1.0], - [math.ldexp(x, SHIFT) for x in MC_BASE]], ids=["six", "max and ones", "monte-carlo"]) -def test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled(row): +@pytest.mark.parametrize(("row", "ends"), [([1.7e308] * 3 + [1.5e308] * 3, ("lo", "hi")), + # Test-narrowed: its lo end is the float product's noise at either scale, not a scaled value + ([BIG, 1.0, 1.0, 1.0, 1.0, 1.0], ("hi",)), + ([math.ldexp(x, SHIFT) for x in MC_BASE], ("lo", "hi"))], + ids=["six", "max and ones", "monte-carlo"]) +def test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled(row, ends): small = [math.ldexp(x, -4) for x in row] with warnings.catch_warnings(): warnings.simplefilter("error") big_row, small_row = sign_flip_row(row, exact=False), sign_flip_row(small, exact=False) assert big_row["p"] == small_row["p"] - for end in ("lo", "hi"): + for end in ends: try: want = None if small_row[end] is None else math.ldexp(small_row[end], 4) except OverflowError: @@ -86,3 +89,20 @@ def test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled(ro assert (big_row[end] is None) == (want is None) if want is not None: assert big_row[end] == pytest.approx(want, rel=1e-12) + + +@pytest.mark.parametrize("m", [1e15, 1e16, 1e17, 1e20, 1e300]) +def test_a_delta_dwarfing_the_others_does_not_hide_them_in_the_exact_p(m): + # benchmarking-and-comparison.md: p is the share of sign patterns whose |sum| reaches the observed one. The five + # deltas of 1 share a sign and none ties, so only the all-same-sign pattern and its mirror reach it: 2 / 2**6 (#816) + assert sign_flip_p([m, 1, 1, 1, 1, 1], exact=True) == 2 / 2 ** 6 + + +def test_a_delta_dwarfing_the_others_does_not_hide_them_in_the_monte_carlo_p(): + # 18 changed deltas, above EXACT_MAX: the same draws as for 1e15, and p at its floor 1 / (N_RESAMPLES + 1) (#816) + assert sign_flip_p([1e17] + [1.0] * 17, exact=True) == sign_flip_p([1e15] + [1.0] * 17, exact=True) == 1 / 20_001 + + +def test_huge_deltas_that_cancel_leave_the_p_of_the_small_ones_to_the_scale_where_float_sums_are_exact(): + # +-1e17 cancel in the observed sum 6, as +-1e5 do; 1e5 + 1 is still a float sum without a rounding error (#816) + assert sign_flip_p([1e17, -1e17] + [1] * 6, exact=True) == sign_flip_p([1e5, -1e5] + [1] * 6, exact=True) == 33 / 64 From a0ba0341ef940c198bd9c4970e0f7d76652017c5 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 10:46:50 +0000 Subject: [PATCH 2/3] fix(eval): paired_test runs on finite deltas whose sum leaves the float range (#832) The issue's input, stats.paired_test([1e308 * g / 9 for g in [2, 4, 3, 5, 6, 9, 7, 8]], [0.0] * 8), is not finite: 1e308 * 2 is inf before the division, so all eight deltas are inf and the inf mean with no p is what the module gives a non-finite delta (test_paired_test_keeps_the_meaning_of_a_non_finite_input). With the finite deltas the issue means, 1e308 * (g / 9), main raised OverflowError from check_sum_range in _describe_deltas, because their magnitudes sum to 4.9e308, though the mean (6.1e307), the sd (2.7e307) and the sign-flip p all fit: _mean and _sample_sd already scale or average exactly, and sign_flip_p and the interval search scale (and with the previous commit add exactly). _describe_deltas now refuses only when the sd of finite deltas is itself past the float range, with the same OverflowError as before, since an inf delta_std next to a finite mean would be the only other result. arm_significance and interaction_test go through _describe_deltas and follow. The model compare section keeps its refusal of a sum past the range, now checked where it calls the test, so its behaviour and message are unchanged. test_paired_test_raises_when_the_deltas_leave_the_float_range pinned the refusal for [9e307, -9e307] * 3; that row has an sd of 9.9e307 and now runs, so the test keeps the 1.7e308 row, whose sd leaves the range (Test-narrowed: 9e307 is no longer past the range). model_compare.py loses a blank line to stay within its growth baseline. Signed-off-by: Anton Volkov --- .../20261117_100100_paired-test-sum-past-range.md | 3 +++ dokimi/docs/benchmarking-and-comparison.md | 3 ++- dokimi/dokimi/eval/stats.py | 5 ++--- dokimi/dokimi/report/model_compare.py | 2 +- dokimi/tests/test_stats.py | 10 ++++++---- dokimi/tests/test_stats_float_range.py | 14 ++++++++++++++ 6 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md diff --git a/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md b/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md new file mode 100644 index 000000000..644627223 --- /dev/null +++ b/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md @@ -0,0 +1,3 @@ +### Fixed + +- **`paired_test` of finite deltas whose sum leaves the float range** — a row of deltas such as `[1e308 * (g / 9) for g in [2, 4, 3, 5, 6, 9, 7, 8]]` (each finite, their magnitudes summing to 4.9e308) was refused with an `OverflowError` for the sum of the magnitudes, though its mean (6.1e307), sd and sign-flip p all fit. `paired_test`, `arm_significance` and `interaction_test` now give the mean, sd, p and interval of such a row, and refuse a row only when its sd is beyond the float range (`[1.7e308, -1.7e308] * 3`), as before. `model compare` sections keep refusing a sum past the range, naming the run. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 542c83f31..658c6c1e3 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -446,7 +446,8 @@ heading, when no arm has a row); the significance of the other arms stays when t family test itself succeeds. A mean of finite scores or of finite paired differences whose sum would pass the float range is computed exactly (`statistics.mean`), so the per-arm `avg_aggregate` and `delta_vs_baseline` are refused only when a single -difference leaves the range. The console summary of `arms run` prints +difference leaves the range. A significance row computes the mean, the sd and the +sign-flip p of such differences too, and is refused only when its sd leaves the range. The console summary of `arms run` prints `(delta not computed: )` for such an arm. Everything above assumes the judge's continuous 0–100 score. The diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index d0a58becb..e90a72a42 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -613,9 +613,8 @@ def _describe_deltas( d_sd = _sample_sd(d) cohens_d = _cohens_d(d_mean, d_sd) if all(math.isfinite(x) for x in d): - # The sign-flip sums would overflow and read as a measured zero effect. - # A finite sum of magnitudes bounds the spread (sd**2 <= sum(x**2) <= that sum squared), so it is finite too. - check_sum_range(d) + if not math.isfinite(d_sd): # the spread of finite deltas past the float range: refused, as before + check_sum_range(d) row = sign_flip_row(d, seed=seed, exact=True, magnitudes=magnitudes) p, lo, hi, shown_lo, shown_hi = row["p"], row["lo"], row["hi"], row["ci_lo"], row["ci_hi"] else: # a NaN delta is no evidence either way: no p, no verdict, open ends diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index 8de772bb9..f6a763b32 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -951,12 +951,12 @@ def _adjusted_significance( if paired: with_paired, without_paired = zip(*paired, strict=True) try: + finite_delta.check_sum_range([w - wo for w, wo in paired]) # a section refuses a sum past the range sigs[run_id] = significance_test(list(with_paired), list(without_paired), rounded=False) except OverflowError as exc: # finite deltas whose sum or spread leaves the float range raise InputError(f"run '{run_id}': {exc}") from exc else: sigs[run_id] = None - # The arms' own correction, so the two verdicts cannot drift; a run with no p, or under the floor rule, stays out. family = [sig for rid in run_ids if (sig := sigs[rid])] arm_stats._apply_correction(family, correction=correction, label="runs") diff --git a/dokimi/tests/test_stats.py b/dokimi/tests/test_stats.py index 9c88b88f2..c9a7e8a75 100644 --- a/dokimi/tests/test_stats.py +++ b/dokimi/tests/test_stats.py @@ -780,11 +780,13 @@ def test_mean_ci_raises_when_the_interval_leaves_the_float_range(): def test_paired_test_raises_when_the_deltas_leave_the_float_range(): - """Sign-flip sums that overflow would read as a measured zero effect: an error instead.""" + """A spread past the float range would be an inf delta_std next to a finite mean: an error instead. + + Test-narrowed: 9e307 no longer raises. Its sd is 9.9e307, so the mean, sd and sign-flip sums are all computed (#832). + """ pytest.importorskip("scipy") - for big in (9e307, 1.7e308): - with pytest.raises(OverflowError): - stats.paired_test([big, -big] * 3, [0.0] * 6, rounded=False) + with pytest.raises(OverflowError): + stats.paired_test([1.7e308, -1.7e308] * 3, [0.0] * 6, rounded=False) res = stats.paired_test([1e300, -1e300] * 3, [0.0] * 6, rounded=False) assert res is not None and math.isfinite(res["delta_std"]) diff --git a/dokimi/tests/test_stats_float_range.py b/dokimi/tests/test_stats_float_range.py index 9a67f0308..22ff097c5 100644 --- a/dokimi/tests/test_stats_float_range.py +++ b/dokimi/tests/test_stats_float_range.py @@ -63,6 +63,20 @@ def test_paired_test_of_a_delta_just_inside_the_range_still_runs(): assert stats._deltas([BIG], [-9e306]) == [BIG + 9e306] +def test_paired_test_of_finite_deltas_whose_sum_leaves_the_range_runs_on_them(): + # benchmarking-and-comparison.md: a row is refused only where its sd leaves the float range, not its sum (#832). + # 1e308 * g / 9 would be inf already at g = 2, so the deltas are scaled first; their sum is 4.9e308 + deltas = [1e308 * (g / 9) for g in [2, 4, 3, 5, 6, 9, 7, 8]] + assert all(math.isfinite(d) for d in deltas) and not math.isfinite(sum(deltas)) + result = stats.paired_test(deltas, [0.0] * 8) + assert result["delta_mean"] == pytest.approx(6.111111111111111e307, rel=1e-12) + assert result["exact"]["delta_mean"] == pytest.approx(6.111111111111111e307, rel=1e-12) + assert result["delta_std"] == pytest.approx(2.721655269759087e307, rel=1e-12) + assert result["cohens_d"] == 2.245 + assert (result["p"], result["n_changed"], result["significant"]) == (0.0078, 8, True) + assert result["ci_lo"] <= result["delta_mean"] <= result["ci_hi"] and math.isfinite(result["ci_hi"]) + + def test_paired_test_refuses_integers_whose_exact_difference_leaves_the_range(): with pytest.raises(OverflowError, match="float range"): stats.paired_test([10 ** 308] * 4, [-(10 ** 308)] * 4) From d465cf16d75502400bf4024e99d518fb5977f81c Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 14:11:00 +0000 Subject: [PATCH 3/3] fix(eval): refuse an inf additive expectation; exact sums in the ends Settles the review of #867. interaction_test now raises OverflowError when the two arm deltas are finite but their sum leaves the float range, instead of returning additive_expectation inf next to a computed p. ShiftTest.p sums up to 16 shifted deltas that span more than 40 binary orders of magnitude exactly (the integer enumeration of sign_flip_p, now shared as _enumerated_p), so the lower end of [1e17, 1, 1, 1, 1, 1] is 1.0 and no longer the 0.0001 a float search read as measured. Rows of more than 16 such deltas keep the float search (#865). sign_flip_p uses the builtin sum on integers directly: the _int_sum alias and the tests' monkeypatches of sign_flip.sum, which no longer change any float sum, are dropped, with the test that compared the ends under three sums. _describe_deltas names the sd in its refusal, and the comment in model_compare says why the sum check stays. Test-narrowed: the monkeypatched sum variants of test_an_end_is_bounded_at_deltas_too_large_to_move_by_one and test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart, and test_the_ends_are_the_same_whether_python_sums_compensated_or_left_to_right, no longer have a float sum to swap: the p and the ends add integers, exact on every Python. Signed-off-by: Anton Volkov --- .../20261117_100000_sign-flip-p-exact-sums.md | 2 +- ...61117_100100_paired-test-sum-past-range.md | 2 +- dokimi/docs/benchmarking-and-comparison.md | 2 +- dokimi/dokimi/eval/interactions.py | 2 + dokimi/dokimi/eval/sign_flip.py | 38 +++++++++++++------ dokimi/dokimi/eval/stats.py | 4 +- dokimi/dokimi/report/model_compare.py | 4 +- dokimi/tests/test_interactions_float_range.py | 9 +++++ dokimi/tests/test_repo_compare_interval.py | 33 ++-------------- dokimi/tests/test_sign_flip_float_range.py | 19 ++++++---- dokimi/tests/test_stats.py | 2 +- 11 files changed, 61 insertions(+), 56 deletions(-) diff --git a/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md b/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md index 7dec7077a..ad8b0794f 100644 --- a/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md +++ b/dokimi/changelog.d/20261117_100000_sign-flip-p-exact-sums.md @@ -1,3 +1,3 @@ ### Fixed -- **Sign-flip p of a delta that dwarfs the others** — `sign_flip_p` added the signed deltas as floats, so from a delta of about 1e16 on the small ones were absorbed by the large one and all 64 sign patterns of `[1e17, 1, 1, 1, 1, 1]` summed to the same 1e17: its p was 1.0 and is now 0.03125, as for `1e15`. The sums are now exact (each delta an integer multiple of one power of two, so the p no longer depends on the order or the Python that adds them), for the exact enumeration and the Monte Carlo draws alike; the rounding slack that lets a pattern tie the observed sum (0.1 + 0.2 against 0.3) is unchanged. Rows of ordinary scale give the same p; the `1e9 + 1/3` row, whose sums only tie in exact arithmetic, is 0.0625 on every Python (0.046875 before on Python 3.11, which sums left to right). The interval's ends are still the float search of `ShiftTest`. +- **Sign-flip p of a delta that dwarfs the others** — `sign_flip_p` added the signed deltas as floats, so from a delta of about 1e16 on the small ones were absorbed by the large one and all 64 sign patterns of `[1e17, 1, 1, 1, 1, 1]` summed to the same 1e17: its p was 1.0 and is now 0.03125, as for `1e15`. The sums are now exact (each delta an integer multiple of one power of two, so the p no longer depends on the order or the Python that adds them), for the exact enumeration and the Monte Carlo draws alike; the rounding slack that lets a pattern tie the observed sum (0.1 + 0.2 against 0.3) is unchanged. Rows of ordinary scale give the same p; the `1e9 + 1/3` row, whose sums only tie in exact arithmetic, is 0.0625 on every Python (0.046875 before on Python 3.11, which sums left to right). The interval search of a row of up to 16 changed deltas that span more than 40 binary orders of magnitude (`[1e17, 1, 1, 1, 1, 1]`) tests each shift with the same exact sums, so its lower end is 1.0, not the 0.0001 the float sums gave; a row of more than 16 such deltas keeps the float search. diff --git a/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md b/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md index 644627223..ba2bb4e5c 100644 --- a/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md +++ b/dokimi/changelog.d/20261117_100100_paired-test-sum-past-range.md @@ -1,3 +1,3 @@ ### Fixed -- **`paired_test` of finite deltas whose sum leaves the float range** — a row of deltas such as `[1e308 * (g / 9) for g in [2, 4, 3, 5, 6, 9, 7, 8]]` (each finite, their magnitudes summing to 4.9e308) was refused with an `OverflowError` for the sum of the magnitudes, though its mean (6.1e307), sd and sign-flip p all fit. `paired_test`, `arm_significance` and `interaction_test` now give the mean, sd, p and interval of such a row, and refuse a row only when its sd is beyond the float range (`[1.7e308, -1.7e308] * 3`), as before. `model compare` sections keep refusing a sum past the range, naming the run. +- **`paired_test` of finite deltas whose sum leaves the float range** — a row of deltas such as `[1e308 * (g / 9) for g in [2, 4, 3, 5, 6, 9, 7, 8]]` (each finite, their magnitudes summing to 4.9e308) was refused with an `OverflowError` for the sum of the magnitudes, though its mean (6.1e307), sd and sign-flip p all fit. `paired_test`, `arm_significance` and `interaction_test` now give the mean, sd, p and interval of such a row, and refuse a row only when its sd is beyond the float range (`[1.7e308, -1.7e308] * 3`), as before, naming the sd. `interaction_test` refuses a row whose two arm deltas add past the float range (its additive expectation would be `inf`). `model compare` sections keep refusing a sum past the range, naming the run. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 658c6c1e3..ab458a6d7 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -447,7 +447,7 @@ family test itself succeeds. A mean of finite scores or of finite paired differe whose sum would pass the float range is computed exactly (`statistics.mean`), so the per-arm `avg_aggregate` and `delta_vs_baseline` are refused only when a single difference leaves the range. A significance row computes the mean, the sd and the -sign-flip p of such differences too, and is refused only when its sd leaves the range. The console summary of `arms run` prints +sign-flip p of such differences too, and is refused only when its sd leaves the range; an interaction row is also refused when the sum of its two arm deltas (its additive expectation) does. The console summary of `arms run` prints `(delta not computed: )` for such an arm. Everything above assumes the judge's continuous 0–100 score. The diff --git a/dokimi/dokimi/eval/interactions.py b/dokimi/dokimi/eval/interactions.py index 629a8a4ef..298eebb6d 100644 --- a/dokimi/dokimi/eval/interactions.py +++ b/dokimi/dokimi/eval/interactions.py @@ -110,6 +110,8 @@ def _interaction_contrast( res = _describe_deltas(contrasts, confidence=confidence, seed=seed) delta_a = _arm_delta(diffs_a, confidence, seed) delta_b = _arm_delta(diffs_b, confidence, seed) + if math.isfinite(delta_a) and math.isfinite(delta_b) and not math.isfinite(delta_a + delta_b): # the row's additive expectation would print as inf next to a computed p + raise OverflowError("deltas too large: the sum of the two arm deltas leaves the float range") res.update( { "arm_a": arm_a, diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index b001aa007..859158aed 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -8,7 +8,7 @@ :func:`min_sign_flip_p` is the floor it cannot go below over k changed deltas, ``2 / 2**k`` exact. ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as -:func:`sign_flip_p` computes it on them (that one adds the deltas as exact integers; this one, below, in floats): a shifted delta within the tie width of its pair (:func:`tie_widths`, kept +:func:`sign_flip_p` computes it on them (that one adds the deltas as exact integers; this one, below, in floats, so its p can differ at the ends of a row whose deltas floats cannot resolve): a shifted delta within the tie width of its pair (:func:`tie_widths`, kept from the original deltas) of 0 is a tie, so a delta tied at 0 counts again once ``theta`` moves off 0. Exact, a sign pattern counts when its sum falls short of the observed one by at most the rounding error of adding the deltas summed (:func:`_sum_slack`; :data:`EPS` at most, and no more than the largest tie width among them), @@ -66,8 +66,6 @@ #: A vector whose values are all below this is searched on a copy multiplied by a power of two (exact), so the #: tie widths, the bisection step and the sums stay clear of the subnormal range. _TINY = 2.0 ** -664 -#: ``sum`` of integers (exact on every Python), kept apart from the module's ``sum`` that tests swap for float sums. -_int_sum = sum def pair_magnitudes(treatment: Sequence[float], baseline: Sequence[float]) -> list[float]: @@ -135,21 +133,33 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL k = _scale_exponent(d, unit) if k: # the same p on the deltas divided by a power of two, as the interval searches them d = [math.ldexp(x, -k) for x in d] - ints, cap = _as_integers(d, EPS * math.ldexp(unit, -k)) - observed = abs(_int_sum(ints)) - min(cap, (len(ints) * _int_sum(map(abs, ints))) >> 52) # the slack, as _sum_slack + cap_eps = EPS * math.ldexp(unit, -k) if len(d) <= EXACT_MAX: - sums = [0] - for x in ints: # every sign pattern's sum, exact: a small delta is not lost against a large one - sums = [s + x for s in sums] + [s - x for s in sums] - p = _int_sum(abs(s) >= observed for s in sums) / 2 ** len(d) + p = _enumerated_p(d, cap_eps) else: + ints, cap = _as_integers(d, cap_eps) + observed = _observed(ints, cap) rng = random.Random(seed) - hits = _int_sum(abs(_int_sum(x if rng.random() < 0.5 else -x for x in ints)) >= observed - for _ in range(n_resamples)) + hits = sum(abs(sum(x if rng.random() < 0.5 else -x for x in ints)) >= observed for _ in range(n_resamples)) p = (hits + 1) / (n_resamples + 1) return p if exact else round(p, 6) +def _observed(ints: list[int], cap: int) -> int: + """The observed ``|sum|`` less the rounding slack of the deltas (:func:`_sum_slack`, in the units of ``ints``).""" + return abs(sum(ints)) - min(cap, (len(ints) * sum(map(abs, ints))) >> 52) + + +def _enumerated_p(d: list[float], cap_eps: float) -> float: + """The exact p over every sign pattern of ``d``, whose sums are integers: a small delta is not lost against a large one.""" + ints, cap = _as_integers(d, cap_eps) + observed = _observed(ints, cap) + sums = [0] + for x in ints: + sums = [s + x for s in sums] + [s - x for s in sums] + return sum(abs(s) >= observed for s in sums) / 2 ** len(d) + + def _as_integers(values: Sequence[float], cap: float) -> tuple[list[int], int]: """``values`` and ``cap`` as integer multiples of one power of two, so that sums of them are exact; ``cap`` rounded down.""" ratios = [v.as_integer_ratio() for v in values] @@ -188,7 +198,8 @@ class ShiftTest: :func:`sign_flip_p` of ``deltas``, Monte Carlo draws included, up to rounding. From deltas of about 1e6 on, a sign pattern whose sum ties the observed one in exact arithmetic (magnitudes that repeat, such as 1e9 + 0.1) can round to one side of it in the product and to the other in the exact sums of :func:`sign_flip_p`, so ``p(0)`` can differ there: the - row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`).""" + row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`). Up to :data:`EXACT_MAX` shifted deltas that span more than + 40 binary orders of magnitude are summed exactly too, as floats lose the small ones there; more of them are not (#865).""" def __init__(self, deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, eps: float | Sequence[float] = EPS) -> None: @@ -211,6 +222,9 @@ def p(self, theta: float) -> float | None: moved = x[x != 0.0] if not moved.size: return None + spread = math.frexp(float(np.abs(moved).max()))[1] - math.frexp(float(np.abs(moved).min()))[1] # in binary orders + if moved.size <= EXACT_MAX and spread > 40: + return _enumerated_p(moved.tolist(), float(self.widths[~tied].max())) # floats lose the small deltas eps = _sum_slack(moved.tolist(), float(self.widths[~tied].max())) observed = abs(math.fsum(moved.tolist())) # exactly rounded: sum() rounds differently before 3.12 if moved.size <= EXACT_MAX: diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index e90a72a42..67d84bc04 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -65,7 +65,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from dokimi.eval.arm_summary import mean_in_range -from dokimi.eval.finite_delta import check_sum_range, checked_delta +from dokimi.eval.finite_delta import checked_delta from dokimi.eval.sign_flip import ( ALPHA, METHOD, @@ -614,7 +614,7 @@ def _describe_deltas( cohens_d = _cohens_d(d_mean, d_sd) if all(math.isfinite(x) for x in d): if not math.isfinite(d_sd): # the spread of finite deltas past the float range: refused, as before - check_sum_range(d) + raise OverflowError("the sd of the deltas leaves the float range") row = sign_flip_row(d, seed=seed, exact=True, magnitudes=magnitudes) p, lo, hi, shown_lo, shown_hi = row["p"], row["lo"], row["hi"], row["ci_lo"], row["ci_hi"] else: # a NaN delta is no evidence either way: no p, no verdict, open ends diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index f6a763b32..eda2365c4 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -951,9 +951,9 @@ def _adjusted_significance( if paired: with_paired, without_paired = zip(*paired, strict=True) try: - finite_delta.check_sum_range([w - wo for w, wo in paired]) # a section refuses a sum past the range + finite_delta.check_sum_range([w - wo for w, wo in paired]) # the section's averages are float sums: +inf sigs[run_id] = significance_test(list(with_paired), list(without_paired), rounded=False) - except OverflowError as exc: # finite deltas whose sum or spread leaves the float range + except OverflowError as exc: # finite deltas whose sum leaves the float range raise InputError(f"run '{run_id}': {exc}") from exc else: sigs[run_id] = None diff --git a/dokimi/tests/test_interactions_float_range.py b/dokimi/tests/test_interactions_float_range.py index 15ed1fb21..0bb37f658 100644 --- a/dokimi/tests/test_interactions_float_range.py +++ b/dokimi/tests/test_interactions_float_range.py @@ -84,6 +84,15 @@ def scores(ab, a, b, base): assert res["p"] is None +def test_two_arm_deltas_whose_sum_leaves_the_float_range_raise_instead_of_an_inf_additive_expectation(): + # every delta and the contrast fit (the sd is finite), but 1e308 + 1e308 is the row's additive expectation + evaluations = [_ev(f"q{i}", 1.5e308 + i * 1e306, 1e308, 1e308, 0.0) for i in range(8)] + with pytest.raises(OverflowError, match="the sum of the two arm deltas leaves the float range"): + _test(evaluations) + with pytest.raises(OverflowError, match="the sum of the two arm deltas leaves the float range"): + interaction_significance(evaluations, ARMS, "baseline") + + def test_a_near_range_contrast_that_stays_finite_returns_finite_numbers(): evaluations = [_ev("q1", BIG, BIG, BIG, BIG), _ev("q2", -BIG, -BIG, -BIG, -BIG), _ev("q3", 1.0, 0.0, 0.0, 0.0)] res = _test(evaluations) diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 2f67e8616..d563c0ada 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -5,10 +5,7 @@ from __future__ import annotations -import functools import json -import math -import operator import random import pytest @@ -59,34 +56,14 @@ def test_an_end_is_open_at_five_or_fewer_tasks_and_bounded_from_six(): assert sign_flip_interval([5.0] * 6, sign_flip_p([5.0] * 6)) == (5.0, 5.0) -def _left_to_right(xs, start=0.0): - """``sum`` of floats as Python 3.11 adds them: in order, uncompensated (3.12 made ``sum`` compensated).""" - return functools.reduce(operator.add, xs, start) - - -@pytest.mark.parametrize("summed", [sum, _left_to_right], ids=["builtin", "left_to_right"]) -def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(monkeypatch, summed): +def test_an_end_is_bounded_at_deltas_too_large_to_move_by_one(): # 1e16 + 1.0 == 1e16: a far bound 1 past the extreme tied every shifted delta and left the end open - monkeypatch.setattr(sign_flip, "sum", summed, raising=False) # 3.11's sum() put 2e16's end at 2e16 + 4 for v in (1e16, 1e150): # finite measures go up to SCORE_BOUND, 1e150 assert sign_flip_interval([v] * 6, sign_flip_p([v] * 6)) == (v, v) assert sign_flip_interval([-v] * 6, sign_flip_p([-v] * 6)) == (-v, -v) assert sign_flip_interval([1e16] * 5 + [2e16], sign_flip_p([1e16] * 5 + [2e16])) == (1e16, 2e16) -def test_the_ends_are_the_same_whether_python_sums_compensated_or_left_to_right(monkeypatch): - # sum() of floats is compensated from Python 3.12 on and not before: a row's ends must not depend on the Python - rng = random.Random(11) - rows = [[scale * rng.uniform(-0.6, 1.0) for _ in range(rng.randint(6, 24))] - for scale in (1e6, 1e13, 1e16, 1e100) for _ in range(10)] - ps = [sign_flip_p(d) for d in rows] - ends = {} - for name, summed in (("builtin", sum), ("compensated", math.fsum), ("left_to_right", _left_to_right)): - monkeypatch.setattr(sign_flip, "sum", summed, raising=False) - ends[name] = [sign_flip_interval(d, p, rounded=False) for d, p in zip(rows, ps, strict=True)] - assert ends["builtin"] == ends["compensated"] == ends["left_to_right"] - - def test_large_deltas_keep_the_ends_and_the_p_of_the_shifted_deltas(): # at 1e7 the product's rounding passed EPS: the observed pattern stopped counting and both ends moved d = [0.62, 0.18, 0.94, -0.37, 0.71, 0.29, 0.83, 0.56, -0.05] @@ -124,11 +101,9 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None -@pytest.mark.parametrize("summed", [math.fsum, _left_to_right], ids=["compensated", "left_to_right"]) -def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(monkeypatch, summed): - # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625. The printed - # p adds the deltas as integers, 0.0625 whatever sum() does (3.12 on compensates, 3.11 adds left to right): 0 follows it - monkeypatch.setattr(sign_flip, "sum", summed, raising=False) +def test_the_row_p_places_zero_where_large_deltas_round_p_at_zero_apart(): + # at 1e9 patterns tying the observed sum round apart: p(0) can read 0.047 against the printed 0.0625, which adds + # the deltas as integers (exact on every Python): 0 follows it a, b, c = 1e9 + 1 / 3, 1e9 + 0.1, 2e9 + 0.2 row = sign_flip_summary(_pairs([a, b, c, a, -b, a, b])) assert row["p"] == 0.0625 and row["delta"]["lo"] == 0.0 and row["interval_p_disagree"] is False diff --git a/dokimi/tests/test_sign_flip_float_range.py b/dokimi/tests/test_sign_flip_float_range.py index 5d900b67b..160ac6c6a 100644 --- a/dokimi/tests/test_sign_flip_float_range.py +++ b/dokimi/tests/test_sign_flip_float_range.py @@ -70,18 +70,15 @@ def test_an_end_at_the_float_maximum_is_the_maximum(row): assert all(end is None or math.isfinite(end) for end in ends) -@pytest.mark.parametrize(("row", "ends"), [([1.7e308] * 3 + [1.5e308] * 3, ("lo", "hi")), - # Test-narrowed: its lo end is the float product's noise at either scale, not a scaled value - ([BIG, 1.0, 1.0, 1.0, 1.0, 1.0], ("hi",)), - ([math.ldexp(x, SHIFT) for x in MC_BASE], ("lo", "hi"))], - ids=["six", "max and ones", "monte-carlo"]) -def test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled(row, ends): +@pytest.mark.parametrize("row", [[1.7e308] * 3 + [1.5e308] * 3, [BIG, 1.0, 1.0, 1.0, 1.0, 1.0], + [math.ldexp(x, SHIFT) for x in MC_BASE]], ids=["six", "max and ones", "monte-carlo"]) +def test_the_p_and_ends_of_a_row_near_the_maximum_are_those_of_the_row_scaled(row): small = [math.ldexp(x, -4) for x in row] with warnings.catch_warnings(): warnings.simplefilter("error") big_row, small_row = sign_flip_row(row, exact=False), sign_flip_row(small, exact=False) assert big_row["p"] == small_row["p"] - for end in ends: + for end in ("lo", "hi"): try: want = None if small_row[end] is None else math.ldexp(small_row[end], 4) except OverflowError: @@ -98,6 +95,14 @@ def test_a_delta_dwarfing_the_others_does_not_hide_them_in_the_exact_p(m): assert sign_flip_p([m, 1, 1, 1, 1, 1], exact=True) == 2 / 2 ** 6 +@pytest.mark.parametrize("m", [1e16, 1e17, 1e300]) +def test_the_lower_end_of_a_row_with_a_dwarfing_delta_is_where_the_small_deltas_stop_rejecting(m): + # benchmarking-and-comparison.md: the interval is every shift the same test does not reject. The five deltas of 1 + # stay on one side until the shift reaches 1, so the end is 1, not the 0.0001 a float search read as measured (#867) + row = sign_flip_row([m, 1, 1, 1, 1, 1], exact=True) + assert row["p"] == 2 / 2 ** 6 and row["lo"] == pytest.approx(1.0, abs=1e-6) + + def test_a_delta_dwarfing_the_others_does_not_hide_them_in_the_monte_carlo_p(): # 18 changed deltas, above EXACT_MAX: the same draws as for 1e15, and p at its floor 1 / (N_RESAMPLES + 1) (#816) assert sign_flip_p([1e17] + [1.0] * 17, exact=True) == sign_flip_p([1e15] + [1.0] * 17, exact=True) == 1 / 20_001 diff --git a/dokimi/tests/test_stats.py b/dokimi/tests/test_stats.py index c9a7e8a75..95fd7f73a 100644 --- a/dokimi/tests/test_stats.py +++ b/dokimi/tests/test_stats.py @@ -785,7 +785,7 @@ def test_paired_test_raises_when_the_deltas_leave_the_float_range(): Test-narrowed: 9e307 no longer raises. Its sd is 9.9e307, so the mean, sd and sign-flip sums are all computed (#832). """ pytest.importorskip("scipy") - with pytest.raises(OverflowError): + with pytest.raises(OverflowError, match="the sd of the deltas leaves the float range"): stats.paired_test([1.7e308, -1.7e308] * 3, [0.0] * 6, rounded=False) res = stats.paired_test([1e300, -1e300] * 3, [0.0] * 6, rounded=False) assert res is not None and math.isfinite(res["delta_std"])