From 2c4d4cb0229f61802b23793f8a6f65570630ebcc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:38:16 +0000 Subject: [PATCH 1/2] fix(repo): mde of deltas near the float maximum is None, not an overflow warning (#769) sign_flip_summary computed mde as a product of numpy float64 t quantiles and the spread, so a spread near the float maximum emitted RuntimeWarning: overflow encountered in scalar multiply (deltas [float max, 1, 1, 1, 1, 1]), and _round on a numpy float64 emitted overflow encountered in multiply for an mde of 9.3e306 (twelve deltas of plus and minus 1e307). Without -W error the mde was inf or the unrounded value. min_detectable now does its arithmetic on Python floats, so a product past the range is inf with no warning. _round converts to float first and returns None for an inf or nan, since the planning figure has no number to show beyond the float range. Every other caller of _round passes a finite mean, delta or task value, or None, so they are unchanged. The scaled-sd option is not taken: a scaled-back mde would overflow again. The existing test that ignored the overflow warning for the 1e307 case no longer needs the filter and loses it. Signed-off-by: Anton Volkov --- .../20261116_130000_mde-float-range.md | 3 +++ dokimi/dokimi/repos/compare.py | 8 ++++-- dokimi/dokimi/repos/compare_reach.py | 6 +++-- dokimi/tests/test_repo_compare_overflow.py | 27 +++++++++++++++++-- 4 files changed, 38 insertions(+), 6 deletions(-) create mode 100644 dokimi/changelog.d/20261116_130000_mde-float-range.md diff --git a/dokimi/changelog.d/20261116_130000_mde-float-range.md b/dokimi/changelog.d/20261116_130000_mde-float-range.md new file mode 100644 index 000000000..f092fb8c9 --- /dev/null +++ b/dokimi/changelog.d/20261116_130000_mde-float-range.md @@ -0,0 +1,3 @@ +### Fixed + +- **`mde` of deltas near the float maximum** — `sign_flip_summary` (`repo compare`) emitted `RuntimeWarning: overflow encountered in scalar multiply` computing `mde` for a spread near the float maximum (`[sys.float_info.max, 1, 1, 1, 1, 1]`), and `RuntimeWarning: overflow encountered in multiply` rounding it for twelve deltas of ±1e307. `min_detectable` now does its arithmetic on Python floats and `_round` converts to a float first, so neither warns. A planning figure beyond the float range, or any `inf` or `nan` passed to `_round`, is `None` (no figure to show) instead of `inf`. Rows with an `mde` in range are unchanged. diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 1d9a28c47..39bdc7719 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -49,7 +49,7 @@ whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, a planning figure: an approximate, uncorrected t-based estimate of the mean delta a two-sided 5% one-sample t test detects with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the per-task deltas, t at ``n - 1`` degrees of -freedom (in every row; ``None`` for one task or no spread), not the power of the sign-flip test nor of the +freedom (in every row; ``None`` for one task, no spread, or a spread so large the figure leaves the float range), not the power of the sign-flip test nor of the Holm-corrected decision. The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no effect of any size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. ``mde`` is ``None`` too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the @@ -105,6 +105,7 @@ from __future__ import annotations import json +import math import statistics from collections import defaultdict from pathlib import Path @@ -753,7 +754,10 @@ def _mean(values: list[Any]) -> float | None: def _round(value: float | None) -> float | None: - return None if value is None else round(value, 6) + if value is None: + return None + number = float(value) # a numpy float64 would warn on overflow while rounding + return round(number, 6) if math.isfinite(number) else None def _num(value: float | None) -> str: diff --git a/dokimi/dokimi/repos/compare_reach.py b/dokimi/dokimi/repos/compare_reach.py index 50937d25f..facfb547b 100644 --- a/dokimi/dokimi/repos/compare_reach.py +++ b/dokimi/dokimi/repos/compare_reach.py @@ -21,9 +21,11 @@ def min_detectable(sd: float, n: int) -> float: """An approximate, uncorrected t-based estimate of the mean delta a two-sided one-sample t test at :data:`ALPHA` detects with :data:`POWER` over ``n`` tasks of spread ``sd``, a planning figure: not the power of the sign-flip test nor of the Holm-corrected decision. The quantiles are t at ``n - 1`` degrees of freedom, those of the - planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`).""" + planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`). + The arithmetic is on Python floats: a spread whose product leaves the float range gives ``inf``, no warning.""" df = n - 1 - return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) + quantiles = float(sp_stats.t.ppf(1 - ALPHA / 2, df)) + float(sp_stats.t.ppf(POWER, df)) + return quantiles * float(sd) / math.sqrt(n) def holm_family(pvals: list[float | None], changed: list[int]) -> list[float | None]: diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 646756fcf..304fb024b 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -6,11 +6,13 @@ from __future__ import annotations import math +import sys import pytest from dokimi.eval.finite_delta import checked_delta, checked_mean -from dokimi.repos.compare import CompareError, _pair, sign_flip_summary +from dokimi.repos.compare import CompareError, _pair, _round, sign_flip_summary +from dokimi.repos.compare_reach import min_detectable def _side(value: float) -> dict: @@ -71,7 +73,6 @@ def test_sign_flip_summary_refuses_finite_deltas_whose_sum_or_spread_leaves_the_ sign_flip_summary([_task(d) for d in deltas]) -@pytest.mark.filterwarnings("ignore:overflow encountered:RuntimeWarning") # the mde of a spread this large def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy(): row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) # cancels pairwise: sum() before 3.12 does not compensate assert row["delta"]["mean"] == 0.0 @@ -79,6 +80,28 @@ def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy(): assert row["delta"]["lo"] < 0 < row["delta"]["hi"] +@pytest.mark.filterwarnings("error") +def test_mde_is_none_when_the_planning_figure_leaves_the_float_range(): + row = sign_flip_summary([_task(d) for d in [sys.float_info.max, 1.0, 1.0, 1.0, 1.0, 1.0]]) + assert row["mde"] is None + + +@pytest.mark.filterwarnings("error") +def test_mde_of_a_large_spread_is_rounded_without_overflowing(): + row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) + assert math.isfinite(row["mde"]) and row["mde"] == pytest.approx(9.276e306, rel=1e-3) + + +@pytest.mark.filterwarnings("error") +def test_min_detectable_past_the_range_is_inf_without_a_warning(): + assert min_detectable(sys.float_info.max, 3) == math.inf + + +@pytest.mark.parametrize("value", [math.inf, -math.inf, math.nan]) +def test_round_gives_no_figure_for_a_non_finite_value(value): + assert _round(value) is None + + def test_checked_mean_is_the_plain_mean_where_it_is_finite(): assert checked_mean([0.1, 0.2, 0.4]) == sum([0.1, 0.2, 0.4]) / 3 assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == sum([1.0e307, -1.0e307, 1.0e307]) / 3 From 74fdba5ea7b3e28ec29e4f729efd1b6388f40617 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 23:06:48 +0000 Subject: [PATCH 2/2] fix(repo): a large mde prints in scientific notation; min_detectable of no task is nan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settles the review of #822: the inconclusive note printed every digit of an mde near the float range (twelve deltas of ±1e307 gave a 300-digit figure); it now prints an mde of 1e6 or more as `9.276e+306`, and one below 1e6 as before. min_detectable(sd, 0) is nan again, as on main, instead of a ZeroDivisionError from the Python-float arithmetic. The contract names the float-range case among those where mde is null and says what the note does for it, the comment of the silent branch of _inconclusive_note names it, and the changelog fragment carries the author segment of its name. Tests pin the note for a large and a small mde, min_detectable at n=0, and _round of a numpy scalar near the range. Signed-off-by: Anton Volkov --- ...1116_130000_antonwolfy_mde-float-range.md} | 2 +- dokimi/docs/repo-evaluation-contract.md | 4 ++-- dokimi/dokimi/repos/compare.py | 5 ++-- dokimi/dokimi/repos/compare_reach.py | 4 +++- dokimi/tests/test_repo_compare_overflow.py | 23 ++++++++++++++++++- 5 files changed, 31 insertions(+), 7 deletions(-) rename dokimi/changelog.d/{20261116_130000_mde-float-range.md => 20261116_130000_antonwolfy_mde-float-range.md} (69%) diff --git a/dokimi/changelog.d/20261116_130000_mde-float-range.md b/dokimi/changelog.d/20261116_130000_antonwolfy_mde-float-range.md similarity index 69% rename from dokimi/changelog.d/20261116_130000_mde-float-range.md rename to dokimi/changelog.d/20261116_130000_antonwolfy_mde-float-range.md index f092fb8c9..b3a0eb86c 100644 --- a/dokimi/changelog.d/20261116_130000_mde-float-range.md +++ b/dokimi/changelog.d/20261116_130000_antonwolfy_mde-float-range.md @@ -1,3 +1,3 @@ ### Fixed -- **`mde` of deltas near the float maximum** — `sign_flip_summary` (`repo compare`) emitted `RuntimeWarning: overflow encountered in scalar multiply` computing `mde` for a spread near the float maximum (`[sys.float_info.max, 1, 1, 1, 1, 1]`), and `RuntimeWarning: overflow encountered in multiply` rounding it for twelve deltas of ±1e307. `min_detectable` now does its arithmetic on Python floats and `_round` converts to a float first, so neither warns. A planning figure beyond the float range, or any `inf` or `nan` passed to `_round`, is `None` (no figure to show) instead of `inf`. Rows with an `mde` in range are unchanged. +- **`mde` of deltas near the float maximum** — `sign_flip_summary` (`repo compare`) emitted `RuntimeWarning: overflow encountered in scalar multiply` computing `mde` for a spread near the float maximum (`[sys.float_info.max, 1, 1, 1, 1, 1]`), and `RuntimeWarning: overflow encountered in multiply` rounding it for twelve deltas of ±1e307. `min_detectable` now does its arithmetic on Python floats and `_round` converts to a float first, so neither warns. A planning figure beyond the float range, or any `inf` or `nan` passed to `_round`, is `None` (no figure to show) instead of `inf`. `min_detectable(sd, 0)` is still `nan`. The inconclusive note prints an `mde` of 1e6 or more in scientific notation (`about ±9.276e+306`) instead of every digit. The `mde` in the data and the note's text for an `mde` below 1e6 are unchanged. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index db26de3c4..a6cb51827 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -2504,8 +2504,8 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the per-task deltas, of the mean delta a one-sample t test at the uncorrected two-sided 5% level detects with 80% power at that spread, the t quantiles at `n-1` degrees of - freedom (every row has `mde` under `data` in `--json`; `null` for one task or deltas with - no spread, and then no note). It is not the power of the sign-flip test nor + freedom (every row has `mde` under `data` in `--json`; `null` for one task, for deltas with + no spread (then no note), or for a figure that leaves the float range (the note then names the changed-task or Holm floor where one keeps p above 0.05, and says nothing otherwise). It is not the power of the sign-flip test nor of the Holm-corrected decision; a Holm-calibrated figure would depend on the other rows' observed p, which a figure for planning the next run's task count cannot know. The sign-flip p over k changed (non-tied) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 39bdc7719..d383c0613 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -672,11 +672,12 @@ def _inconclusive_note(r: dict[str, Any], family: int) -> str | None: return None head = f" {r['measure']} inconclusive: the interval includes 0; " if r["mde"] is not None: + mde = f"{r['mde']:.3f}" if r["mde"] < 1e6 else f"{r['mde']:.3e}" # a figure near the float range is no 300-digit string return head + (f"an uncorrected t-test estimate puts the effect 80% power detects at this spread at about " - f"±{r['mde']:.3f} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test") + f"±{mde} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test") changed, holm = r["up"] + r["down"], r["p_holm_floor"] if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA and holm < ALPHA: - return None # every delta tied, or no spread + return None # every delta tied, no spread, or an mde beyond the float range if floor >= ALPHA: return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below " f"{floor:.4g}: no effect can reach p < {ALPHA} here") diff --git a/dokimi/dokimi/repos/compare_reach.py b/dokimi/dokimi/repos/compare_reach.py index facfb547b..63447e438 100644 --- a/dokimi/dokimi/repos/compare_reach.py +++ b/dokimi/dokimi/repos/compare_reach.py @@ -22,7 +22,9 @@ def min_detectable(sd: float, n: int) -> float: detects with :data:`POWER` over ``n`` tasks of spread ``sd``, a planning figure: not the power of the sign-flip test nor of the Holm-corrected decision. The quantiles are t at ``n - 1`` degrees of freedom, those of the planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`). - The arithmetic is on Python floats: a spread whose product leaves the float range gives ``inf``, no warning.""" + The arithmetic is on Python floats: a spread whose product leaves the float range gives ``inf``, no warning; no task (``n == 0``) gives ``nan``.""" + if n == 0: + return math.nan df = n - 1 quantiles = float(sp_stats.t.ppf(1 - ALPHA / 2, df)) + float(sp_stats.t.ppf(POWER, df)) return quantiles * float(sd) / math.sqrt(n) diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 304fb024b..c7927b514 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -8,10 +8,11 @@ import math import sys +import numpy as np import pytest from dokimi.eval.finite_delta import checked_delta, checked_mean -from dokimi.repos.compare import CompareError, _pair, _round, sign_flip_summary +from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary from dokimi.repos.compare_reach import min_detectable @@ -97,6 +98,26 @@ def test_min_detectable_past_the_range_is_inf_without_a_warning(): assert min_detectable(sys.float_info.max, 3) == math.inf +def test_min_detectable_of_no_task_is_nan(): + assert math.isnan(min_detectable(0.5, 0)) + + +@pytest.mark.filterwarnings("error") +def test_round_of_a_numpy_scalar_near_the_range_does_not_warn(): + assert _round(np.float64(1.0e307)) == 1.0e307 + + +def test_inconclusive_note_prints_a_large_mde_in_scientific_notation(): + row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) + note = _inconclusive_note({**row, "measure": "gate"}, 1) + assert "about ±9.276e+306 (12 tasks)" in note + + +def test_inconclusive_note_keeps_three_decimals_for_a_small_mde(): + row = {"measure": "gate", "n_tasks": 6, "delta": {"lo": -1.0, "hi": 1.0}, "mde": 123456.7891} + assert "about ±123456.789 (6 tasks)" in _inconclusive_note(row, 1) + + @pytest.mark.parametrize("value", [math.inf, -math.inf, math.nan]) def test_round_gives_no_figure_for_a_non_finite_value(value): assert _round(value) is None