diff --git a/dokimi/changelog.d/20261116_130000_antonwolfy_mde-float-range.md b/dokimi/changelog.d/20261116_130000_antonwolfy_mde-float-range.md new file mode 100644 index 000000000..b3a0eb86c --- /dev/null +++ b/dokimi/changelog.d/20261116_130000_antonwolfy_mde-float-range.md @@ -0,0 +1,3 @@ +### Fixed + +- **`mde` of deltas near the float maximum** — `sign_flip_summary` (`repo compare`) emitted `RuntimeWarning: overflow encountered in scalar multiply` computing `mde` for a spread near the float maximum (`[sys.float_info.max, 1, 1, 1, 1, 1]`), and `RuntimeWarning: overflow encountered in multiply` rounding it for twelve deltas of ±1e307. `min_detectable` now does its arithmetic on Python floats and `_round` converts to a float first, so neither warns. A planning figure beyond the float range, or any `inf` or `nan` passed to `_round`, is `None` (no figure to show) instead of `inf`. `min_detectable(sd, 0)` is still `nan`. The inconclusive note prints an `mde` of 1e6 or more in scientific notation (`about ±9.276e+306`) instead of every digit. The `mde` in the data and the note's text for an `mde` below 1e6 are unchanged. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index db26de3c4..a6cb51827 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -2504,8 +2504,8 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric `(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the per-task deltas, of the mean delta a one-sample t test at the uncorrected two-sided 5% level detects with 80% power at that spread, the t quantiles at `n-1` degrees of - freedom (every row has `mde` under `data` in `--json`; `null` for one task or deltas with - no spread, and then no note). It is not the power of the sign-flip test nor + freedom (every row has `mde` under `data` in `--json`; `null` for one task, for deltas with + no spread (then no note), or for a figure that leaves the float range (the note then names the changed-task or Holm floor where one keeps p above 0.05, and says nothing otherwise). It is not the power of the sign-flip test nor of the Holm-corrected decision; a Holm-calibrated figure would depend on the other rows' observed p, which a figure for planning the next run's task count cannot know. The sign-flip p over k changed (non-tied) diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 1d9a28c47..d383c0613 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -49,7 +49,7 @@ whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, a planning figure: an approximate, uncorrected t-based estimate of the mean delta a two-sided 5% one-sample t test detects with 80% power at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the per-task deltas, t at ``n - 1`` degrees of -freedom (in every row; ``None`` for one task or no spread), not the power of the sign-flip test nor of the +freedom (in every row; ``None`` for one task, no spread, or a spread so large the figure leaves the float range), not the power of the sign-flip test nor of the Holm-corrected decision. The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no effect of any size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead. ``mde`` is ``None`` too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the @@ -105,6 +105,7 @@ from __future__ import annotations import json +import math import statistics from collections import defaultdict from pathlib import Path @@ -671,11 +672,12 @@ def _inconclusive_note(r: dict[str, Any], family: int) -> str | None: return None head = f" {r['measure']} inconclusive: the interval includes 0; " if r["mde"] is not None: + mde = f"{r['mde']:.3f}" if r["mde"] < 1e6 else f"{r['mde']:.3e}" # a figure near the float range is no 300-digit string return head + (f"an uncorrected t-test estimate puts the effect 80% power detects at this spread at about " - f"±{r['mde']:.3f} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test") + f"±{mde} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test") changed, holm = r["up"] + r["down"], r["p_holm_floor"] if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA and holm < ALPHA: - return None # every delta tied, or no spread + return None # every delta tied, no spread, or an mde beyond the float range if floor >= ALPHA: return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below " f"{floor:.4g}: no effect can reach p < {ALPHA} here") @@ -753,7 +755,10 @@ def _mean(values: list[Any]) -> float | None: def _round(value: float | None) -> float | None: - return None if value is None else round(value, 6) + if value is None: + return None + number = float(value) # a numpy float64 would warn on overflow while rounding + return round(number, 6) if math.isfinite(number) else None def _num(value: float | None) -> str: diff --git a/dokimi/dokimi/repos/compare_reach.py b/dokimi/dokimi/repos/compare_reach.py index 50937d25f..63447e438 100644 --- a/dokimi/dokimi/repos/compare_reach.py +++ b/dokimi/dokimi/repos/compare_reach.py @@ -21,9 +21,13 @@ def min_detectable(sd: float, n: int) -> float: """An approximate, uncorrected t-based estimate of the mean delta a two-sided one-sample t test at :data:`ALPHA` detects with :data:`POWER` over ``n`` tasks of spread ``sd``, a planning figure: not the power of the sign-flip test nor of the Holm-corrected decision. The quantiles are t at ``n - 1`` degrees of freedom, those of the - planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`).""" + planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`). + The arithmetic is on Python floats: a spread whose product leaves the float range gives ``inf``, no warning; no task (``n == 0``) gives ``nan``.""" + if n == 0: + return math.nan df = n - 1 - return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n) + quantiles = float(sp_stats.t.ppf(1 - ALPHA / 2, df)) + float(sp_stats.t.ppf(POWER, df)) + return quantiles * float(sd) / math.sqrt(n) def holm_family(pvals: list[float | None], changed: list[int]) -> list[float | None]: diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index 646756fcf..c7927b514 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -6,11 +6,14 @@ from __future__ import annotations import math +import sys +import numpy as np import pytest from dokimi.eval.finite_delta import checked_delta, checked_mean -from dokimi.repos.compare import CompareError, _pair, sign_flip_summary +from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary +from dokimi.repos.compare_reach import min_detectable def _side(value: float) -> dict: @@ -71,7 +74,6 @@ def test_sign_flip_summary_refuses_finite_deltas_whose_sum_or_spread_leaves_the_ sign_flip_summary([_task(d) for d in deltas]) -@pytest.mark.filterwarnings("ignore:overflow encountered:RuntimeWarning") # the mde of a spread this large def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy(): row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) # cancels pairwise: sum() before 3.12 does not compensate assert row["delta"]["mean"] == 0.0 @@ -79,6 +81,48 @@ def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy(): assert row["delta"]["lo"] < 0 < row["delta"]["hi"] +@pytest.mark.filterwarnings("error") +def test_mde_is_none_when_the_planning_figure_leaves_the_float_range(): + row = sign_flip_summary([_task(d) for d in [sys.float_info.max, 1.0, 1.0, 1.0, 1.0, 1.0]]) + assert row["mde"] is None + + +@pytest.mark.filterwarnings("error") +def test_mde_of_a_large_spread_is_rounded_without_overflowing(): + row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) + assert math.isfinite(row["mde"]) and row["mde"] == pytest.approx(9.276e306, rel=1e-3) + + +@pytest.mark.filterwarnings("error") +def test_min_detectable_past_the_range_is_inf_without_a_warning(): + assert min_detectable(sys.float_info.max, 3) == math.inf + + +def test_min_detectable_of_no_task_is_nan(): + assert math.isnan(min_detectable(0.5, 0)) + + +@pytest.mark.filterwarnings("error") +def test_round_of_a_numpy_scalar_near_the_range_does_not_warn(): + assert _round(np.float64(1.0e307)) == 1.0e307 + + +def test_inconclusive_note_prints_a_large_mde_in_scientific_notation(): + row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) + note = _inconclusive_note({**row, "measure": "gate"}, 1) + assert "about ±9.276e+306 (12 tasks)" in note + + +def test_inconclusive_note_keeps_three_decimals_for_a_small_mde(): + row = {"measure": "gate", "n_tasks": 6, "delta": {"lo": -1.0, "hi": 1.0}, "mde": 123456.7891} + assert "about ±123456.789 (6 tasks)" in _inconclusive_note(row, 1) + + +@pytest.mark.parametrize("value", [math.inf, -math.inf, math.nan]) +def test_round_gives_no_figure_for_a_non_finite_value(value): + assert _round(value) is None + + def test_checked_mean_is_the_plain_mean_where_it_is_finite(): assert checked_mean([0.1, 0.2, 0.4]) == sum([0.1, 0.2, 0.4]) / 3 assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == sum([1.0e307, -1.0e307, 1.0e307]) / 3