Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- **`mde` of deltas near the float maximum** — `sign_flip_summary` (`repo compare`) emitted `RuntimeWarning: overflow encountered in scalar multiply` computing `mde` for a spread near the float maximum (`[sys.float_info.max, 1, 1, 1, 1, 1]`), and `RuntimeWarning: overflow encountered in multiply` rounding it for twelve deltas of ±1e307. `min_detectable` now does its arithmetic on Python floats and `_round` converts to a float first, so neither warns. A planning figure beyond the float range, or any `inf` or `nan` passed to `_round`, is `None` (no figure to show) instead of `inf`. `min_detectable(sd, 0)` is still `nan`. The inconclusive note prints an `mde` of 1e6 or more in scientific notation (`about ±9.276e+306`) instead of every digit. The `mde` in the data and the note's text for an `mde` below 1e6 are unchanged.
4 changes: 2 additions & 2 deletions dokimi/docs/repo-evaluation-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -2504,8 +2504,8 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric
`(t(0.975, n-1) + t(0.80, n-1)) * sd / sqrt(n)` over the per-task deltas, of
the mean delta a one-sample t test at the uncorrected two-sided 5% level
detects with 80% power at that spread, the t quantiles at `n-1` degrees of
freedom (every row has `mde` under `data` in `--json`; `null` for one task or deltas with
no spread, and then no note). It is not the power of the sign-flip test nor
freedom (every row has `mde` under `data` in `--json`; `null` for one task, for deltas with
no spread (then no note), or for a figure that leaves the float range (the note then names the changed-task or Holm floor where one keeps p above 0.05, and says nothing otherwise). It is not the power of the sign-flip test nor
of the Holm-corrected decision; a Holm-calibrated figure would depend on the
other rows' observed p, which a figure for planning the next run's task count
cannot know. The sign-flip p over k changed (non-tied)
Expand Down
13 changes: 9 additions & 4 deletions dokimi/dokimi/repos/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
whose interval includes 0 is inconclusive, not "no effect": the note gives ``mde``, a planning figure: an
approximate, uncorrected t-based estimate of the mean delta a two-sided 5% one-sample t test detects with 80% power
at the row's spread, ``(t_0.975 + t_0.80) * sd / sqrt(n)`` over the per-task deltas, t at ``n - 1`` degrees of
freedom (in every row; ``None`` for one task or no spread), not the power of the sign-flip test nor of the
freedom (in every row; ``None`` for one task, no spread, or a spread so large the figure leaves the float range), not the power of the sign-flip test nor of the
Holm-corrected decision. The sign-flip p over k changed tasks cannot go below ``2 / 2**k``, so at 5 or fewer no
effect of any size reaches p < 0.05: ``mde`` is ``None`` there too, and the note says how low p can go instead.
``mde`` is ``None`` too where a gate or score row's ``p_holm_floor`` (its ``p_holm`` with its p at that floor and the
Expand Down Expand Up @@ -105,6 +105,7 @@
from __future__ import annotations

import json
import math
import statistics
from collections import defaultdict
from pathlib import Path
Expand Down Expand Up @@ -671,11 +672,12 @@ def _inconclusive_note(r: dict[str, Any], family: int) -> str | None:
return None
head = f" {r['measure']} inconclusive: the interval includes 0; "
if r["mde"] is not None:
mde = f"{r['mde']:.3f}" if r["mde"] < 1e6 else f"{r['mde']:.3e}" # a figure near the float range is no 300-digit string
return head + (f"an uncorrected t-test estimate puts the effect 80% power detects at this spread at about "
f"±{r['mde']:.3f} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test")
f"±{mde} ({r['n_tasks']} tasks), before Holm and not for the sign-flip test")
changed, holm = r["up"] + r["down"], r["p_holm_floor"]
if r["p"] is None or (floor := min_sign_flip_p(changed)) < ALPHA and holm < ALPHA:
return None # every delta tied, or no spread
return None # every delta tied, no spread, or an mde beyond the float range
if floor >= ALPHA:
return head + (f"with {changed} changed task{'' if changed == 1 else 's'} the sign-flip p cannot go below "
f"{floor:.4g}: no effect can reach p < {ALPHA} here")
Expand Down Expand Up @@ -753,7 +755,10 @@ def _mean(values: list[Any]) -> float | None:

Comment thread
antonwolfy marked this conversation as resolved.
Resolved

def _round(value: float | None) -> float | None:
return None if value is None else round(value, 6)
if value is None:
return None
number = float(value) # a numpy float64 would warn on overflow while rounding
return round(number, 6) if math.isfinite(number) else None


def _num(value: float | None) -> str:
Expand Down
8 changes: 6 additions & 2 deletions dokimi/dokimi/repos/compare_reach.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,13 @@ def min_detectable(sd: float, n: int) -> float:
"""An approximate, uncorrected t-based estimate of the mean delta a two-sided one-sample t test at :data:`ALPHA`
detects with :data:`POWER` over ``n`` tasks of spread ``sd``, a planning figure: not the power of the sign-flip
test nor of the Holm-corrected decision. The quantiles are t at ``n - 1`` degrees of freedom, those of the
planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`)."""
planning figure: a row's interval inverts the sign-flip test and has no quantiles (:mod:`dokimi.eval.sign_flip`).
The arithmetic is on Python floats: a spread whose product leaves the float range gives ``inf``, no warning; no task (``n == 0``) gives ``nan``."""
if n == 0:
return math.nan
df = n - 1
return (sp_stats.t.ppf(1 - ALPHA / 2, df) + sp_stats.t.ppf(POWER, df)) * sd / math.sqrt(n)
quantiles = float(sp_stats.t.ppf(1 - ALPHA / 2, df)) + float(sp_stats.t.ppf(POWER, df))
return quantiles * float(sd) / math.sqrt(n)
Comment thread
antonwolfy marked this conversation as resolved.
Resolved


def holm_family(pvals: list[float | None], changed: list[int]) -> list[float | None]:
Expand Down
48 changes: 46 additions & 2 deletions dokimi/tests/test_repo_compare_overflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@
from __future__ import annotations

import math
import sys

import numpy as np
import pytest

from dokimi.eval.finite_delta import checked_delta, checked_mean
from dokimi.repos.compare import CompareError, _pair, sign_flip_summary
from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary
from dokimi.repos.compare_reach import min_detectable


def _side(value: float) -> dict:
Expand Down Expand Up @@ -71,14 +74,55 @@ def test_sign_flip_summary_refuses_finite_deltas_whose_sum_or_spread_leaves_the_
sign_flip_summary([_task(d) for d in deltas])


@pytest.mark.filterwarnings("ignore:overflow encountered:RuntimeWarning") # the mde of a spread this large
def test_sign_flip_summary_shifts_deltas_near_the_range_on_a_scaled_copy():
row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6) # cancels pairwise: sum() before 3.12 does not compensate
assert row["delta"]["mean"] == 0.0
assert math.isfinite(row["delta"]["lo"]) and math.isfinite(row["delta"]["hi"])
assert row["delta"]["lo"] < 0 < row["delta"]["hi"]


@pytest.mark.filterwarnings("error")
def test_mde_is_none_when_the_planning_figure_leaves_the_float_range():
row = sign_flip_summary([_task(d) for d in [sys.float_info.max, 1.0, 1.0, 1.0, 1.0, 1.0]])
assert row["mde"] is None


@pytest.mark.filterwarnings("error")
def test_mde_of_a_large_spread_is_rounded_without_overflowing():
row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6)
assert math.isfinite(row["mde"]) and row["mde"] == pytest.approx(9.276e306, rel=1e-3)


@pytest.mark.filterwarnings("error")
def test_min_detectable_past_the_range_is_inf_without_a_warning():
assert min_detectable(sys.float_info.max, 3) == math.inf


def test_min_detectable_of_no_task_is_nan():
assert math.isnan(min_detectable(0.5, 0))


@pytest.mark.filterwarnings("error")
def test_round_of_a_numpy_scalar_near_the_range_does_not_warn():
assert _round(np.float64(1.0e307)) == 1.0e307


def test_inconclusive_note_prints_a_large_mde_in_scientific_notation():
row = sign_flip_summary([_task(1.0e307), _task(-1.0e307)] * 6)
note = _inconclusive_note({**row, "measure": "gate"}, 1)
assert "about ±9.276e+306 (12 tasks)" in note


def test_inconclusive_note_keeps_three_decimals_for_a_small_mde():
row = {"measure": "gate", "n_tasks": 6, "delta": {"lo": -1.0, "hi": 1.0}, "mde": 123456.7891}
assert "about ±123456.789 (6 tasks)" in _inconclusive_note(row, 1)


@pytest.mark.parametrize("value", [math.inf, -math.inf, math.nan])
def test_round_gives_no_figure_for_a_non_finite_value(value):
assert _round(value) is None


def test_checked_mean_is_the_plain_mean_where_it_is_finite():
assert checked_mean([0.1, 0.2, 0.4]) == sum([0.1, 0.2, 0.4]) / 3
assert checked_mean([1.0e307, -1.0e307, 1.0e307]) == sum([1.0e307, -1.0e307, 1.0e307]) / 3
Expand Down
Loading