From 2228387b67213f379069079fcdf8ae32129a9d2d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 10:54:56 +0000 Subject: [PATCH 1/5] fix(eval): the sign-flip tie width follows the scale of the deltas (#762) The tie tolerance EPS = 1e-9 was absolute, so a delta vector that varies at a tiny scale (largest delta 1e-300) was all ties: paired_test gave it n_changed 0 and no p next to a large effect size, and repo compare counted every pair as tied. A new helper tie_tolerance(deltas) in eval/sign_flip.py gives the width of a tie for a vector: EPS when its largest finite delta is EPS or more, so every result at score scale is unchanged, and EPS times that largest delta below it. The scale is the vector's, not a pair's, and an all-zero vector keeps EPS, so it has no p. sign_flip_p, ShiftTest, sign_flip_interval and sign_flip_row, the n_changed of _describe_deltas, the up, down, tied and mde of repo compare, its headroom checks and the non-inferiority slack of paired_cluster_difference all take it from that helper. The interval shifts the deltas by theta but keeps the tolerance (and the bisection step and far bound) of the original vector, so p(theta) stays one step function. The ledger's abs(mean) > EPS guard and checklist.TIE_EPSILON compare scores in [0, 1] and keep the absolute width. Two tests that pinned the absolute tie on a vector whose largest delta is 1e-12 now pin the scaled one. EPS stays importable from eval.sign_flip, eval.stats and repos.compare. Signed-off-by: Anton Volkov --- .../20261116_120000_sign-flip-relative-tie.md | 3 + dokimi/docs/benchmarking-and-comparison.md | 4 +- dokimi/dokimi/eval/paired_cluster.py | 11 +- dokimi/dokimi/eval/sign_flip.py | 56 +++++++--- dokimi/dokimi/eval/stats.py | 16 ++- dokimi/dokimi/repos/compare.py | 26 +++-- dokimi/tests/test_repo_compare.py | 5 +- dokimi/tests/test_repo_compare_interval.py | 5 +- dokimi/tests/test_sign_flip_relative_tie.py | 104 ++++++++++++++++++ 9 files changed, 195 insertions(+), 35 deletions(-) create mode 100644 dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md create mode 100644 dokimi/tests/test_sign_flip_relative_tie.py diff --git a/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md b/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md new file mode 100644 index 000000000..de9a46098 --- /dev/null +++ b/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md @@ -0,0 +1,3 @@ +### Fixed + +- **The sign-flip test no longer reads deltas that vary at a tiny scale as ties** — deltas such as `[1e-300, 3e-300, 2e-300, ...]` were all within the absolute 1e-9 tie width, so `paired_test` and `repo compare` gave them `n_changed` 0 and no `p` next to a `large` effect size. A tie is now within 1e-9 of 0 as before, but within 1e-9 times the largest delta when that is below 1e-9, so such a vector gets a `p`, an interval and the same verdict as its 1e-3 scale copy. A vector whose largest delta is 1e-9 or more keeps the 1e-9 tolerance, and every result for it is unchanged; an all-zero vector still has no `p`. `repo compare` counts `up`, `down`, `tied` and `mde` with the same width, and the non-inferiority check of `paired_cluster_difference` scales its slack the same way. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4bc8639e5..56ab75dc1 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -369,7 +369,9 @@ Each row's `p` is one two-sided sign-flip test on the per-question deltas: under the null each delta's sign is a fair coin, and p is the share of sign patterns whose |sum| reaches the observed one (exact up to 16 changed pairs, Monte Carlo with a fixed seed above). A delta within 1e-9 of 0 is a tie and carries no sign, -so over k changed pairs (`n_changed`) p cannot go below 2/2^k: at 5 or fewer no +unless the largest delta is itself below 1e-9: a tie is then within 1e-9 times that +largest delta, so deltas that vary at the 1e-12 scale (largest 5e-12) tie only +within 5e-21 and get a p. So over k changed pairs (`n_changed`) p cannot go below 2/2^k: at 5 or fewer no data whatsoever reaches 0.05. The test runs from 2 pairs and its p is printed. An arm under that floor can never be significant, so it stays out of the multiplier, as a `repo compare` row does: it counts in `family_size` but not in diff --git a/dokimi/dokimi/eval/paired_cluster.py b/dokimi/dokimi/eval/paired_cluster.py index efceb657e..684c719a9 100644 --- a/dokimi/dokimi/eval/paired_cluster.py +++ b/dokimi/dokimi/eval/paired_cluster.py @@ -35,6 +35,7 @@ from collections.abc import Hashable, Sequence from typing import Any, Dict, List, Optional, Tuple +from dokimi.eval.sign_flip import EPS, tie_tolerance from dokimi.eval.stats import bootstrap_ci #: Fewest clusters with a usable pair before an interval is reported at all. @@ -43,8 +44,9 @@ #: Bootstrap resamples; the same default as :func:`dokimi.eval.stats.bootstrap_ci`. DEFAULT_RESAMPLES = 10_000 -#: Tolerance for comparing a bound with ``-margin``, so float noise does not flip a verdict. -_EPS = 1e-9 +#: Tolerance for comparing a bound with ``-margin``, so float noise does not flip a verdict; the cap of +#: :func:`~dokimi.eval.sign_flip.tie_tolerance`, which scales it down for bounds that are all below it. +_EPS = EPS #: Display precision of the delta and its bounds; the status is decided on unrounded values. DISPLAY_DIGITS = 4 @@ -183,9 +185,10 @@ def _verdict(lo: float, hi: float, margin: Optional[float]) -> Optional[str]: """Non-inferiority status of the interval ``[lo, hi]`` against ``-margin``.""" if margin is None: return None - if hi < -margin - _EPS: + eps = tie_tolerance([lo, hi, margin]) # float slack at the scale of the bounds: _EPS, less below it + if hi < -margin - eps: return "inferior" # A zero-width interval has no spread estimate, so it cannot show non-inferiority. - if lo > -margin + _EPS and hi - lo > _EPS: + if lo > -margin + eps and hi - lo > eps: return "noninferior" return "inconclusive" diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index f0e400233..f55eb31a4 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -4,7 +4,7 @@ """The paired sign-flip test and the 95% interval that inverts it, shared by ``repo compare`` and the arms track. :func:`sign_flip_p` is the two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null, -over the deltas not within :data:`EPS` of 0: exact up to :data:`EXACT_MAX` of them, Monte Carlo above. +over the deltas not within :func:`tie_tolerance` of 0 (:data:`EPS`, or less for a vector below it): exact up to :data:`EXACT_MAX` of them, Monte Carlo above. :func:`min_sign_flip_p` is the floor it cannot go below over k changed deltas, ``2 / 2**k`` exact. ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as @@ -41,6 +41,7 @@ import math import random import sys +from collections.abc import Sequence from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -52,7 +53,8 @@ EXACT_MAX = 16 #: Resamples of the Monte Carlo sign-flip test above :data:`EXACT_MAX`. N_RESAMPLES = 20_000 -#: Deltas closer to zero than this are ties. +#: Deltas closer to zero than this are ties: the cap of :func:`tie_tolerance`, which a vector whose largest +#: ``|delta|`` is below it scales down. EPS = 1e-9 #: How close a bisected end comes to the step of ``p(theta)`` it stands for: far below the 3 printed decimals. TOL = 1e-7 @@ -60,6 +62,25 @@ METHOD = "sign_flip_inversion" +def tie_scale(deltas: Sequence[float]) -> float: + """The scale a vector of deltas is judged at: 1.0 unless its largest finite ``|delta|`` is below :data:`EPS` + (and not 0), where it is that largest ``|delta|``. A vector of scores keeps the absolute :data:`EPS`.""" + largest = max((abs(x) for x in deltas if math.isfinite(x)), default=0.0) + return largest if 0.0 < largest < EPS else 1.0 + + +def tie_tolerance(deltas: Sequence[float]) -> float: + """The width of a tie for this vector: a delta is a tie when ``abs(x) <= tie_tolerance(deltas)``. + + :data:`EPS` (1e-9, the float noise on a score near 1) for a vector whose largest finite ``|delta|`` is + :data:`EPS` or more, so every result at score scale is the same as with the absolute rule; ``EPS`` times that + largest ``|delta|`` below it, so a vector that varies at 1e-300 ties only its own float noise. The scale is the + vector's, not a pair's. An all-zero vector (or none finite) keeps :data:`EPS`: every delta of it is a tie. + The interval shifts the deltas by theta and takes the tolerance of the original vector, not of the shifted one. + """ + return EPS * tie_scale(deltas) + + def min_sign_flip_p(changed: int) -> float: """The smallest p :func:`sign_flip_p` can give over ``changed`` non-zero deltas, every sign the same: only that pattern and its mirror reach the observed ``|sum|``, ``2 / 2**changed`` exact, ``1 / (N + 1)`` Monte Carlo.""" @@ -74,12 +95,13 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL Rounded to 6 places as printed, unless ``exact``. Deltas whose largest magnitude is above ``max float / (8 * n)`` are divided by a power of two (exact) first, as in :func:`sign_flip_interval`, so no sum leaves the float range. """ - d = [x for x in deltas if abs(x) > EPS] + eps = tie_tolerance(deltas) + d = [x for x in deltas if abs(x) > eps] if not d: return None - eps, k = EPS, _scale_exponent(d) + k = _scale_exponent(d) if k: # near the float maximum the sums go to inf/nan: the same p on the deltas divided by a power of two - d, eps = [math.ldexp(x, -k) for x in d], math.ldexp(EPS, -k) + d, eps = [math.ldexp(x, -k) for x in d], math.ldexp(eps, -k) observed = abs(sum(d)) - eps if len(d) <= EXACT_MAX: hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed @@ -126,9 +148,11 @@ class ShiftTest: row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`).""" def __init__(self, deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, - eps: float = EPS) -> None: + eps: float | None = None) -> None: import numpy as np + if eps is None: + eps = tie_tolerance(deltas) changed = [x for x in deltas if abs(x) > eps] self.d = np.array(changed + [x for x in deltas if abs(x) <= eps], dtype=float) # sign_flip_p's order first self.changed, self.seed, self.n_resamples, self.eps = len(changed), seed, n_resamples, eps @@ -169,13 +193,13 @@ def sign_flip_interval(deltas: list[float], p_zero: float | None, seed: int = 0, :data:`EPS` and :data:`TOL` divided the same, and each end is multiplied back. An end that comes back past the float maximum is the maximum (``±max``), never ``inf``: from 6 deltas on a shift past it puts every delta on one side, which the test rejects, so the search overshoots a true end at the maximum by a few ulps.""" - k = _scale_exponent(deltas) + k, unit = _scale_exponent(deltas), tie_scale(deltas) # the tie width and the search step are the vector's if k: scaled = [math.ldexp(x, -k) for x in deltas] - ends = _ends(scaled, p_zero, seed, math.ldexp(EPS, -k), math.ldexp(TOL, -k)) + ends = _ends(scaled, p_zero, seed, math.ldexp(EPS * unit, -k), math.ldexp(TOL * unit, -k), unit) ends = (_scaled_back(ends[0], k), _scaled_back(ends[1], k)) else: - ends = _ends(deltas, p_zero, seed, EPS, TOL) + ends = _ends(deltas, p_zero, seed, EPS * unit, TOL * unit, unit) return (_rounded(ends[0]), _rounded(ends[1])) if rounded else ends @@ -198,13 +222,14 @@ def _scaled_back(end: float | None, k: int) -> float | None: return math.copysign(sys.float_info.max, end) -def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float, tol: float +def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float, tol: float, unit: float = 1.0 ) -> tuple[float | None, float | None]: test, mean = ShiftTest(deltas, seed, eps=eps), math.fsum(deltas) / len(deltas) zero_inside, low, high = p_zero is None or p_zero >= ALPHA, min(*deltas, 0.0), max(*deltas, 0.0) - # far is 1 past the extreme, or twice it from 1 on: 1e16 + 1.0 rounds back onto 1e16, all tied - return (_end(test, mean, low - max(1.0, -low), zero_inside, tol), - _end(test, mean, high + max(1.0, high), zero_inside, tol)) + # far is one unit (1, or the scale of a vector below EPS) past the extreme, or twice it from one unit on: + # 1e16 + 1.0 rounds back onto 1e16, all tied + return (_end(test, mean, low - max(unit, -low), zero_inside, tol), + _end(test, mean, high + max(unit, high), zero_inside, tol)) def _end(test: ShiftTest, mean: float, far: float, zero_inside: bool, tol: float = TOL) -> float | None: @@ -240,12 +265,13 @@ def sign_flip_row(deltas: list[float], seed: int = 0, *, exact: bool) -> dict: ``p`` is :func:`sign_flip_p` (rounded to 6 places unless ``exact``, as ``repo compare`` prints it). ``lo``/``hi`` are the unrounded ends of the interval that inverts it, and ``ci_lo``/``ci_hi`` the same ends as printed - (:func:`sign_flip_interval`, rounded). ``n_changed`` counts the deltas not within :data:`EPS` of 0. + (:func:`sign_flip_interval`, rounded). ``n_changed`` counts the deltas not within :func:`tie_tolerance` of 0. """ p = sign_flip_p(deltas, seed=seed, exact=exact) lo, hi = sign_flip_interval(deltas, p, seed=seed, rounded=False) + tol = tie_tolerance(deltas) return {"p": p, "lo": lo, "hi": hi, "ci_lo": _rounded(lo), "ci_hi": _rounded(hi), - "n_changed": sum(abs(d) > EPS for d in deltas)} + "n_changed": sum(abs(d) > tol for d in deltas)} def round_mean(mean: float) -> float: diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index ddc217e33..8eed18fd7 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -65,7 +65,16 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from dokimi.eval.finite_delta import check_sum_range, checked_delta -from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, round_mean, sign_flip_row, within +from dokimi.eval.sign_flip import ( # noqa: F401 (EPS stays importable from here) + ALPHA, + EPS, + METHOD, + min_sign_flip_p, + round_mean, + sign_flip_row, + tie_tolerance, + within, +) from dokimi.records import BASELINE, CONTEXT, RecordLike, records #: Checked without importing scipy, whose OpenBLAS starts a thread per core @@ -569,7 +578,7 @@ def _describe_deltas( ) -> Dict[str, Any]: """Effect size, the sign-flip p, and the interval that inverts it, for a delta vector. - ``p`` is the two-sided sign-flip p over the deltas not within ``EPS`` of 0 + ``p`` is the two-sided sign-flip p over the deltas not within ``tie_tolerance`` of 0 (:func:`dokimi.eval.sign_flip.sign_flip_p`, exact up to 16 of them, Monte Carlo from ``seed`` above), left unrounded for the caller to correct and then :func:`_finalize`. Correcting an already-rounded p multiplies its @@ -608,7 +617,8 @@ def _describe_deltas( p, lo, hi, shown_lo, shown_hi = row["p"], row["lo"], row["hi"], row["ci_lo"], row["ci_hi"] else: # a NaN delta is no evidence either way: no p, no verdict, open ends p, lo, hi, shown_lo, shown_hi = None, None, None, None, None - n_changed = sum(1 for x in d if abs(x) > EPS) + tol = tie_tolerance(d) + n_changed = sum(1 for x in d if abs(x) > tol) return { "n": len(d), diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 1d9a28c47..de8ce7673 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -111,7 +111,7 @@ from typing import Any from dokimi.context.consistency import Run, run_from_events -from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, sign_flip_row, within +from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, sign_flip_row, tie_tolerance, within # noqa: F401 from dokimi.eval.stats import correct_family from dokimi.eval.finite_delta import checked_delta, checked_mean from dokimi.utils import same_model @@ -491,18 +491,22 @@ def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | if failed and any(failed): return None # its 0 is not a measurement, at either end (a score.max of 0 would read it as the top) + # a tie is as wide as the vector it is read on: the pairs' values for a score, their deltas for a move + value_tol = tie_tolerance([v for p in pairs for v in (p["baseline"], p["treatment"])]) + delta_tol = tie_tolerance([p["treatment"] - p["baseline"] for p in pairs]) + def at_top(value: float, top: float | None) -> bool: - return top is not None and value >= top - EPS + return top is not None and value >= top - value_tol # score.max is what the oracle must reach, not a bound: a pair that moved past it is an effect, not a top - base_top = [at_top(p["baseline"], b) and abs(p["treatment"] - p["baseline"]) <= EPS + base_top = [at_top(p["baseline"], b) and abs(p["treatment"] - p["baseline"]) <= delta_tol for p, (b, _) in zip(pairs, tops, strict=True)] if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): return "ceiling" - if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): + if all(abs(p["baseline"]) <= value_tol and abs(p["treatment"]) <= value_tol for p in pairs): return "floor" if (len(pairs) >= 2 and base_top.count(False) <= 1 - and all(p["treatment"] >= p["baseline"] - EPS for p in pairs)): + and all(p["treatment"] >= p["baseline"] - delta_tol for p in pairs)): return "near_ceiling" return None @@ -588,6 +592,7 @@ def sign_flip_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: raise CompareError("the deltas of the tasks are too large to test: a sum or a spread of them leaves the float range") from exc p, lo, hi = row["p"], row["ci_lo"], row["ci_hi"] reachable = min_sign_flip_p(row["n_changed"]) < ALPHA # else no effect can be detected + tol = tie_tolerance(deltas) return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), @@ -597,11 +602,11 @@ def sign_flip_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: "p": p, "p_holm": None, "p_holm_floor": None, - "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS and reachable else None, + "mde": _round(min_detectable(sd, len(deltas))) if sd > tol and reachable else None, "interval_p_disagree": p is not None and includes_zero(lo, hi) == (p < ALPHA), - "up": sum(d > EPS for d in deltas), - "down": sum(d < -EPS for d in deltas), - "tied": sum(abs(d) <= EPS for d in deltas), + "up": sum(d > tol for d in deltas), + "down": sum(d < -tol for d in deltas), + "tied": sum(abs(d) <= tol for d in deltas), "tasks": [{**p, "baseline": _round(p["baseline"]), "treatment": _round(p["treatment"]), "delta": _round(p["delta"])} for p in pairs], } @@ -626,7 +631,8 @@ def format_compare(summary: dict[str, Any]) -> str: f"{r['up']}/{r['down']}/{r['tied']}") for r in rows: if r["measure"] in PRIMARY: - moved = [t for t in r["tasks"] if abs(t["delta"]) > EPS] + tol = tie_tolerance([t["delta"] for t in r["tasks"]]) + moved = [t for t in r["tasks"] if abs(t["delta"]) > tol] if moved: lines.append(f" {r['measure']} changed: " + ", ".join( f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index af313212b..ec8d390db 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -103,7 +103,10 @@ def test_rates_of_three_reps_give_the_exact_p_end_to_end(tmp_path): def test_sign_flip_has_no_p_when_every_task_tied(): - assert sign_flip_p([0.0, 0.0, 1e-12]) is None + assert sign_flip_p([0.0, 0.0, 0.0]) is None + # the 1e-12 is the vector's own scale, not float noise on a score: a change, with a p over that one delta + assert sign_flip_p([0.0, 0.0, 1e-12]) == 1.0 + assert sign_flip_p([1.0] * 6 + [1e-12]) == 0.03125 # beside deltas of 1.0 it is the noise it was: 6 changed def test_sign_flip_monte_carlo_counts_the_observed_pattern_and_follows_its_seed(): diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 0fa4f2c86..2d14feda2 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -120,8 +120,11 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(d, seed).p(0.0) == sign_flip_p(d, seed=seed, exact=True), (d, seed) near_tie = [1.0] * 16 + [1e-12] # within EPS of 0: a tie, so 16 changed and still exact, as sign_flip_p has it assert ShiftTest(near_tie).p(0.0) == sign_flip_p(near_tie, exact=True) == 2 / 2 ** 16 - tied = [0.0, 1e-12, -1e-12] # every delta tied: no p at 0, nor where a shift ties every delta again + tied = [0.0] * 3 # every delta tied: no p at 0, nor where a shift ties every delta again assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None + # deltas of 1e-12 are no ties of their own vector (largest 1e-12): they cancel to a p of 1, not to no p + small = [0.0, 1e-12, -1e-12] + assert ShiftTest(small).p(0.0) == sign_flip_p(small, exact=True) == 1.0 @pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)], diff --git a/dokimi/tests/test_sign_flip_relative_tie.py b/dokimi/tests/test_sign_flip_relative_tie.py new file mode 100644 index 000000000..0d1e5b8b4 --- /dev/null +++ b/dokimi/tests/test_sign_flip_relative_tie.py @@ -0,0 +1,104 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The sign-flip tie width is the delta vector's (dokimi.eval.sign_flip.tie_tolerance), capped at EPS (issue #762).""" + +from __future__ import annotations + +import pytest + +from dokimi.eval import stats +from dokimi.eval.paired_cluster import paired_cluster_difference +from dokimi.eval.sign_flip import ALPHA, EPS, ShiftTest, sign_flip_interval, sign_flip_p, sign_flip_row, tie_tolerance +from dokimi.repos.compare import headroom, sign_flip_summary + +GAINS = [2, 4, 3, 5, 6, 9, 7, 8] + + +def _pairs(deltas: list[float]) -> list[dict]: + return [{"task": f"t{i}", "baseline": 0.0, "treatment": d, "delta": d} for i, d in enumerate(deltas)] + + +def _includes_zero(lo: float | None, hi: float | None) -> bool: + return (lo is None or lo <= 0) and (hi is None or hi >= 0) + + +def test_tie_tolerance_is_eps_from_a_largest_delta_of_eps_up_and_scales_below_it(): + assert tie_tolerance([1.0, 1e-12]) == tie_tolerance([0.5, 0.2]) == tie_tolerance([EPS, 0.0]) == EPS + assert tie_tolerance([1e-12, -4e-12, 2e-12]) == pytest.approx(EPS * 4e-12) + assert tie_tolerance([1e-300]) == pytest.approx(EPS * 1e-300) + # nothing to scale by: every delta of such a vector is a tie under EPS, as before + assert tie_tolerance([0.0, 0.0]) == tie_tolerance([]) == tie_tolerance([float("nan"), float("inf")]) == EPS + assert tie_tolerance([3e-12, float("inf"), float("nan")]) == pytest.approx(EPS * 3e-12) # non-finite: not a scale + + +def test_deltas_at_1e_300_get_the_p_and_the_verdict_of_the_same_deltas_at_1e_3(): + tiny = stats.paired_test([g * 1e-300 for g in GAINS], [1e-300] * 8) + usual = stats.paired_test([g * 1e-3 for g in GAINS], [1e-3] * 8) + assert (tiny["p"], tiny["n_changed"], tiny["significant"]) == (usual["p"], usual["n_changed"], usual["significant"]) + assert (tiny["p"], tiny["n_changed"], tiny["significant"]) == (0.0078, 8, True) + + +def test_the_interval_and_the_p_agree_at_a_tiny_scale_and_on_a_mixed_vector(): + tiny = [g * 1e-300 for g in GAINS] + mixed = [1e-3, 2e-3, 1.5e-3, 3e-3, 2.5e-3, 1.2e-3, 1e-12] # the 1e-12 stays a tie under the 1e-9 cap + small = [-1e-12, 1e-12, 3e-12, 2e-12, 5e-13, 4e-12, -2e-12] + for deltas in (tiny, mixed, small): + row = sign_flip_row(deltas, exact=True) + assert row["p"] is not None + assert _includes_zero(row["lo"], row["hi"]) == (row["p"] >= ALPHA), deltas + assert sign_flip_row(mixed, exact=True)["n_changed"] == 6 + assert sign_flip_row(tiny, exact=True)["n_changed"] == 8 + # the ends are step points of the one p(theta) of the original vector, at its own scale + lo, hi = sign_flip_interval(tiny, sign_flip_p(tiny), rounded=False) + test = ShiftTest(tiny) + step = 1e-6 * 1e-300 + assert 0 < lo < hi + assert test.p(lo) >= ALPHA > test.p(lo - step) and test.p(hi) >= ALPHA > test.p(hi + step) + + +def test_the_shifted_deltas_keep_the_tolerance_of_the_original_vector(): + # shifted by 1.0 the deltas are 1e-10 off 0, and 1e-10 is a tie of the original (largest 1.0002): no p, not a p + # of a vector that would be read at its own 1e-10 scale + d = [1.0, 1.0 + 1e-10, 1.0 - 1e-10, 1.0 + 2e-10, 1.0, 1.0] + assert ShiftTest(d).p(1.0) is None + assert sign_flip_p([x - 1.0 for x in d]) is not None # the vector of the shifted values alone is another scale + + +def test_a_one_e_minus_12_scale_vector_is_not_all_ties_but_zeros_still_are(): + assert sign_flip_row([0.0] * 8, exact=True)["p"] is None + assert sign_flip_row([0.0] * 8, exact=True)["n_changed"] == 0 + small = sign_flip_row([g * 1e-12 for g in GAINS], exact=True) + usual = sign_flip_row([g * 1e-3 for g in GAINS], exact=True) + assert (small["p"], small["n_changed"]) == (usual["p"], usual["n_changed"]) == (0.0078125, 8) + + +def test_repo_compare_counts_up_down_and_tied_at_the_scale_of_the_deltas(): + tiny = sign_flip_summary(_pairs([g * 1e-300 for g in GAINS]), exact=True) + assert (tiny["up"], tiny["down"], tiny["tied"]) == (8, 0, 0) + assert tiny["p"] == 0.0078125 and tiny["mde"] is not None + mixed = sign_flip_summary(_pairs([1e-3] * 6 + [1e-12, -1e-12])) + assert (mixed["up"], mixed["down"], mixed["tied"]) == (6, 0, 2) + zeros = sign_flip_summary(_pairs([0.0] * 8)) + assert (zeros["up"], zeros["down"], zeros["tied"], zeros["p"]) == (0, 0, 8, None) + + +def test_headroom_reads_a_floor_only_in_zeros_not_in_values_that_vary_at_a_tiny_scale(): + def pairs(baseline: float, treatment: float) -> list[dict]: + return [{"baseline": baseline, "treatment": treatment} for _ in range(3)] + + tops = [(1.0, 1.0)] * 3 + assert headroom(pairs(0.0, 0.0), tops) == "floor" + assert headroom(pairs(1e-12, 1e-12), tops) is None + + +def test_the_non_inferiority_slack_scales_down_with_bounds_that_are_all_below_it(): + clusters = [f"c{i}" for i in range(12)] + control = [0.0] * 12 + gain = [1e-12 * (1 + i / 10) for i in range(12)] + assert paired_cluster_difference(clusters, gain, control, margin=0.0)["noninferiority"] == "noninferior" + loss = [-g for g in gain] + assert paired_cluster_difference(clusters, loss, control, margin=0.0)["noninferiority"] == "inferior" + # at score scale the 1e-9 slack is kept: a lower bound within it of -margin is not a win + score = [0.0] * 11 + [1.0] + assert paired_cluster_difference(clusters, score, control, margin=0.0)["noninferiority"] == "inconclusive" From 464c159f1dc009301338dec29ccb143e5180b7b1 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 13:01:57 +0000 Subject: [PATCH 2/5] fix(eval): the sign-flip tie width follows the values of each pair Settles the review of #818: a tie width taken from the delta vector made a vector of float noise significant (`paired_test([0.1 + 0.2] * 8, [0.3] * 8)` gave p 0.0078 and 8 changed), made more evidence in one direction remove a significant result (`[5e-10] * 8` is p 0.0078, with one more delta of 1.1e-9 it was 1.0), and gave deltas of the smallest floats an interval of `[0, 0]` beside a significant p. A pair now ties within `EPS * min(1, max(|treatment|, |baseline|))` (`pair_magnitudes`, `tie_widths`), since float noise scales with the values subtracted. `sign_flip_p`, `ShiftTest` (a width per delta), `sign_flip_interval`, `sign_flip_row`, `paired_test`, `arm_significance` and `sign_flip_summary` take the pairs' magnitudes; deltas alone keep the absolute `EPS`. Values all below 2**-664 are searched multiplied by a power of two, and an end that comes back below the smallest float is that float on its side of 0, so the interval excludes 0 with the p. The `headroom`, `format_compare` and `paired_cluster` changes of the first commit are taken back: they read tolerances off deltas, which are float noise next to the values they come from. The unused `EPS` imports go with them. `format_compare` keeps `EPS` and says its line follows the printed, rounded deltas. Signed-off-by: Anton Volkov --- .../20261116_120000_sign-flip-relative-tie.md | 2 +- dokimi/docs/benchmarking-and-comparison.md | 7 +- dokimi/dokimi/eval/paired_cluster.py | 11 +- dokimi/dokimi/eval/sign_flip.py | 137 +++++++++------ dokimi/dokimi/eval/stats.py | 22 +-- dokimi/dokimi/repos/compare.py | 41 +++-- dokimi/tests/test_repo_compare.py | 5 +- dokimi/tests/test_repo_compare_interval.py | 5 +- dokimi/tests/test_sign_flip_relative_tie.py | 162 +++++++++++------- 9 files changed, 231 insertions(+), 161 deletions(-) diff --git a/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md b/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md index de9a46098..3ce7f733f 100644 --- a/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md +++ b/dokimi/changelog.d/20261116_120000_sign-flip-relative-tie.md @@ -1,3 +1,3 @@ ### Fixed -- **The sign-flip test no longer reads deltas that vary at a tiny scale as ties** — deltas such as `[1e-300, 3e-300, 2e-300, ...]` were all within the absolute 1e-9 tie width, so `paired_test` and `repo compare` gave them `n_changed` 0 and no `p` next to a `large` effect size. A tie is now within 1e-9 of 0 as before, but within 1e-9 times the largest delta when that is below 1e-9, so such a vector gets a `p`, an interval and the same verdict as its 1e-3 scale copy. A vector whose largest delta is 1e-9 or more keeps the 1e-9 tolerance, and every result for it is unchanged; an all-zero vector still has no `p`. `repo compare` counts `up`, `down`, `tied` and `mde` with the same width, and the non-inferiority check of `paired_cluster_difference` scales its slack the same way. +- **The sign-flip test no longer reads deltas between values of a tiny scale as ties** — `paired_test([2e-300, 4e-300, ...], [1e-300] * 8)` had `n_changed` 0 and no `p` next to a `large` effect size, because every delta was within the absolute 1e-9 tie width. A pair whose larger value is below 1 now ties within 1e-9 times that value (1e-9 for values of 1 or more, as before), since float noise in `t - b` scales with the values and not with the delta: that vector gets a `p`, an interval and the verdict of its 1e-3 scale copy, and `0.1 + 0.2` against `0.3` stays a tie. Each pair keeps its own width whatever the other deltas are. `paired_test` and `repo compare` (`n_changed`, `up`, `down`, `tied`, `mde`) use it; a caller that passes deltas alone keeps the absolute 1e-9. An interval of deltas near the smallest floats excludes 0 with its `p`, where it was `[0, 0]`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 56ab75dc1..fd714a8cb 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -369,9 +369,10 @@ Each row's `p` is one two-sided sign-flip test on the per-question deltas: under the null each delta's sign is a fair coin, and p is the share of sign patterns whose |sum| reaches the observed one (exact up to 16 changed pairs, Monte Carlo with a fixed seed above). A delta within 1e-9 of 0 is a tie and carries no sign, -unless the largest delta is itself below 1e-9: a tie is then within 1e-9 times that -largest delta, so deltas that vary at the 1e-12 scale (largest 5e-12) tie only -within 5e-21 and get a p. So over k changed pairs (`n_changed`) p cannot go below 2/2^k: at 5 or fewer no +unless the values of its pair are below 1: float noise in a delta scales with the +values subtracted, so a pair whose larger value is m below 1 ties within 1e-9 times m +(a score of 0.3 against 0.1 + 0.2 is a tie; 1e-12 against 0 is a change, as 1e-3 against 0 +is). Each pair keeps its own width whatever the other deltas are. So over k changed pairs (`n_changed`) p cannot go below 2/2^k: at 5 or fewer no data whatsoever reaches 0.05. The test runs from 2 pairs and its p is printed. An arm under that floor can never be significant, so it stays out of the multiplier, as a `repo compare` row does: it counts in `family_size` but not in diff --git a/dokimi/dokimi/eval/paired_cluster.py b/dokimi/dokimi/eval/paired_cluster.py index 684c719a9..efceb657e 100644 --- a/dokimi/dokimi/eval/paired_cluster.py +++ b/dokimi/dokimi/eval/paired_cluster.py @@ -35,7 +35,6 @@ from collections.abc import Hashable, Sequence from typing import Any, Dict, List, Optional, Tuple -from dokimi.eval.sign_flip import EPS, tie_tolerance from dokimi.eval.stats import bootstrap_ci #: Fewest clusters with a usable pair before an interval is reported at all. @@ -44,9 +43,8 @@ #: Bootstrap resamples; the same default as :func:`dokimi.eval.stats.bootstrap_ci`. DEFAULT_RESAMPLES = 10_000 -#: Tolerance for comparing a bound with ``-margin``, so float noise does not flip a verdict; the cap of -#: :func:`~dokimi.eval.sign_flip.tie_tolerance`, which scales it down for bounds that are all below it. -_EPS = EPS +#: Tolerance for comparing a bound with ``-margin``, so float noise does not flip a verdict. +_EPS = 1e-9 #: Display precision of the delta and its bounds; the status is decided on unrounded values. DISPLAY_DIGITS = 4 @@ -185,10 +183,9 @@ def _verdict(lo: float, hi: float, margin: Optional[float]) -> Optional[str]: """Non-inferiority status of the interval ``[lo, hi]`` against ``-margin``.""" if margin is None: return None - eps = tie_tolerance([lo, hi, margin]) # float slack at the scale of the bounds: _EPS, less below it - if hi < -margin - eps: + if hi < -margin - _EPS: return "inferior" # A zero-width interval has no spread estimate, so it cannot show non-inferiority. - if lo > -margin + eps and hi - lo > eps: + if lo > -margin + _EPS and hi - lo > _EPS: return "noninferior" return "inconclusive" diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index f55eb31a4..743bbce64 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -4,13 +4,14 @@ """The paired sign-flip test and the 95% interval that inverts it, shared by ``repo compare`` and the arms track. :func:`sign_flip_p` is the two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null, -over the deltas not within :func:`tie_tolerance` of 0 (:data:`EPS`, or less for a vector below it): exact up to :data:`EXACT_MAX` of them, Monte Carlo above. +over the deltas not within their tie width of 0 (:data:`EPS`, or :func:`tie_widths` of the pairs' values): exact up to :data:`EXACT_MAX` of them, Monte Carlo above. :func:`min_sign_flip_p` is the floor it cannot go below over k changed deltas, ``2 / 2**k`` exact. ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as -:func:`sign_flip_p` computes it on them: a shifted delta within :data:`EPS` of 0 is a tie, so a delta tied -at 0 counts again once ``theta`` moves off 0. Exact, a sign pattern counts when its sum falls short of the observed -one by at most :data:`EPS`, the observed sum taken as the least of its exactly rounded sum and the product's sums for +:func:`sign_flip_p` computes it on them: a shifted delta within the tie width of its pair (:func:`tie_widths`, kept +from the original deltas) of 0 is a tie, so a delta tied at 0 counts again once ``theta`` moves off 0. Exact, a sign +pattern counts when its sum falls short of the observed one by at most the largest tie width (:data:`EPS` at most), +the observed sum taken as the least of its exactly rounded sum and the product's sums for the observed signs and their mirror: from deltas of about 1e6 on the two round apart by more than :data:`EPS`, and the observed pattern would not count itself. The observed sum and the mean the bisection starts from are :func:`math.fsum`'s, not ``sum``'s, which is compensated from Python 3.12 on and adds left to right before: so the @@ -53,32 +54,43 @@ EXACT_MAX = 16 #: Resamples of the Monte Carlo sign-flip test above :data:`EXACT_MAX`. N_RESAMPLES = 20_000 -#: Deltas closer to zero than this are ties: the cap of :func:`tie_tolerance`, which a vector whose largest -#: ``|delta|`` is below it scales down. +#: Deltas closer to zero than this are ties; for a pair whose values are below 1, ``EPS`` times the larger of them +#: (:func:`tie_widths`). EPS = 1e-9 #: How close a bisected end comes to the step of ``p(theta)`` it stands for: far below the 3 printed decimals. TOL = 1e-7 #: The interval's method name: ``delta.method`` of a ``repo compare`` row, ``ci_method`` of an arms result. METHOD = "sign_flip_inversion" +#: A vector whose values are all below this is searched on a copy multiplied by a power of two (exact), so the +#: tie widths, the bisection step and the sums stay clear of the subnormal range. +_TINY = 2.0 ** -664 -def tie_scale(deltas: Sequence[float]) -> float: - """The scale a vector of deltas is judged at: 1.0 unless its largest finite ``|delta|`` is below :data:`EPS` - (and not 0), where it is that largest ``|delta|``. A vector of scores keeps the absolute :data:`EPS`.""" - largest = max((abs(x) for x in deltas if math.isfinite(x)), default=0.0) - return largest if 0.0 < largest < EPS else 1.0 +def pair_magnitudes(treatment: Sequence[float], baseline: Sequence[float]) -> list[float]: + """The larger of each pair's two values in magnitude: the scale of the float noise in that pair's delta.""" + return [max(abs(t), abs(b)) for t, b in zip(treatment, baseline, strict=True)] -def tie_tolerance(deltas: Sequence[float]) -> float: - """The width of a tie for this vector: a delta is a tie when ``abs(x) <= tie_tolerance(deltas)``. +def tie_scale(magnitudes: Sequence[float] | None) -> float: + """The scale the vector's values are read at: the largest finite magnitude, capped at 1. 1.0 where there is none + to read (no magnitudes given, or all of them 0 or not finite).""" + largest = max((m for m in magnitudes or () if math.isfinite(m)), default=0.0) + return min(1.0, largest) if largest > 0.0 else 1.0 - :data:`EPS` (1e-9, the float noise on a score near 1) for a vector whose largest finite ``|delta|`` is - :data:`EPS` or more, so every result at score scale is the same as with the absolute rule; ``EPS`` times that - largest ``|delta|`` below it, so a vector that varies at 1e-300 ties only its own float noise. The scale is the - vector's, not a pair's. An all-zero vector (or none finite) keeps :data:`EPS`: every delta of it is a tie. - The interval shifts the deltas by theta and takes the tolerance of the original vector, not of the shifted one. + +def tie_widths(magnitudes: Sequence[float] | None, n: int, k: int = 0) -> list[float]: + """The width of a tie for each of ``n`` deltas: a delta is a tie when ``abs(delta) <= width``, then 0. + + Float noise in ``t - b`` scales with the values subtracted, not with the delta, so a pair with the larger value + ``m`` (``magnitudes``) ties within ``EPS * min(1, m)``: ``EPS`` for values of 1 or more, as a score near 1 has it, + and less below that. Each pair keeps its own width whatever the other deltas are. A pair whose values are 0 (or + not finite) takes the width of :func:`tie_scale`. Without ``magnitudes`` every width is the absolute ``EPS``. + ``k`` multiplies the widths by ``2**-k``, as the search on scaled deltas needs. """ - return EPS * tie_scale(deltas) + unit = tie_scale(magnitudes) + if magnitudes is None: + return [EPS * math.ldexp(1.0, -k)] * n + return [EPS * math.ldexp(min(1.0, m) if math.isfinite(m) and m > 0.0 else unit, -k) for m in magnitudes] def min_sign_flip_p(changed: int) -> float: @@ -88,20 +100,25 @@ def min_sign_flip_p(changed: int) -> float: def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, - exact: bool = False) -> float | None: + exact: bool = False, magnitudes: Sequence[float] | None = None) -> float | None: """Two-sided p of ``|sum(deltas)|`` when each delta's sign is a fair coin under the null. ``None`` when every delta is a tie: there is no evidence either way, and 1.0 would read as a measured null. Rounded to 6 places as printed, unless ``exact``. Deltas whose largest magnitude is above ``max float / (8 * n)`` - are divided by a power of two (exact) first, as in :func:`sign_flip_interval`, so no sum leaves the float range. + (or whose values are all below 2**-664) are scaled by a power of two (exact) first, as in + :func:`sign_flip_interval`, so no sum leaves the float range. + + ``magnitudes`` holds the larger value of each pair (:func:`pair_magnitudes`), which sets each tie width + (:func:`tie_widths`); without it a tie is within :data:`EPS` of 0. """ - eps = tie_tolerance(deltas) - d = [x for x in deltas if abs(x) > eps] + d = [x for x, w in zip(deltas, tie_widths(magnitudes, len(deltas)), strict=True) if abs(x) > w] if not d: return None - k = _scale_exponent(d) + unit = tie_scale(magnitudes) + k = _scale_exponent(d, unit) + eps = EPS * math.ldexp(unit, -k) if k: # near the float maximum the sums go to inf/nan: the same p on the deltas divided by a power of two - d, eps = [math.ldexp(x, -k) for x in d], math.ldexp(eps, -k) + d = [math.ldexp(x, -k) for x in d] observed = abs(sum(d)) - eps if len(d) <= EXACT_MAX: hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed @@ -148,14 +165,15 @@ class ShiftTest: row's own p, not ``p(0)``, places 0 (:func:`sign_flip_interval`).""" def __init__(self, deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPLES, *, - eps: float | None = None) -> None: + eps: float | Sequence[float] = EPS) -> None: import numpy as np - if eps is None: - eps = tie_tolerance(deltas) - changed = [x for x in deltas if abs(x) > eps] - self.d = np.array(changed + [x for x in deltas if abs(x) <= eps], dtype=float) # sign_flip_p's order first - self.changed, self.seed, self.n_resamples, self.eps = len(changed), seed, n_resamples, eps + widths = list(eps) if isinstance(eps, Sequence) else [eps] * len(deltas) # a tie width per delta + order = sorted(range(len(deltas)), key=lambda i: abs(deltas[i]) <= widths[i]) # changed first, stably + self.changed = len([x for x, w in zip(deltas, widths, strict=True) if abs(x) > w]) # sign_flip_p's order first + self.d = np.array([deltas[i] for i in order], dtype=float) + self.widths = np.array([widths[i] for i in order], dtype=float) + self.seed, self.n_resamples, self.eps = seed, n_resamples, max(widths, default=EPS) # eps: slack on a sum def p(self, theta: float) -> float | None: """``None`` when every shifted delta ties, as :func:`sign_flip_p` has no p then.""" @@ -163,7 +181,7 @@ def p(self, theta: float) -> float | None: x = self.d - theta eps = self.eps - x[np.abs(x) <= eps] = 0.0 + x[np.abs(x) <= self.widths] = 0.0 moved = x[x != 0.0] if not moved.size: return None @@ -177,7 +195,8 @@ def p(self, theta: float) -> float | None: def sign_flip_interval(deltas: list[float], p_zero: float | None, seed: int = 0, *, - rounded: bool = True) -> tuple[float | None, float | None]: + rounded: bool = True, magnitudes: Sequence[float] | None = None + ) -> tuple[float | None, float | None]: """``(lo, hi)`` of one row (module docstring), ``None`` for an open end; rounded to 4 places unless not ``rounded``, even for an exact p: an end is only known within :data:`TOL`, so its further digits are noise. @@ -192,20 +211,35 @@ def sign_flip_interval(deltas: list[float], p_zero: float | None, seed: int = 0, sums out of the float range: the search then runs on the deltas divided by a power of two (exact), with :data:`EPS` and :data:`TOL` divided the same, and each end is multiplied back. An end that comes back past the float maximum is the maximum (``±max``), never ``inf``: from 6 deltas on a shift past it puts every - delta on one side, which the test rejects, so the search overshoots a true end at the maximum by a few ulps.""" - k, unit = _scale_exponent(deltas), tie_scale(deltas) # the tie width and the search step are the vector's + delta on one side, which the test rejects, so the search overshoots a true end at the maximum by a few ulps. + + ``magnitudes`` (:func:`pair_magnitudes`) set the tie widths (:func:`tie_widths`) and the bisection step, which + scales with the largest of them below 1. Values all below 2**-664 are searched multiplied by a power of two, as + huge ones are divided, so their widths and the step do not underflow; an end that comes back below the smallest + float is that float, on its side of 0, never 0, since the end is off 0 whenever the row's p is below + :data:`ALPHA`.""" + unit = tie_scale(magnitudes) + k = _scale_exponent(deltas, unit) + scaled_unit = math.ldexp(unit, -k) + widths = tie_widths(magnitudes, len(deltas), k) + far_unit = 1.0 if k > 0 else scaled_unit # the far bound is 1 past the extreme, or the scale of tiny values if k: scaled = [math.ldexp(x, -k) for x in deltas] - ends = _ends(scaled, p_zero, seed, math.ldexp(EPS * unit, -k), math.ldexp(TOL * unit, -k), unit) + ends = _ends(scaled, p_zero, seed, widths, TOL * scaled_unit, far_unit) ends = (_scaled_back(ends[0], k), _scaled_back(ends[1], k)) else: - ends = _ends(deltas, p_zero, seed, EPS * unit, TOL * unit, unit) + ends = _ends(deltas, p_zero, seed, widths, TOL * scaled_unit, far_unit) return (_rounded(ends[0]), _rounded(ends[1])) if rounded else ends -def _scale_exponent(deltas: list[float]) -> int: - """``k`` such that ``deltas / 2**k`` stays clear of the float maximum in the search; 0 when they already do.""" - largest, limit = max(map(abs, deltas), default=0.0),sys.float_info.max / (8 * len(deltas)) +def _scale_exponent(deltas: list[float], unit: float = 1.0) -> int: + """``k`` such that ``deltas / 2**k`` stays clear of the float limits in the search; 0 when they already do. + + A ``unit`` (the scale of the values, :func:`tie_scale`) below :data:`_TINY` makes ``k`` negative, bringing it + into ``[0.5, 1)``.""" + if 0.0 < unit < _TINY: + return math.frexp(unit)[1] + largest, limit = max(map(abs, deltas), default=0.0), sys.float_info.max / (8 * len(deltas)) if not math.isfinite(largest) or largest <= limit: return 0 return math.frexp(largest)[1] - math.frexp(limit)[1] + 1 @@ -215,18 +249,19 @@ def _scaled_back(end: float | None, k: int) -> float | None: if end is None: return None try: - return math.ldexp(end, k) + out = math.ldexp(end, k) + return math.copysign(5e-324, end) if out == 0.0 and end != 0.0 else out # the smallest float, not 0 except OverflowError: # a shift past the maximum puts every delta on one side and is rejected, so no true end is past it: the # bisection stops a few ulps beyond an end at the maximum, where the sums cannot resolve TOL return math.copysign(sys.float_info.max, end) -def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float, tol: float, unit: float = 1.0 - ) -> tuple[float | None, float | None]: +def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float | Sequence[float], tol: float, + unit: float = 1.0) -> tuple[float | None, float | None]: test, mean = ShiftTest(deltas, seed, eps=eps), math.fsum(deltas) / len(deltas) zero_inside, low, high = p_zero is None or p_zero >= ALPHA, min(*deltas, 0.0), max(*deltas, 0.0) - # far is one unit (1, or the scale of a vector below EPS) past the extreme, or twice it from one unit on: + # far is one unit (1, or the scale of tiny values) past the extreme, or twice it from one unit on: # 1e16 + 1.0 rounds back onto 1e16, all tied return (_end(test, mean, low - max(unit, -low), zero_inside, tol), _end(test, mean, high + max(unit, high), zero_inside, tol)) @@ -259,19 +294,21 @@ def _rounded(end: float | None) -> float | None: return math.copysign(1e-4, end) if out == 0 and end != 0 else out + 0.0 # + 0.0: no -0.0 -def sign_flip_row(deltas: list[float], seed: int = 0, *, exact: bool) -> dict: +def sign_flip_row(deltas: list[float], seed: int = 0, *, exact: bool, + magnitudes: Sequence[float] | None = None) -> dict: """The test of one row, as both row builders print it: :func:`eval.stats._describe_deltas` (arms) and :func:`repos.compare.sign_flip_summary` (repo compare). They shape their own keys around this. ``p`` is :func:`sign_flip_p` (rounded to 6 places unless ``exact``, as ``repo compare`` prints it). ``lo``/``hi`` are the unrounded ends of the interval that inverts it, and ``ci_lo``/``ci_hi`` the same ends as printed - (:func:`sign_flip_interval`, rounded). ``n_changed`` counts the deltas not within :func:`tie_tolerance` of 0. + (:func:`sign_flip_interval`, rounded). ``n_changed`` counts the deltas not within their :func:`tie_widths` of 0. + ``magnitudes`` is :func:`pair_magnitudes` of the pairs the deltas come from; without it a tie is within + :data:`EPS` of 0. """ - p = sign_flip_p(deltas, seed=seed, exact=exact) - lo, hi = sign_flip_interval(deltas, p, seed=seed, rounded=False) - tol = tie_tolerance(deltas) + p = sign_flip_p(deltas, seed=seed, exact=exact, magnitudes=magnitudes) + lo, hi = sign_flip_interval(deltas, p, seed=seed, rounded=False, magnitudes=magnitudes) return {"p": p, "lo": lo, "hi": hi, "ci_lo": _rounded(lo), "ci_hi": _rounded(hi), - "n_changed": sum(abs(d) > tol for d in deltas)} + "n_changed": sum(abs(d) > w for d, w in zip(deltas, tie_widths(magnitudes, len(deltas)), strict=True))} def round_mean(mean: float) -> float: diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 8eed18fd7..6a723e2bf 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -65,14 +65,14 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple from dokimi.eval.finite_delta import check_sum_range, checked_delta -from dokimi.eval.sign_flip import ( # noqa: F401 (EPS stays importable from here) +from dokimi.eval.sign_flip import ( ALPHA, - EPS, METHOD, min_sign_flip_p, + pair_magnitudes, round_mean, sign_flip_row, - tie_tolerance, + tie_widths, within, ) from dokimi.records import BASELINE, CONTEXT, RecordLike, records @@ -574,13 +574,15 @@ def _cohens_d(d_mean: float, d_sd: float) -> Optional[float]: def _describe_deltas( - deltas: Sequence[float], *, confidence: float, seed: int + deltas: Sequence[float], *, confidence: float, seed: int, magnitudes: Optional[Sequence[float]] = None ) -> Dict[str, Any]: """Effect size, the sign-flip p, and the interval that inverts it, for a delta vector. - ``p`` is the two-sided sign-flip p over the deltas not within ``tie_tolerance`` of 0 + ``p`` is the two-sided sign-flip p over the deltas not within ``EPS`` of 0 (:func:`dokimi.eval.sign_flip.sign_flip_p`, exact up to 16 of them, Monte - Carlo from ``seed`` above), left unrounded for the caller to correct and + Carlo from ``seed`` above), or not within ``EPS`` times the larger value of + their pair when ``magnitudes`` (:func:`dokimi.eval.sign_flip.pair_magnitudes`) + are given and below 1, left unrounded for the caller to correct and then :func:`_finalize`. Correcting an already-rounded p multiplies its quantization error by the family size: a true p of 0.016666 in a family of three gives an exact Holm p of 0.049998 (significant), but @@ -613,12 +615,11 @@ def _describe_deltas( # The sign-flip sums would overflow and read as a measured zero effect. # A finite sum of magnitudes bounds the spread (sd**2 <= sum(x**2) <= that sum squared), so it is finite too. check_sum_range(d) - row = sign_flip_row(d, seed=seed, exact=True) + row = sign_flip_row(d, seed=seed, exact=True, magnitudes=magnitudes) p, lo, hi, shown_lo, shown_hi = row["p"], row["lo"], row["hi"], row["ci_lo"], row["ci_hi"] else: # a NaN delta is no evidence either way: no p, no verdict, open ends p, lo, hi, shown_lo, shown_hi = None, None, None, None, None - tol = tie_tolerance(d) - n_changed = sum(1 for x in d if abs(x) > tol) + n_changed = sum(1 for x, w in zip(d, tie_widths(magnitudes, len(d)), strict=True) if abs(x) > w) return { "n": len(d), @@ -670,7 +671,7 @@ def paired_test( if len(treatment) != len(control) or len(treatment) < 2: return None deltas = _deltas(treatment, control) - result = _describe_deltas(deltas, confidence=confidence, seed=seed) + result = _describe_deltas(deltas, confidence=confidence, seed=seed, magnitudes=pair_magnitudes(treatment, control)) return _finalize(result) if rounded else result @@ -712,6 +713,7 @@ def arm_significance( _deltas(treat, base), confidence=confidence, seed=seed, + magnitudes=pair_magnitudes(treat, base), ) ordered = [a for a in targets if a in out] diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index de8ce7673..71f0c1204 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -111,7 +111,17 @@ from typing import Any from dokimi.context.consistency import Run, run_from_events -from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, sign_flip_row, tie_tolerance, within # noqa: F401 +from dokimi.eval.sign_flip import ( + ALPHA, + EPS, + METHOD, + min_sign_flip_p, + pair_magnitudes, + sign_flip_row, + tie_scale, + tie_widths, + within, +) from dokimi.eval.stats import correct_family from dokimi.eval.finite_delta import checked_delta, checked_mean from dokimi.utils import same_model @@ -491,22 +501,18 @@ def headroom(pairs: list[dict[str, Any]], tops: list[tuple[float | None, float | if failed and any(failed): return None # its 0 is not a measurement, at either end (a score.max of 0 would read it as the top) - # a tie is as wide as the vector it is read on: the pairs' values for a score, their deltas for a move - value_tol = tie_tolerance([v for p in pairs for v in (p["baseline"], p["treatment"])]) - delta_tol = tie_tolerance([p["treatment"] - p["baseline"] for p in pairs]) - def at_top(value: float, top: float | None) -> bool: - return top is not None and value >= top - value_tol + return top is not None and value >= top - EPS # score.max is what the oracle must reach, not a bound: a pair that moved past it is an effect, not a top - base_top = [at_top(p["baseline"], b) and abs(p["treatment"] - p["baseline"]) <= delta_tol + base_top = [at_top(p["baseline"], b) and abs(p["treatment"] - p["baseline"]) <= EPS for p, (b, _) in zip(pairs, tops, strict=True)] if all(base_top) and all(at_top(p["treatment"], t) for p, (_, t) in zip(pairs, tops, strict=True)): return "ceiling" - if all(abs(p["baseline"]) <= value_tol and abs(p["treatment"]) <= value_tol for p in pairs): + if all(abs(p["baseline"]) <= EPS and abs(p["treatment"]) <= EPS for p in pairs): return "floor" if (len(pairs) >= 2 and base_top.count(False) <= 1 - and all(p["treatment"] >= p["baseline"] - delta_tol for p in pairs)): + and all(p["treatment"] >= p["baseline"] - EPS for p in pairs)): return "near_ceiling" return None @@ -584,15 +590,16 @@ def sign_flip_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: Holm's p is known. ``measure`` is not read: every measure gets the same statistics. """ deltas = [p["delta"] for p in pairs] + magnitudes = pair_magnitudes([p["treatment"] for p in pairs], [p["baseline"] for p in pairs]) + widths = tie_widths(magnitudes, len(deltas)) # a tie is within float noise of the pair's own values mean = checked_mean(deltas, CompareError) try: - row = sign_flip_row(deltas, seed=seed, exact=exact) + row = sign_flip_row(deltas, seed=seed, exact=exact, magnitudes=magnitudes) sd = statistics.stdev(deltas) if len(deltas) > 1 else 0.0 except OverflowError as exc: # finite deltas whose sum or spread leaves the float range: no test to run raise CompareError("the deltas of the tasks are too large to test: a sum or a spread of them leaves the float range") from exc p, lo, hi = row["p"], row["ci_lo"], row["ci_hi"] reachable = min_sign_flip_p(row["n_changed"]) < ALPHA # else no effect can be detected - tol = tie_tolerance(deltas) return { "n_tasks": len(pairs), "baseline_mean": _round(_mean([p["baseline"] for p in pairs])), @@ -602,11 +609,11 @@ def sign_flip_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: "p": p, "p_holm": None, "p_holm_floor": None, - "mde": _round(min_detectable(sd, len(deltas))) if sd > tol and reachable else None, + "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS * tie_scale(magnitudes) and reachable else None, "interval_p_disagree": p is not None and includes_zero(lo, hi) == (p < ALPHA), - "up": sum(d > tol for d in deltas), - "down": sum(d < -tol for d in deltas), - "tied": sum(abs(d) <= tol for d in deltas), + "up": sum(d > w for d, w in zip(deltas, widths, strict=True)), + "down": sum(d < -w for d, w in zip(deltas, widths, strict=True)), + "tied": sum(abs(d) <= w for d, w in zip(deltas, widths, strict=True)), "tasks": [{**p, "baseline": _round(p["baseline"]), "treatment": _round(p["treatment"]), "delta": _round(p["delta"])} for p in pairs], } @@ -631,8 +638,8 @@ def format_compare(summary: dict[str, Any]) -> str: f"{r['up']}/{r['down']}/{r['tied']}") for r in rows: if r["measure"] in PRIMARY: - tol = tie_tolerance([t["delta"] for t in r["tasks"]]) - moved = [t for t in r["tasks"] if abs(t["delta"]) > tol] + # the tasks hold rounded values: this line follows the printed deltas, which are 0 or 1e-6 and more + moved = [t for t in r["tasks"] if abs(t["delta"]) > EPS] if moved: lines.append(f" {r['measure']} changed: " + ", ".join( f"{t['task']} {t['delta']:+.2f}" for t in sorted(moved, key=lambda t: t["delta"]))) diff --git a/dokimi/tests/test_repo_compare.py b/dokimi/tests/test_repo_compare.py index ec8d390db..af313212b 100644 --- a/dokimi/tests/test_repo_compare.py +++ b/dokimi/tests/test_repo_compare.py @@ -103,10 +103,7 @@ def test_rates_of_three_reps_give_the_exact_p_end_to_end(tmp_path): def test_sign_flip_has_no_p_when_every_task_tied(): - assert sign_flip_p([0.0, 0.0, 0.0]) is None - # the 1e-12 is the vector's own scale, not float noise on a score: a change, with a p over that one delta - assert sign_flip_p([0.0, 0.0, 1e-12]) == 1.0 - assert sign_flip_p([1.0] * 6 + [1e-12]) == 0.03125 # beside deltas of 1.0 it is the noise it was: 6 changed + assert sign_flip_p([0.0, 0.0, 1e-12]) is None def test_sign_flip_monte_carlo_counts_the_observed_pattern_and_follows_its_seed(): diff --git a/dokimi/tests/test_repo_compare_interval.py b/dokimi/tests/test_repo_compare_interval.py index 2d14feda2..0fa4f2c86 100644 --- a/dokimi/tests/test_repo_compare_interval.py +++ b/dokimi/tests/test_repo_compare_interval.py @@ -120,11 +120,8 @@ def test_p_at_zero_is_the_sign_flip_p_exact_and_monte_carlo(): assert ShiftTest(d, seed).p(0.0) == sign_flip_p(d, seed=seed, exact=True), (d, seed) near_tie = [1.0] * 16 + [1e-12] # within EPS of 0: a tie, so 16 changed and still exact, as sign_flip_p has it assert ShiftTest(near_tie).p(0.0) == sign_flip_p(near_tie, exact=True) == 2 / 2 ** 16 - tied = [0.0] * 3 # every delta tied: no p at 0, nor where a shift ties every delta again + tied = [0.0, 1e-12, -1e-12] # every delta tied: no p at 0, nor where a shift ties every delta again assert ShiftTest(tied).p(0.0) is sign_flip_p(tied) is None and ShiftTest([2.0] * 3).p(2.0) is None - # deltas of 1e-12 are no ties of their own vector (largest 1e-12): they cancel to a p of 1, not to no p - small = [0.0, 1e-12, -1e-12] - assert ShiftTest(small).p(0.0) == sign_flip_p(small, exact=True) == 1.0 @pytest.mark.parametrize(("summed", "p", "lo"), [(math.fsum, 0.0625, 0.0), (_left_to_right, 0.046875, 1e-4)], diff --git a/dokimi/tests/test_sign_flip_relative_tie.py b/dokimi/tests/test_sign_flip_relative_tie.py index 0d1e5b8b4..2e127f163 100644 --- a/dokimi/tests/test_sign_flip_relative_tie.py +++ b/dokimi/tests/test_sign_flip_relative_tie.py @@ -1,35 +1,65 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""The sign-flip tie width is the delta vector's (dokimi.eval.sign_flip.tie_tolerance), capped at EPS (issue #762).""" +"""The sign-flip tie width follows the values of each pair (dokimi.eval.sign_flip.tie_widths), capped at EPS (issue #762).""" from __future__ import annotations import pytest from dokimi.eval import stats -from dokimi.eval.paired_cluster import paired_cluster_difference -from dokimi.eval.sign_flip import ALPHA, EPS, ShiftTest, sign_flip_interval, sign_flip_p, sign_flip_row, tie_tolerance -from dokimi.repos.compare import headroom, sign_flip_summary +from dokimi.eval.sign_flip import ( + ALPHA, + EPS, + ShiftTest, + pair_magnitudes, + sign_flip_interval, + sign_flip_p, + sign_flip_row, + tie_scale, + tie_widths, +) +from dokimi.repos.compare import sign_flip_summary GAINS = [2, 4, 3, 5, 6, 9, 7, 8] +SMALLEST = 5e-324 # the smallest subnormal float -def _pairs(deltas: list[float]) -> list[dict]: - return [{"task": f"t{i}", "baseline": 0.0, "treatment": d, "delta": d} for i, d in enumerate(deltas)] +def _pairs(treatment: list[float], baseline: list[float]) -> list[dict]: + return [{"task": f"t{i}", "baseline": b, "treatment": t, "delta": t - b} + for i, (t, b) in enumerate(zip(treatment, baseline, strict=True))] def _includes_zero(lo: float | None, hi: float | None) -> bool: return (lo is None or lo <= 0) and (hi is None or hi >= 0) -def test_tie_tolerance_is_eps_from_a_largest_delta_of_eps_up_and_scales_below_it(): - assert tie_tolerance([1.0, 1e-12]) == tie_tolerance([0.5, 0.2]) == tie_tolerance([EPS, 0.0]) == EPS - assert tie_tolerance([1e-12, -4e-12, 2e-12]) == pytest.approx(EPS * 4e-12) - assert tie_tolerance([1e-300]) == pytest.approx(EPS * 1e-300) - # nothing to scale by: every delta of such a vector is a tie under EPS, as before - assert tie_tolerance([0.0, 0.0]) == tie_tolerance([]) == tie_tolerance([float("nan"), float("inf")]) == EPS - assert tie_tolerance([3e-12, float("inf"), float("nan")]) == pytest.approx(EPS * 3e-12) # non-finite: not a scale +def test_a_tie_is_eps_times_the_larger_value_of_its_pair_up_to_one_and_eps_without_values(): + assert tie_widths(None, 3) == [EPS] * 3 # deltas alone: the absolute width + assert tie_widths([1.0, 7.0, 1e300], 3) == [EPS] * 3 # values of 1 or more: float noise near 1, as ever + assert tie_widths([4e-12, 1e-300, 0.5], 3) == pytest.approx([EPS * 4e-12, EPS * 1e-300, EPS * 0.5], rel=1e-6, abs=0) + # a pair of zeros, or of values that are not finite, takes the scale of the others: the vector's largest value + assert tie_widths([0.0, 2e-12], 2) == pytest.approx([EPS * 2e-12] * 2, rel=1e-6, abs=0) + assert tie_widths([float("nan"), float("inf"), 3e-12], 3) == pytest.approx([EPS * 3e-12] * 3, rel=1e-6, abs=0) + assert tie_widths([0.0, float("nan")], 2) == [EPS] * 2 + assert tie_scale([1e-12, 4e-12, 2e-12]) == 4e-12 and tie_scale([0.0]) == tie_scale(None) == tie_scale([5.0]) == 1.0 + assert pair_magnitudes([0.1, -3.0], [-0.2, 1.0]) == [0.2, 3.0] + + +def test_deltas_that_are_float_noise_on_the_values_subtracted_are_ties_not_a_significant_change(): + # 0.1 + 0.2 and 0.3 differ in the last place: the same scores in another order, not a change + noise = stats.paired_test([0.1 + 0.2] * 8, [0.3] * 8) + assert (noise["p"], noise["n_changed"], noise["significant"]) == (None, 0, None) + summary = sign_flip_summary(_pairs([0.1 + 0.2] * 8, [0.3] * 8), exact=True) + assert (summary["up"], summary["down"], summary["tied"], summary["p"]) == (0, 0, 8, None) + # the same rounding residue next to a score of 1 is still a tie + assert sign_flip_p([1.1e-16] * 8, magnitudes=[1.0] * 8) is None + # a change of the same size at the scale of its own values is not + real = sign_flip_summary(_pairs([1.1e-16] * 8, [0.0] * 8), exact=True) + assert (real["up"], real["down"], real["tied"]) == (8, 0, 0) and real["p"] == 0.0078125 + # a spread at the 1e-300 scale is a spread, so the row has a minimum detectable effect + tiny = sign_flip_summary(_pairs([g * 1e-300 for g in GAINS], [1e-300] * 8), exact=True) + assert (tiny["up"], tiny["down"], tiny["tied"]) == (8, 0, 0) and tiny["mde"] is not None def test_deltas_at_1e_300_get_the_p_and_the_verdict_of_the_same_deltas_at_1e_3(): @@ -37,68 +67,70 @@ def test_deltas_at_1e_300_get_the_p_and_the_verdict_of_the_same_deltas_at_1e_3() usual = stats.paired_test([g * 1e-3 for g in GAINS], [1e-3] * 8) assert (tiny["p"], tiny["n_changed"], tiny["significant"]) == (usual["p"], usual["n_changed"], usual["significant"]) assert (tiny["p"], tiny["n_changed"], tiny["significant"]) == (0.0078, 8, True) + # a vector of zeros is still all ties, with no p + zeros = stats.paired_test([0.0] * 8, [0.0] * 8) + assert (zeros["p"], zeros["n_changed"]) == (None, 0) + + +def test_more_evidence_in_the_same_direction_does_not_remove_a_significant_result(): + # the width is each pair's own: a ninth delta of 1.1e-9 no longer widens the tie of the eight 5e-10 + eight = stats.paired_test([5e-10] * 8, [0.0] * 8) + nine = stats.paired_test([5e-10] * 8 + [1.1e-9], [0.0] * 9) + assert (eight["p"], eight["n_changed"]) == (0.0078, 8) + assert (nine["p"], nine["n_changed"]) == (0.0039, 9) + assert sign_flip_p([5e-10] * 8 + [1.1e-9], exact=True, magnitudes=[5e-10] * 8 + [1.1e-9]) == 2 / 2 ** 9 def test_the_interval_and_the_p_agree_at_a_tiny_scale_and_on_a_mixed_vector(): + def row_of(treatment: list[float], baseline: list[float]) -> dict: + deltas = [t - b for t, b in zip(treatment, baseline, strict=True)] + return sign_flip_row(deltas, exact=True, magnitudes=pair_magnitudes(treatment, baseline)) + tiny = [g * 1e-300 for g in GAINS] - mixed = [1e-3, 2e-3, 1.5e-3, 3e-3, 2.5e-3, 1.2e-3, 1e-12] # the 1e-12 stays a tie under the 1e-9 cap + mixed = [1e-3, 2e-3, 1.5e-3, 3e-3, 2.5e-3, 1.2e-3, 1e-12] small = [-1e-12, 1e-12, 3e-12, 2e-12, 5e-13, 4e-12, -2e-12] - for deltas in (tiny, mixed, small): - row = sign_flip_row(deltas, exact=True) + for treatment in (tiny, mixed, small): + row = row_of(treatment, [0.0] * len(treatment)) assert row["p"] is not None - assert _includes_zero(row["lo"], row["hi"]) == (row["p"] >= ALPHA), deltas - assert sign_flip_row(mixed, exact=True)["n_changed"] == 6 - assert sign_flip_row(tiny, exact=True)["n_changed"] == 8 + assert _includes_zero(row["lo"], row["hi"]) == (row["p"] >= ALPHA), treatment + assert row_of(tiny, [0.0] * 8)["n_changed"] == 8 + # the 1e-12 beside 1e-3 values is a change of its own (width 1e-21), as a pair whose values are that small + assert row_of(mixed, [0.0] * 7)["n_changed"] == 7 # the ends are step points of the one p(theta) of the original vector, at its own scale - lo, hi = sign_flip_interval(tiny, sign_flip_p(tiny), rounded=False) - test = ShiftTest(tiny) + magnitudes = pair_magnitudes(tiny, [0.0] * 8) + lo, hi = sign_flip_interval(tiny, sign_flip_p(tiny, magnitudes=magnitudes), rounded=False, magnitudes=magnitudes) + test = ShiftTest(tiny, eps=tie_widths(magnitudes, 8)) step = 1e-6 * 1e-300 assert 0 < lo < hi assert test.p(lo) >= ALPHA > test.p(lo - step) and test.p(hi) >= ALPHA > test.p(hi + step) -def test_the_shifted_deltas_keep_the_tolerance_of_the_original_vector(): - # shifted by 1.0 the deltas are 1e-10 off 0, and 1e-10 is a tie of the original (largest 1.0002): no p, not a p - # of a vector that would be read at its own 1e-10 scale +def test_deltas_of_the_smallest_floats_have_an_interval_that_excludes_zero_with_their_significant_p(): + treatment = [SMALLEST] * 6 + [0.0] * 7 + row = sign_flip_row(treatment, exact=True, magnitudes=pair_magnitudes(treatment, [0.0] * 13)) + assert row["n_changed"] == 6 and row["p"] == 0.03125 < ALPHA + assert not _includes_zero(row["lo"], row["hi"]) # the ends are not rounded onto 0 by the float range + assert 0 < row["lo"] <= row["hi"] <= 1e-322 + # 6 changed pairs of -SMALLEST are the mirror image: both ends below 0 + mirror = [-x for x in treatment] + row = sign_flip_row(mirror, exact=True, magnitudes=pair_magnitudes(mirror, [0.0] * 13)) + assert row["hi"] < 0 and row["lo"] <= row["hi"] + # and the arms track reports the same significant row, not a p beside a zero-width interval + result = stats.paired_test(treatment, [0.0] * 13) + assert (result["p"], result["n_changed"], result["significant"]) == (0.0312, 6, True) + assert result["exact"]["ci_lo"] > 0 + + +def test_the_shifted_deltas_keep_the_tie_widths_of_the_original_pairs(): + # shifted by 1.0 the deltas are 1e-10 off 0, and 1e-10 is a tie of the original pairs (values near 1) d = [1.0, 1.0 + 1e-10, 1.0 - 1e-10, 1.0 + 2e-10, 1.0, 1.0] - assert ShiftTest(d).p(1.0) is None - assert sign_flip_p([x - 1.0 for x in d]) is not None # the vector of the shifted values alone is another scale - - -def test_a_one_e_minus_12_scale_vector_is_not_all_ties_but_zeros_still_are(): - assert sign_flip_row([0.0] * 8, exact=True)["p"] is None - assert sign_flip_row([0.0] * 8, exact=True)["n_changed"] == 0 - small = sign_flip_row([g * 1e-12 for g in GAINS], exact=True) - usual = sign_flip_row([g * 1e-3 for g in GAINS], exact=True) - assert (small["p"], small["n_changed"]) == (usual["p"], usual["n_changed"]) == (0.0078125, 8) - - -def test_repo_compare_counts_up_down_and_tied_at_the_scale_of_the_deltas(): - tiny = sign_flip_summary(_pairs([g * 1e-300 for g in GAINS]), exact=True) - assert (tiny["up"], tiny["down"], tiny["tied"]) == (8, 0, 0) - assert tiny["p"] == 0.0078125 and tiny["mde"] is not None - mixed = sign_flip_summary(_pairs([1e-3] * 6 + [1e-12, -1e-12])) - assert (mixed["up"], mixed["down"], mixed["tied"]) == (6, 0, 2) - zeros = sign_flip_summary(_pairs([0.0] * 8)) - assert (zeros["up"], zeros["down"], zeros["tied"], zeros["p"]) == (0, 0, 8, None) - - -def test_headroom_reads_a_floor_only_in_zeros_not_in_values_that_vary_at_a_tiny_scale(): - def pairs(baseline: float, treatment: float) -> list[dict]: - return [{"baseline": baseline, "treatment": treatment} for _ in range(3)] - - tops = [(1.0, 1.0)] * 3 - assert headroom(pairs(0.0, 0.0), tops) == "floor" - assert headroom(pairs(1e-12, 1e-12), tops) is None - - -def test_the_non_inferiority_slack_scales_down_with_bounds_that_are_all_below_it(): - clusters = [f"c{i}" for i in range(12)] - control = [0.0] * 12 - gain = [1e-12 * (1 + i / 10) for i in range(12)] - assert paired_cluster_difference(clusters, gain, control, margin=0.0)["noninferiority"] == "noninferior" - loss = [-g for g in gain] - assert paired_cluster_difference(clusters, loss, control, margin=0.0)["noninferiority"] == "inferior" - # at score scale the 1e-9 slack is kept: a lower bound within it of -margin is not a win - score = [0.0] * 11 + [1.0] - assert paired_cluster_difference(clusters, score, control, margin=0.0)["noninferiority"] == "inconclusive" + assert ShiftTest(d, eps=tie_widths([1.0] * 6, 6)).p(1.0) is None + # a width per delta: at a shift of 1.0 the last delta is 1e-10 off 0, a change under its width of 0, a tie under EPS + narrow = ShiftTest([1.0, 1.0, 1.0 + 1e-10], eps=[EPS, EPS, 0.0]) + assert narrow.changed == 3 and narrow.p(1.0) == 2 / 2 + assert ShiftTest([1.0, 1.0, 1.0 + 1e-10], eps=EPS).p(1.0) is None + + +def test_deltas_alone_keep_the_absolute_tie_width(): + assert sign_flip_p([0.0, 0.0, 1e-12]) is None + assert sign_flip_row([1e-300] * 8, exact=True)["n_changed"] == 0 From aae990b933e3ee08de1d6608719f15577703e81d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 14:07:26 +0000 Subject: [PATCH 3/5] fix(eval): the sign-flip slack on a sum follows the pairs summed, not tied pairs Settles the review point that an unchanged pair of large values loosened the sums of small changed pairs: paired_test([1e-12] * 8 + [1.0], [0.0] * 8 + [1.0]) gave p 1.0, where the eight small pairs alone give 0.0078125. sign_flip_p takes the scale of the slack from the magnitudes of the deltas it sums, and ShiftTest takes it from the tie widths of the deltas that are not ties at that theta, instead of the widest width of every pair. Signed-off-by: Anton Volkov --- ...1116_130000_sign-flip-slack-active-pairs.md | 3 +++ dokimi/dokimi/eval/sign_flip.py | 18 +++++++++++------- dokimi/tests/test_sign_flip_relative_tie.py | 12 ++++++++++++ 3 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 dokimi/changelog.d/20261116_130000_sign-flip-slack-active-pairs.md diff --git a/dokimi/changelog.d/20261116_130000_sign-flip-slack-active-pairs.md b/dokimi/changelog.d/20261116_130000_sign-flip-slack-active-pairs.md new file mode 100644 index 000000000..361c4db56 --- /dev/null +++ b/dokimi/changelog.d/20261116_130000_sign-flip-slack-active-pairs.md @@ -0,0 +1,3 @@ +### Fixed + +- **An unchanged pair of large values no longer widens the sign-flip slack on the sums of small changed pairs** — `paired_test([1e-12] * 8 + [1.0], [0.0] * 8 + [1.0])` gave `p` 1.0 and no significance, where the eight small pairs alone give 0.0078125, because the slack on a pattern's sum followed the largest value of any pair, tied ones included. The slack now follows the pairs that are summed (the changed ones, and at a shift of the interval search the deltas that are not ties there), in the test and in the interval that inverts it. diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index 743bbce64..1ce894e4b 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -10,7 +10,7 @@ ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as :func:`sign_flip_p` computes it on them: a shifted delta within the tie width of its pair (:func:`tie_widths`, kept from the original deltas) of 0 is a tie, so a delta tied at 0 counts again once ``theta`` moves off 0. Exact, a sign -pattern counts when its sum falls short of the observed one by at most the largest tie width (:data:`EPS` at most), +pattern counts when its sum falls short of the observed one by at most the largest tie width among the deltas summed (:data:`EPS` at most), the observed sum taken as the least of its exactly rounded sum and the product's sums for the observed signs and their mirror: from deltas of about 1e6 on the two round apart by more than :data:`EPS`, and the observed pattern would not count itself. The observed sum and the mean the bisection starts from are @@ -111,10 +111,13 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL ``magnitudes`` holds the larger value of each pair (:func:`pair_magnitudes`), which sets each tie width (:func:`tie_widths`); without it a tie is within :data:`EPS` of 0. """ - d = [x for x, w in zip(deltas, tie_widths(magnitudes, len(deltas)), strict=True) if abs(x) > w] - if not d: + widths = tie_widths(magnitudes, len(deltas)) + active = [i for i, (x, w) in enumerate(zip(deltas, widths, strict=True)) if abs(x) > w] + if not active: return None - unit = tie_scale(magnitudes) + d = [deltas[i] for i in active] + # the slack on a sum follows the pairs the sum is of, not a tied pair's larger values + unit = tie_scale(None if magnitudes is None else [magnitudes[i] for i in active]) k = _scale_exponent(d, unit) eps = EPS * math.ldexp(unit, -k) if k: # near the float maximum the sums go to inf/nan: the same p on the deltas divided by a power of two @@ -173,18 +176,19 @@ def __init__(self, deltas: list[float], seed: int = 0, n_resamples: int = N_RESA self.changed = len([x for x, w in zip(deltas, widths, strict=True) if abs(x) > w]) # sign_flip_p's order first self.d = np.array([deltas[i] for i in order], dtype=float) self.widths = np.array([widths[i] for i in order], dtype=float) - self.seed, self.n_resamples, self.eps = seed, n_resamples, max(widths, default=EPS) # eps: slack on a sum + self.seed, self.n_resamples = seed, n_resamples def p(self, theta: float) -> float | None: """``None`` when every shifted delta ties, as :func:`sign_flip_p` has no p then.""" import numpy as np x = self.d - theta - eps = self.eps - x[np.abs(x) <= self.widths] = 0.0 + tied = np.abs(x) <= self.widths + x[tied] = 0.0 moved = x[x != 0.0] if not moved.size: return None + eps = float(self.widths[~tied].max()) # slack on a sum: the widest tie width among the deltas summed observed = abs(math.fsum(moved.tolist())) # exactly rounded: sum() rounds differently before 3.12 if moved.size <= EXACT_MAX: sums = np.abs(_patterns(moved.size) @ moved) # rows 0 and -1: the observed signs and their mirror diff --git a/dokimi/tests/test_sign_flip_relative_tie.py b/dokimi/tests/test_sign_flip_relative_tie.py index 2e127f163..b6655996a 100644 --- a/dokimi/tests/test_sign_flip_relative_tie.py +++ b/dokimi/tests/test_sign_flip_relative_tie.py @@ -134,3 +134,15 @@ def test_the_shifted_deltas_keep_the_tie_widths_of_the_original_pairs(): def test_deltas_alone_keep_the_absolute_tie_width(): assert sign_flip_p([0.0, 0.0, 1e-12]) is None assert sign_flip_row([1e-300] * 8, exact=True)["n_changed"] == 0 + + +def test_an_unchanged_pair_of_large_values_does_not_loosen_the_sums_of_the_small_changed_pairs(): + treatment, baseline = [1e-12] * 8 + [1.0], [0.0] * 8 + [1.0] + deltas, mags = [t - b for t, b in zip(treatment, baseline, strict=True)], pair_magnitudes(treatment, baseline) + alone = sign_flip_p(deltas[:8], magnitudes=mags[:8], exact=True) + assert alone == 0.0078125 and sign_flip_p(deltas, magnitudes=mags, exact=True) == alone + assert stats.paired_test(treatment, baseline)["significant"] is True + lo, hi = sign_flip_interval(deltas, alone, rounded=False, magnitudes=mags) + assert lo is not None and hi is not None and 0 < lo <= hi # p(0) is below ALPHA: the interval excludes 0 + # the shifted test: the unchanged pair is a tie at theta 0, and the sums of the 8 small deltas have the small slack + assert ShiftTest(deltas, eps=tie_widths(mags, 9)).p(0.0) == alone From eb1a92428f40b952a6820f29bbfd5da296cfaab4 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 19:37:06 +0000 Subject: [PATCH 4/5] fix(eval): the sign-flip interval starts from a shift the test accepts Settles the review of #818: eight pairs of 1e-12 beside one tied pair of 1.0 gave an interval at 8.9e-13 at both ends, the mean of the nine deltas, a shift the test rejects. The search returned its starting point unchecked; it now starts from the mean of the changed deltas when the mean of all is rejected, so the ends are 1e-12 as for the eight pairs alone. The docs sentence on tie widths now says a pair of zeros takes the width of the largest value among the pairs. Signed-off-by: Anton Volkov --- ...16_140000_sign-flip-interval-start-accepted.md | 3 +++ dokimi/docs/benchmarking-and-comparison.md | 2 +- dokimi/dokimi/eval/sign_flip.py | 3 +++ dokimi/tests/test_sign_flip_relative_tie.py | 15 +++++++++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 dokimi/changelog.d/20261116_140000_sign-flip-interval-start-accepted.md diff --git a/dokimi/changelog.d/20261116_140000_sign-flip-interval-start-accepted.md b/dokimi/changelog.d/20261116_140000_sign-flip-interval-start-accepted.md new file mode 100644 index 000000000..fb6daeccc --- /dev/null +++ b/dokimi/changelog.d/20261116_140000_sign-flip-interval-start-accepted.md @@ -0,0 +1,3 @@ +### Fixed + +- **The sign-flip interval starts from a shift the test accepts** — eight pairs of `1e-12` beside one tied pair of `1.0` gave an interval of `8.9e-13` at both ends, a shift the test rejects, where the eight pairs alone give `1e-12`. A tied delta pulled the mean off the one accepted shift, so the search returned its starting point unchecked; it now starts from the mean of the changed deltas when the mean of all is rejected. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index fd714a8cb..0688e91e9 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -372,7 +372,7 @@ with a fixed seed above). A delta within 1e-9 of 0 is a tie and carries no sign, unless the values of its pair are below 1: float noise in a delta scales with the values subtracted, so a pair whose larger value is m below 1 ties within 1e-9 times m (a score of 0.3 against 0.1 + 0.2 is a tie; 1e-12 against 0 is a change, as 1e-3 against 0 -is). Each pair keeps its own width whatever the other deltas are. So over k changed pairs (`n_changed`) p cannot go below 2/2^k: at 5 or fewer no +is). Each pair keeps its own width whatever the other deltas are, except a pair of zeros, which takes the width of the largest value among all the pairs (at most 1). So over k changed pairs (`n_changed`) p cannot go below 2/2^k: at 5 or fewer no data whatsoever reaches 0.05. The test runs from 2 pairs and its p is printed. An arm under that floor can never be significant, so it stays out of the multiplier, as a `repo compare` row does: it counts in `family_size` but not in diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index 1ce894e4b..c3e3dcbed 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -264,6 +264,9 @@ def _scaled_back(end: float | None, k: int) -> float | None: def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float | Sequence[float], tol: float, unit: float = 1.0) -> tuple[float | None, float | None]: test, mean = ShiftTest(deltas, seed, eps=eps), math.fsum(deltas) / len(deltas) + if test.changed and not _accepts(test.p(mean)): + # tied deltas pull the mean off the shift the test accepts: start from the mean of the changed ones + mean = math.fsum(test.d[:test.changed].tolist()) / test.changed zero_inside, low, high = p_zero is None or p_zero >= ALPHA, min(*deltas, 0.0), max(*deltas, 0.0) # far is one unit (1, or the scale of tiny values) past the extreme, or twice it from one unit on: # 1e16 + 1.0 rounds back onto 1e16, all tied diff --git a/dokimi/tests/test_sign_flip_relative_tie.py b/dokimi/tests/test_sign_flip_relative_tie.py index b6655996a..12b3c776c 100644 --- a/dokimi/tests/test_sign_flip_relative_tie.py +++ b/dokimi/tests/test_sign_flip_relative_tie.py @@ -146,3 +146,18 @@ def test_an_unchanged_pair_of_large_values_does_not_loosen_the_sums_of_the_small assert lo is not None and hi is not None and 0 < lo <= hi # p(0) is below ALPHA: the interval excludes 0 # the shifted test: the unchanged pair is a tie at theta 0, and the sums of the 8 small deltas have the small slack assert ShiftTest(deltas, eps=tie_widths(mags, 9)).p(0.0) == alone + + +def test_an_unchanged_pair_of_large_values_does_not_pull_the_interval_off_the_shift_the_small_pairs_accept(): + treatment, baseline = [1e-12] * 8 + [1.0], [0.0] * 8 + [1.0] + deltas, mags = [t - b for t, b in zip(treatment, baseline, strict=True)], pair_magnitudes(treatment, baseline) + p = sign_flip_p(deltas, magnitudes=mags, exact=True) + alone = sign_flip_interval(deltas[:8], p, rounded=False, magnitudes=mags[:8]) + assert alone == (1e-12, 1e-12) + # the mean of all nine deltas, 8.9e-13, is a shift the test rejects; the ends are those of the eight pairs alone + assert sign_flip_interval(deltas, p, rounded=False, magnitudes=mags) == alone + + +def test_a_pair_of_zeros_takes_the_width_of_the_largest_value_among_the_pairs(): + assert tie_widths([0.0, 1.0], 2) == [EPS, EPS] + assert tie_widths([0.0, 2e-12], 2) == [EPS * 2e-12, EPS * 2e-12] From a4ad627fbac0ee25f336180fe57148e9aa18bda0 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 20:25:57 +0000 Subject: [PATCH 5/5] fix(eval): the sign-flip slack on a sum is rounding error Settles the three open threads of the review. The slack on a sum of deltas was the widest tie width among them, so a changed pair just past its width (1.0 + 1.1e-9 against 1.0) made every sign pattern count and turned eight significant pairs of 1e-12 into p 1.0. The slack is now the rounding error of adding the deltas (n * 2**-52 times their absolute sum), capped by the tie width it replaces, in sign_flip_p and ShiftTest.p. The interval search started from the mean of the deltas, or of the changed ones, without checking either was accepted: with every delta changed, a mean that ties the largest delta returned two rejected ends. It now tries the mean, the mean of the changed deltas, then up to 16 of the deltas nearest the median, and keeps the first the test accepts. The mde of a repo compare row read the spread against 1e-9 times the largest value of any pair, so an unchanged pair of large values removed it. It now reads the scale of the changed pairs only. Signed-off-by: Anton Volkov --- ...116_150000_sign-flip-sum-slack-rounding.md | 3 ++ dokimi/dokimi/eval/sign_flip.py | 33 +++++++++++++++---- dokimi/dokimi/repos/compare.py | 3 +- dokimi/tests/test_sign_flip_relative_tie.py | 30 +++++++++++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 dokimi/changelog.d/20261116_150000_sign-flip-sum-slack-rounding.md diff --git a/dokimi/changelog.d/20261116_150000_sign-flip-sum-slack-rounding.md b/dokimi/changelog.d/20261116_150000_sign-flip-sum-slack-rounding.md new file mode 100644 index 000000000..137f40859 --- /dev/null +++ b/dokimi/changelog.d/20261116_150000_sign-flip-sum-slack-rounding.md @@ -0,0 +1,3 @@ +### Fixed + +- **The sign-flip slack on a sum is rounding error, not a tie width** — a ninth changed pair just past its tie width (`1.0 + 1.1e-9` against `1.0`) beside eight pairs of `1e-12` gave `p` 1.0, because its width of `1e-9` let every sign pattern count; it now gives `2/2**9`. The interval starts from an accepted shift when the mean of the deltas is rejected (it could return two rejected ends), and the `mde` of `repo compare` reads the spread at the scale of the changed pairs, so an unchanged pair of large values no longer removes it. diff --git a/dokimi/dokimi/eval/sign_flip.py b/dokimi/dokimi/eval/sign_flip.py index c3e3dcbed..4b26af6ee 100644 --- a/dokimi/dokimi/eval/sign_flip.py +++ b/dokimi/dokimi/eval/sign_flip.py @@ -10,7 +10,8 @@ ``p(theta)`` is the two-sided sign-flip p of the shifted deltas ``d - theta`` over every delta, as :func:`sign_flip_p` computes it on them: a shifted delta within the tie width of its pair (:func:`tie_widths`, kept from the original deltas) of 0 is a tie, so a delta tied at 0 counts again once ``theta`` moves off 0. Exact, a sign -pattern counts when its sum falls short of the observed one by at most the largest tie width among the deltas summed (:data:`EPS` at most), +pattern counts when its sum falls short of the observed one by at most the rounding error of adding the deltas summed (:func:`_sum_slack`; :data:`EPS` at most, and no more than the +largest tie width among them), the observed sum taken as the least of its exactly rounded sum and the product's sums for the observed signs and their mirror: from deltas of about 1e6 on the two round apart by more than :data:`EPS`, and the observed pattern would not count itself. The observed sum and the mean the bisection starts from are @@ -41,6 +42,7 @@ import itertools import math import random +import statistics import sys from collections.abc import Sequence from typing import TYPE_CHECKING @@ -93,6 +95,13 @@ def tie_widths(magnitudes: Sequence[float] | None, n: int, k: int = 0) -> list[f return [EPS * math.ldexp(min(1.0, m) if math.isfinite(m) and m > 0.0 else unit, -k) for m in magnitudes] +def _sum_slack(summed: Sequence[float], cap: float) -> float: + """How far two sign patterns' sums may differ and still be the same sum: the rounding error of adding the deltas + in two orders (``n * 2**-52`` times their absolute sum), never more than ``cap``. A tie width, which is the noise + of one pair's subtraction, is no slack on a sum: it would swallow a changed delta of the size of that width.""" + return min(cap, len(summed) * 2.0 ** -52 * math.fsum(abs(x) for x in summed)) + + def min_sign_flip_p(changed: int) -> float: """The smallest p :func:`sign_flip_p` can give over ``changed`` non-zero deltas, every sign the same: only that pattern and its mirror reach the observed ``|sum|``, ``2 / 2**changed`` exact, ``1 / (N + 1)`` Monte Carlo.""" @@ -119,9 +128,9 @@ def sign_flip_p(deltas: list[float], seed: int = 0, n_resamples: int = N_RESAMPL # the slack on a sum follows the pairs the sum is of, not a tied pair's larger values unit = tie_scale(None if magnitudes is None else [magnitudes[i] for i in active]) k = _scale_exponent(d, unit) - eps = EPS * math.ldexp(unit, -k) if k: # near the float maximum the sums go to inf/nan: the same p on the deltas divided by a power of two d = [math.ldexp(x, -k) for x in d] + eps = _sum_slack(d, EPS * math.ldexp(unit, -k)) observed = abs(sum(d)) - eps if len(d) <= EXACT_MAX: hits = sum(abs(sum(s * x for s, x in zip(signs, d, strict=True))) >= observed @@ -188,7 +197,7 @@ def p(self, theta: float) -> float | None: moved = x[x != 0.0] if not moved.size: return None - eps = float(self.widths[~tied].max()) # slack on a sum: the widest tie width among the deltas summed + eps = _sum_slack(moved.tolist(), float(self.widths[~tied].max())) observed = abs(math.fsum(moved.tolist())) # exactly rounded: sum() rounds differently before 3.12 if moved.size <= EXACT_MAX: sums = np.abs(_patterns(moved.size) @ moved) # rows 0 and -1: the observed signs and their mirror @@ -263,10 +272,8 @@ def _scaled_back(end: float | None, k: int) -> float | None: def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float | Sequence[float], tol: float, unit: float = 1.0) -> tuple[float | None, float | None]: - test, mean = ShiftTest(deltas, seed, eps=eps), math.fsum(deltas) / len(deltas) - if test.changed and not _accepts(test.p(mean)): - # tied deltas pull the mean off the shift the test accepts: start from the mean of the changed ones - mean = math.fsum(test.d[:test.changed].tolist()) / test.changed + test = ShiftTest(deltas, seed, eps=eps) + mean = _start(test, math.fsum(deltas) / len(deltas)) zero_inside, low, high = p_zero is None or p_zero >= ALPHA, min(*deltas, 0.0), max(*deltas, 0.0) # far is one unit (1, or the scale of tiny values) past the extreme, or twice it from one unit on: # 1e16 + 1.0 rounds back onto 1e16, all tied @@ -274,6 +281,18 @@ def _ends(deltas: list[float], p_zero: float | None, seed: int, eps: float | Seq _end(test, mean, high + max(unit, high), zero_inside, tol)) +def _start(test: ShiftTest, mean: float) -> float: + """The shift the bisection starts from: ``mean`` if the test accepts it, else the first accepted of the mean of the + changed deltas and of up to :data:`EXACT_MAX` of the deltas, nearest the median first (shifted by a delta, that + delta and the ones within their width of it tie). ``mean`` again if none is, as there is no accepted shift to find.""" + if not test.changed or _accepts(test.p(mean)): + return mean + changed = test.d[:test.changed].tolist() + median = statistics.median(test.d.tolist()) + nearest = sorted(set(test.d.tolist()), key=lambda x: abs(x - median))[:EXACT_MAX] + return next((c for c in [math.fsum(changed) / len(changed), *nearest] if _accepts(test.p(c))), mean) + + def _end(test: ShiftTest, mean: float, far: float, zero_inside: bool, tol: float = TOL) -> float | None: """The end between ``mean`` and ``far``: the last shift bisection finds not rejected; ``None`` if ``far`` is not.""" inside, outside = mean, far diff --git a/dokimi/dokimi/repos/compare.py b/dokimi/dokimi/repos/compare.py index 71f0c1204..c0b47c967 100644 --- a/dokimi/dokimi/repos/compare.py +++ b/dokimi/dokimi/repos/compare.py @@ -592,6 +592,7 @@ def sign_flip_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: deltas = [p["delta"] for p in pairs] magnitudes = pair_magnitudes([p["treatment"] for p in pairs], [p["baseline"] for p in pairs]) widths = tie_widths(magnitudes, len(deltas)) # a tie is within float noise of the pair's own values + changed = [m for m, d, w in zip(magnitudes, deltas, widths, strict=True) if abs(d) > w] # the spread is read at their scale mean = checked_mean(deltas, CompareError) try: row = sign_flip_row(deltas, seed=seed, exact=exact, magnitudes=magnitudes) @@ -609,7 +610,7 @@ def sign_flip_summary(pairs: list[dict[str, Any]], measure: str = "gate", seed: "p": p, "p_holm": None, "p_holm_floor": None, - "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS * tie_scale(magnitudes) and reachable else None, + "mde": _round(min_detectable(sd, len(deltas))) if sd > EPS * tie_scale(changed) and reachable else None, "interval_p_disagree": p is not None and includes_zero(lo, hi) == (p < ALPHA), "up": sum(d > w for d, w in zip(deltas, widths, strict=True)), "down": sum(d < -w for d, w in zip(deltas, widths, strict=True)), diff --git a/dokimi/tests/test_sign_flip_relative_tie.py b/dokimi/tests/test_sign_flip_relative_tie.py index 12b3c776c..6e1c7dec7 100644 --- a/dokimi/tests/test_sign_flip_relative_tie.py +++ b/dokimi/tests/test_sign_flip_relative_tie.py @@ -158,6 +158,36 @@ def test_an_unchanged_pair_of_large_values_does_not_pull_the_interval_off_the_sh assert sign_flip_interval(deltas, p, rounded=False, magnitudes=mags) == alone +def test_a_changed_pair_the_size_of_its_tie_width_does_not_make_every_sign_pattern_count(): + # the ninth delta, 1.1e-9, is a change just past its width of 1e-9: the slack on a sum is rounding, not that width + treatment, baseline = [1e-12] * 8 + [1.0 + 1.1e-9], [0.0] * 8 + [1.0] + deltas, mags = [t - b for t, b in zip(treatment, baseline, strict=True)], pair_magnitudes(treatment, baseline) + assert sign_flip_p(deltas, magnitudes=mags, exact=True) == 2 / 2 ** 9 # the ordinary enumeration of 9 deltas + assert ShiftTest(deltas, eps=tie_widths(mags, 9)).p(0.0) == 2 / 2 ** 9 + result = stats.paired_test(treatment, baseline, rounded=False) + assert (result["p"], result["n_changed"], result["significant"]) == (2 / 2 ** 9, 9, True) + + +def test_the_ends_of_the_interval_are_shifts_the_test_accepts_when_the_mean_is_not_one(): + # every delta is a change, the mean 1.09e-12 ties the ninth delta (width 1.7e-12) and leaves 8 on one side: rejected + treatment, baseline = [1e-12] * 8 + [0.0017 + 1.8e-12], [0.0] * 8 + [0.0017] + deltas, mags = [t - b for t, b in zip(treatment, baseline, strict=True)], pair_magnitudes(treatment, baseline) + test = ShiftTest(deltas, eps=tie_widths(mags, 9)) + assert test.changed == 9 and test.p(sum(deltas) / 9) < ALPHA + lo, hi = sign_flip_interval(deltas, sign_flip_p(deltas, magnitudes=mags), rounded=False, magnitudes=mags) + assert lo is not None and hi is not None + assert test.p(lo) is None and test.p(hi) is None # all tied: accepted, where the mean was not + exact = stats.paired_test(treatment, baseline, rounded=False)["exact"] + assert (exact["ci_lo"], exact["ci_hi"]) == (lo, hi) + + +def test_the_mde_of_changed_pairs_does_not_depend_on_an_unchanged_pair_of_large_values(): + alone = sign_flip_summary(_pairs([g * 1e-12 for g in GAINS], [0.0] * 8), exact=True) + appended = sign_flip_summary(_pairs([g * 1e-12 for g in GAINS] + [1.0], [0.0] * 8 + [1.0]), exact=True) + assert alone["mde"] is not None and appended["mde"] is not None + assert (appended["up"], appended["tied"]) == (8, 1) + + def test_a_pair_of_zeros_takes_the_width_of_the_largest_value_among_the_pairs(): assert tie_widths([0.0, 1.0], 2) == [EPS, EPS] assert tie_widths([0.0, 2e-12], 2) == [EPS * 2e-12, EPS * 2e-12]