diff --git a/dokimi/changelog.d/20261117_120000_interaction-tie-width.md b/dokimi/changelog.d/20261117_120000_interaction-tie-width.md new file mode 100644 index 000000000..42383883a --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_interaction-tie-width.md @@ -0,0 +1,3 @@ +### Fixed + +- **`interaction_test` no longer reads contrasts of tiny-scale scores as ties** — eight evaluations whose `a+b` scores were `g * 1e-300` (g in 2, 4, 3, 5, 6, 9, 7, 8) and 0 for the other three arms gave `p` null, `n_changed` 0 and `significant` null, where the same scores at `1e-3` give `p` 0.0078, `n_changed` 8 and `significant` true, because every contrast was within the absolute 1e-9 tie width. The contrast `AB − A − B + baseline` of a question now ties within 1e-9 times the largest of its four scores when that is below 1 (1e-9 for 1 or more, as before), and each arm delta of the row (`delta_a`, `delta_b`, `delta_combined`) uses the larger score of its pair, as `paired_test` does. Scores of 1 or more give the same rows as before. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4298fd90d..72c0776b5 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -431,7 +431,10 @@ artifact also gets `summary.interactions`: the contrast `AB − A − B + baseline` computed per question against one shared baseline, which is what tells you whether combining two treatments beats the sum of their separate effects. Running the arms separately and subtracting run averages does -not answer that question — the baselines drift between runs. +not answer that question — the baselines drift between runs. The tie rule above +applies to the contrast with the largest of the question's four scores as its +value (AB = 1e-300 with A, B and baseline all zero is a change), and to each arm +delta with the larger score of its pair. Scores whose paired difference or interaction contrast leaves the float range are refused by the statistics. The run is still saved: `summary.significance_error` or diff --git a/dokimi/dokimi/eval/interactions.py b/dokimi/dokimi/eval/interactions.py index 629a8a4ef..63fb134d2 100644 --- a/dokimi/dokimi/eval/interactions.py +++ b/dokimi/dokimi/eval/interactions.py @@ -36,6 +36,7 @@ _describe_deltas, _finalize, ) +from dokimi.eval.sign_flip import pair_magnitudes from dokimi.treatments.specs import split_combined_name @@ -91,11 +92,15 @@ def _interaction_contrast( if len(rows) < 2: return None + col_ab = [ab for ab, _, _, _ in rows] + col_a = [a for _, a, _, _ in rows] + col_b = [b for _, _, b, _ in rows] bases = [base for _, _, _, base in rows] - diffs_a = _deltas([a for _, a, _, _ in rows], bases) - diffs_b = _deltas([b for _, _, b, _ in rows], bases) - diffs_ab = _deltas([ab for ab, _, _, _ in rows], bases) + diffs_a = _deltas(col_a, bases) + diffs_b = _deltas(col_b, bases) + diffs_ab = _deltas(col_ab, bases) contrasts: List[float] = [] + magnitudes = [max(abs(x) for x in row) for row in rows] # float noise of the contrast follows the largest score for row in rows: ab, a, b, base = row contrast = ab - a - b + base @@ -107,9 +112,9 @@ def _interaction_contrast( if not math.isfinite(contrast) and finite_row: raise OverflowError("deltas too large: a contrast of the four scores leaves the float range") contrasts.append(contrast) - res = _describe_deltas(contrasts, confidence=confidence, seed=seed) - delta_a = _arm_delta(diffs_a, confidence, seed) - delta_b = _arm_delta(diffs_b, confidence, seed) + res = _describe_deltas(contrasts, confidence=confidence, seed=seed, magnitudes=magnitudes) + delta_a = _arm_delta(diffs_a, confidence, seed, pair_magnitudes(col_a, bases)) + delta_b = _arm_delta(diffs_b, confidence, seed, pair_magnitudes(col_b, bases)) res.update( { "arm_a": arm_a, @@ -118,7 +123,7 @@ def _interaction_contrast( "baseline_arm": baseline_arm, "delta_a": delta_a, "delta_b": delta_b, - "delta_combined": _arm_delta(diffs_ab, confidence, seed), + "delta_combined": _arm_delta(diffs_ab, confidence, seed, pair_magnitudes(col_ab, bases)), "interaction": res["delta_mean"], # the sum of the two printed deltas, so the row adds up; round() only drops float noise "additive_expectation": round(delta_a + delta_b, 12) + 0.0, @@ -127,9 +132,9 @@ def _interaction_contrast( return res -def _arm_delta(deltas: List[float], confidence: float, seed: int) -> float: +def _arm_delta(deltas: List[float], confidence: float, seed: int, magnitudes: List[float]) -> float: """The ``delta_mean`` the arms table prints for these deltas: rounded and held inside their own interval.""" - return _describe_deltas(deltas, confidence=confidence, seed=seed)["delta_mean"] + return _describe_deltas(deltas, confidence=confidence, seed=seed, magnitudes=magnitudes)["delta_mean"] def interaction_test( @@ -156,7 +161,9 @@ def interaction_test( interval inverts it, so it excludes 0 exactly when ``p < 0.05``. It assumes the contrast symmetric about 0 under no interaction, and its level is 5% only then (module docstring). ``significant`` is ``None`` with no p or under - the floor rule (:func:`dokimi.eval.stats._describe_deltas`). + the floor rule (:func:`dokimi.eval.stats._describe_deltas`). The tie width of + a contrast is ``EPS`` times the largest of its four scores when that is + below 1, and each arm delta ties by the larger score of its pair. Only questions where all four arms produced a score contribute. Returns ``None`` if fewer than two such questions exist. diff --git a/dokimi/tests/test_interactions_tie_width.py b/dokimi/tests/test_interactions_tie_width.py new file mode 100644 index 000000000..bd4aba113 --- /dev/null +++ b/dokimi/tests/test_interactions_tie_width.py @@ -0,0 +1,74 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The sign-flip tie width of an interaction row follows the scores of the row (issue #821). + +The rule is the one of docs/benchmarking-and-comparison.md ("A delta within 1e-9 of 0 is a tie ... unless the values +of its pair are below 1"), applied to the contrast ``AB - A - B + baseline`` with the largest of the four scores as +the scale of its float noise, and to each arm delta with the larger score of its pair. +""" + +from __future__ import annotations + +import pytest + +from dokimi.eval.interactions import interaction_test +from dokimi.eval.stats import paired_test + +G = [2, 4, 3, 5, 6, 9, 7, 8] +ARMS = {"arm_a": "a", "arm_b": "b", "combined_arm": "a+b", "baseline_arm": "baseline"} + + +def _ev(qid: str, ab: float, a: float, b: float, base: float) -> dict: + scores = {"a+b": ab, "a": a, "b": b, "baseline": base} + return {"question_id": qid, "scores": {k: {"aggregate": v} for k, v in scores.items()}} + + +def _gains(scale: float, arm: str) -> list[dict]: + rows = [] + for i, g in enumerate(G): + vals = {"a+b": 0.0, "a": 0.0, "b": 0.0, "baseline": 0.0} + vals[arm] = g * scale + rows.append(_ev(f"q{i}", vals["a+b"], vals["a"], vals["b"], vals["baseline"])) + return rows + + +@pytest.mark.parametrize("scale", [1e-3, 1e-300]) +def test_a_contrast_of_tiny_scores_is_a_change_with_a_p_at_any_scale(scale): + res = interaction_test(_gains(scale, "a+b"), **ARMS) + assert res is not None + assert res["n_changed"] == 8 + assert res["p"] == 0.0078 + assert res["significant"] is True + + +def test_the_tie_width_of_a_row_follows_its_largest_score(): + # 1e-14 on scores of 1e-3 is float noise (a tie), 1e-6 on scores of 100 is a change + tie_row = _ev("t", 1e-3 + 1e-14, 1e-3, 1e-3, 1e-3) + change_row = _ev("c", 100 + 1e-6, 100.0, 100.0, 100.0) + big = [_ev(f"q{i}", g * 1e-3, 0.0, 0.0, 0.0) for i, g in enumerate(G)] + tie = interaction_test([*big, tie_row], **ARMS) + change = interaction_test([*big, change_row], **ARMS) + assert tie is not None and change is not None + assert tie["n_changed"] == 8 + assert change["n_changed"] == 9 + + +def test_the_scale_of_a_contrast_is_the_largest_of_all_four_scores_not_of_ab_and_baseline(): + # the contrast is -7e-13 and `a` is the largest score: a tie at 1e-12 (EPS times a); EPS times max(ab, base) = 5e-13 would count a change + row = _ev("x", 5e-4, 1e-3 + 7e-13, 0.0, 5e-4) + big = [_ev(f"q{i}", g * 1e-3, 0.0, 0.0, 0.0) for i, g in enumerate(G)] + res = interaction_test([*big, row], **ARMS) + assert res is not None + assert res["n_changed"] == 8 + + +@pytest.mark.parametrize("scale", [1e-3, 1e-300]) +@pytest.mark.parametrize("arm", ["a", "b", "a+b"]) +def test_an_arm_delta_of_tiny_scores_is_read_like_the_arms_table_reads_it(arm, scale): + res = interaction_test(_gains(scale, arm), **ARMS) + assert res is not None + treatment = [g * scale for g in G] + expected = paired_test(treatment, [0.0] * len(G))["delta_mean"] + key = {"a": "delta_a", "b": "delta_b", "a+b": "delta_combined"}[arm] + assert res[key] == expected