From 8b00389968f1298a76cefd5ae425dd565cb3608f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 11:05:19 +0000 Subject: [PATCH 1/2] fix(eval): interaction_test reads contrasts of tiny-scale scores by their values (#821) interaction_test and interaction_significance passed the per-question contrasts AB - A - B + baseline, and the three arm deltas of the row, to the sign-flip description without the values they come from, so every tie was within the absolute 1e-9. Eight evaluations with a+b scores g * 1e-300 and 0 elsewhere gave p None, n_changed 0 and significant None, where the same scores at 1e-3 give p 0.0078, n_changed 8 and significant True. The contrast of a row now passes the largest magnitude of its four scores, and each arm delta the larger magnitude of its pair against the baseline, as paired_test does since #818. A tie is within 1e-9 times that value when it is below 1, and 1e-9 otherwise, so rows at ordinary score scales are unchanged. The 1e-300 rows read as their 1e-3 copies. Signed-off-by: Anton Volkov --- .../20261117_120000_interaction-tie-width.md | 3 + dokimi/docs/benchmarking-and-comparison.md | 5 +- dokimi/dokimi/eval/interactions.py | 16 +++-- dokimi/tests/test_interactions_tie_width.py | 65 +++++++++++++++++++ 4 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 dokimi/changelog.d/20261117_120000_interaction-tie-width.md create mode 100644 dokimi/tests/test_interactions_tie_width.py diff --git a/dokimi/changelog.d/20261117_120000_interaction-tie-width.md b/dokimi/changelog.d/20261117_120000_interaction-tie-width.md new file mode 100644 index 000000000..42383883a --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_interaction-tie-width.md @@ -0,0 +1,3 @@ +### Fixed + +- **`interaction_test` no longer reads contrasts of tiny-scale scores as ties** — eight evaluations whose `a+b` scores were `g * 1e-300` (g in 2, 4, 3, 5, 6, 9, 7, 8) and 0 for the other three arms gave `p` null, `n_changed` 0 and `significant` null, where the same scores at `1e-3` give `p` 0.0078, `n_changed` 8 and `significant` true, because every contrast was within the absolute 1e-9 tie width. The contrast `AB − A − B + baseline` of a question now ties within 1e-9 times the largest of its four scores when that is below 1 (1e-9 for 1 or more, as before), and each arm delta of the row (`delta_a`, `delta_b`, `delta_combined`) uses the larger score of its pair, as `paired_test` does. Scores of 1 or more give the same rows as before. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4298fd90d..5bc7f9a03 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -431,7 +431,10 @@ artifact also gets `summary.interactions`: the contrast `AB − A − B + baseline` computed per question against one shared baseline, which is what tells you whether combining two treatments beats the sum of their separate effects. Running the arms separately and subtracting run averages does -not answer that question — the baselines drift between runs. +not answer that question — the baselines drift between runs. The tie rule above +applies to the contrast with the largest of the question's four scores as its +value (a contrast of 1e-12 between scores of 1e-300 is a change), and to each arm +delta with the larger score of its pair. Scores whose paired difference or interaction contrast leaves the float range are refused by the statistics. The run is still saved: `summary.significance_error` or diff --git a/dokimi/dokimi/eval/interactions.py b/dokimi/dokimi/eval/interactions.py index 629a8a4ef..c1de78a23 100644 --- a/dokimi/dokimi/eval/interactions.py +++ b/dokimi/dokimi/eval/interactions.py @@ -36,6 +36,7 @@ _describe_deltas, _finalize, ) +from dokimi.eval.sign_flip import pair_magnitudes from dokimi.treatments.specs import split_combined_name @@ -96,6 +97,7 @@ def _interaction_contrast( diffs_b = _deltas([b for _, _, b, _ in rows], bases) diffs_ab = _deltas([ab for ab, _, _, _ in rows], bases) contrasts: List[float] = [] + magnitudes = [max(abs(x) for x in row) for row in rows] # float noise of the contrast follows the largest score for row in rows: ab, a, b, base = row contrast = ab - a - b + base @@ -107,9 +109,9 @@ def _interaction_contrast( if not math.isfinite(contrast) and finite_row: raise OverflowError("deltas too large: a contrast of the four scores leaves the float range") contrasts.append(contrast) - res = _describe_deltas(contrasts, confidence=confidence, seed=seed) - delta_a = _arm_delta(diffs_a, confidence, seed) - delta_b = _arm_delta(diffs_b, confidence, seed) + res = _describe_deltas(contrasts, confidence=confidence, seed=seed, magnitudes=magnitudes) + delta_a = _arm_delta(diffs_a, confidence, seed, pair_magnitudes([a for _, a, _, _ in rows], bases)) + delta_b = _arm_delta(diffs_b, confidence, seed, pair_magnitudes([b for _, _, b, _ in rows], bases)) res.update( { "arm_a": arm_a, @@ -118,7 +120,9 @@ def _interaction_contrast( "baseline_arm": baseline_arm, "delta_a": delta_a, "delta_b": delta_b, - "delta_combined": _arm_delta(diffs_ab, confidence, seed), + "delta_combined": _arm_delta( + diffs_ab, confidence, seed, pair_magnitudes([ab for ab, _, _, _ in rows], bases) + ), "interaction": res["delta_mean"], # the sum of the two printed deltas, so the row adds up; round() only drops float noise "additive_expectation": round(delta_a + delta_b, 12) + 0.0, @@ -127,9 +131,9 @@ def _interaction_contrast( return res -def _arm_delta(deltas: List[float], confidence: float, seed: int) -> float: +def _arm_delta(deltas: List[float], confidence: float, seed: int, magnitudes: List[float]) -> float: """The ``delta_mean`` the arms table prints for these deltas: rounded and held inside their own interval.""" - return _describe_deltas(deltas, confidence=confidence, seed=seed)["delta_mean"] + return _describe_deltas(deltas, confidence=confidence, seed=seed, magnitudes=magnitudes)["delta_mean"] def interaction_test( diff --git a/dokimi/tests/test_interactions_tie_width.py b/dokimi/tests/test_interactions_tie_width.py new file mode 100644 index 000000000..29f655ae1 --- /dev/null +++ b/dokimi/tests/test_interactions_tie_width.py @@ -0,0 +1,65 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""The sign-flip tie width of an interaction row follows the scores of the row (issue #821). + +The rule is the one of docs/benchmarking-and-comparison.md ("A delta within 1e-9 of 0 is a tie ... unless the values +of its pair are below 1"), applied to the contrast ``AB - A - B + baseline`` with the largest of the four scores as +the scale of its float noise, and to each arm delta with the larger score of its pair. +""" + +from __future__ import annotations + +import pytest + +from dokimi.eval.interactions import interaction_test +from dokimi.eval.stats import paired_test + +G = [2, 4, 3, 5, 6, 9, 7, 8] +ARMS = {"arm_a": "a", "arm_b": "b", "combined_arm": "a+b", "baseline_arm": "baseline"} + + +def _ev(qid: str, ab: float, a: float, b: float, base: float) -> dict: + scores = {"a+b": ab, "a": a, "b": b, "baseline": base} + return {"question_id": qid, "scores": {k: {"aggregate": v} for k, v in scores.items()}} + + +def _gains(scale: float, arm: str) -> list[dict]: + rows = [] + for i, g in enumerate(G): + vals = {"a+b": 0.0, "a": 0.0, "b": 0.0, "baseline": 0.0} + vals[arm] = g * scale + rows.append(_ev(f"q{i}", vals["a+b"], vals["a"], vals["b"], vals["baseline"])) + return rows + + +@pytest.mark.parametrize("scale", [1e-3, 1e-300]) +def test_a_contrast_of_tiny_scores_is_a_change_with_a_p_at_any_scale(scale): + res = interaction_test(_gains(scale, "a+b"), **ARMS) + assert res is not None + assert res["n_changed"] == 8 + assert res["p"] == 0.0078 + assert res["significant"] is True + + +def test_the_tie_width_of_a_row_follows_its_largest_score(): + # 1e-14 on scores of 1e-3 is float noise (a tie), 1e-6 on scores of 100 is a change + tie_row = _ev("t", 1e-3 + 1e-14, 1e-3, 1e-3, 1e-3) + change_row = _ev("c", 100 + 1e-6, 100.0, 100.0, 100.0) + big = [_ev(f"q{i}", g * 1e-3, 0.0, 0.0, 0.0) for i, g in enumerate(G)] + tie = interaction_test([*big, tie_row], **ARMS) + change = interaction_test([*big, change_row], **ARMS) + assert tie is not None and change is not None + assert tie["n_changed"] == 8 + assert change["n_changed"] == 9 + + +@pytest.mark.parametrize("scale", [1e-3, 1e-300]) +@pytest.mark.parametrize("arm", ["a", "b", "a+b"]) +def test_an_arm_delta_of_tiny_scores_is_read_like_the_arms_table_reads_it(arm, scale): + res = interaction_test(_gains(scale, arm), **ARMS) + assert res is not None + treatment = [g * scale for g in G] + expected = paired_test(treatment, [0.0] * len(G))["delta_mean"] + key = {"a": "delta_a", "b": "delta_b", "a+b": "delta_combined"}[arm] + assert res[key] == expected From 42b9f4b016f932711814cafecb1cc63cc14ee961 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 12:34:05 +0000 Subject: [PATCH 2/2] fix(eval): interaction tie-width docs and a test of the contrast's scale (#868) Settles the review of #868: the tie-rule example in benchmarking-and-comparison.md named a contrast of 1e-12 that scores of 1e-300 cannot produce, and now uses AB = 1e-300 with A, B and baseline zero. A new test row where `a` is the largest score (contrast -7e-13, a tie at 1e-12) fails when the contrast's scale is taken from `ab` and `baseline` only. The `interaction_test` docstring states the tie width of the contrast and of each arm delta, and `_interaction_contrast` takes the ab, a and b columns once for both `_deltas` and `pair_magnitudes`. Signed-off-by: Anton Volkov --- dokimi/docs/benchmarking-and-comparison.md | 2 +- dokimi/dokimi/eval/interactions.py | 21 ++++++++++++--------- dokimi/tests/test_interactions_tie_width.py | 9 +++++++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 5bc7f9a03..72c0776b5 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -433,7 +433,7 @@ is what tells you whether combining two treatments beats the sum of their separate effects. Running the arms separately and subtracting run averages does not answer that question — the baselines drift between runs. The tie rule above applies to the contrast with the largest of the question's four scores as its -value (a contrast of 1e-12 between scores of 1e-300 is a change), and to each arm +value (AB = 1e-300 with A, B and baseline all zero is a change), and to each arm delta with the larger score of its pair. Scores whose paired difference or interaction contrast leaves the float range are diff --git a/dokimi/dokimi/eval/interactions.py b/dokimi/dokimi/eval/interactions.py index c1de78a23..63fb134d2 100644 --- a/dokimi/dokimi/eval/interactions.py +++ b/dokimi/dokimi/eval/interactions.py @@ -92,10 +92,13 @@ def _interaction_contrast( if len(rows) < 2: return None + col_ab = [ab for ab, _, _, _ in rows] + col_a = [a for _, a, _, _ in rows] + col_b = [b for _, _, b, _ in rows] bases = [base for _, _, _, base in rows] - diffs_a = _deltas([a for _, a, _, _ in rows], bases) - diffs_b = _deltas([b for _, _, b, _ in rows], bases) - diffs_ab = _deltas([ab for ab, _, _, _ in rows], bases) + diffs_a = _deltas(col_a, bases) + diffs_b = _deltas(col_b, bases) + diffs_ab = _deltas(col_ab, bases) contrasts: List[float] = [] magnitudes = [max(abs(x) for x in row) for row in rows] # float noise of the contrast follows the largest score for row in rows: @@ -110,8 +113,8 @@ def _interaction_contrast( raise OverflowError("deltas too large: a contrast of the four scores leaves the float range") contrasts.append(contrast) res = _describe_deltas(contrasts, confidence=confidence, seed=seed, magnitudes=magnitudes) - delta_a = _arm_delta(diffs_a, confidence, seed, pair_magnitudes([a for _, a, _, _ in rows], bases)) - delta_b = _arm_delta(diffs_b, confidence, seed, pair_magnitudes([b for _, _, b, _ in rows], bases)) + delta_a = _arm_delta(diffs_a, confidence, seed, pair_magnitudes(col_a, bases)) + delta_b = _arm_delta(diffs_b, confidence, seed, pair_magnitudes(col_b, bases)) res.update( { "arm_a": arm_a, @@ -120,9 +123,7 @@ def _interaction_contrast( "baseline_arm": baseline_arm, "delta_a": delta_a, "delta_b": delta_b, - "delta_combined": _arm_delta( - diffs_ab, confidence, seed, pair_magnitudes([ab for ab, _, _, _ in rows], bases) - ), + "delta_combined": _arm_delta(diffs_ab, confidence, seed, pair_magnitudes(col_ab, bases)), "interaction": res["delta_mean"], # the sum of the two printed deltas, so the row adds up; round() only drops float noise "additive_expectation": round(delta_a + delta_b, 12) + 0.0, @@ -160,7 +161,9 @@ def interaction_test( interval inverts it, so it excludes 0 exactly when ``p < 0.05``. It assumes the contrast symmetric about 0 under no interaction, and its level is 5% only then (module docstring). ``significant`` is ``None`` with no p or under - the floor rule (:func:`dokimi.eval.stats._describe_deltas`). + the floor rule (:func:`dokimi.eval.stats._describe_deltas`). The tie width of + a contrast is ``EPS`` times the largest of its four scores when that is + below 1, and each arm delta ties by the larger score of its pair. Only questions where all four arms produced a score contribute. Returns ``None`` if fewer than two such questions exist. diff --git a/dokimi/tests/test_interactions_tie_width.py b/dokimi/tests/test_interactions_tie_width.py index 29f655ae1..bd4aba113 100644 --- a/dokimi/tests/test_interactions_tie_width.py +++ b/dokimi/tests/test_interactions_tie_width.py @@ -54,6 +54,15 @@ def test_the_tie_width_of_a_row_follows_its_largest_score(): assert change["n_changed"] == 9 +def test_the_scale_of_a_contrast_is_the_largest_of_all_four_scores_not_of_ab_and_baseline(): + # the contrast is -7e-13 and `a` is the largest score: a tie at 1e-12 (EPS times a); EPS times max(ab, base) = 5e-13 would count a change + row = _ev("x", 5e-4, 1e-3 + 7e-13, 0.0, 5e-4) + big = [_ev(f"q{i}", g * 1e-3, 0.0, 0.0, 0.0) for i, g in enumerate(G)] + res = interaction_test([*big, row], **ARMS) + assert res is not None + assert res["n_changed"] == 8 + + @pytest.mark.parametrize("scale", [1e-3, 1e-300]) @pytest.mark.parametrize("arm", ["a", "b", "a+b"]) def test_an_arm_delta_of_tiny_scores_is_read_like_the_arms_table_reads_it(arm, scale):