Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dokimi/changelog.d/20261117_120000_interaction-tie-width.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
### Fixed

- **`interaction_test` no longer reads contrasts of tiny-scale scores as ties** — eight evaluations whose `a+b` scores were `g * 1e-300` (g in 2, 4, 3, 5, 6, 9, 7, 8) and 0 for the other three arms gave `p` null, `n_changed` 0 and `significant` null, where the same scores at `1e-3` give `p` 0.0078, `n_changed` 8 and `significant` true, because every contrast was within the absolute 1e-9 tie width. The contrast `AB − A − B + baseline` of a question now ties within 1e-9 times the largest of its four scores when that is below 1 (1e-9 for 1 or more, as before), and each arm delta of the row (`delta_a`, `delta_b`, `delta_combined`) uses the larger score of its pair, as `paired_test` does. Scores of 1 or more give the same rows as before.
5 changes: 4 additions & 1 deletion dokimi/docs/benchmarking-and-comparison.md
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,10 @@ artifact also gets `summary.interactions`: the contrast
`AB − A − B + baseline` computed per question against one shared baseline, which
is what tells you whether combining two treatments beats the sum of their
separate effects. Running the arms separately and subtracting run averages does
not answer that question — the baselines drift between runs.
not answer that question — the baselines drift between runs. The tie rule above
applies to the contrast with the largest of the question's four scores as its
value (AB = 1e-300 with A, B and baseline all zero is a change), and to each arm
delta with the larger score of its pair.

Scores whose paired difference or interaction contrast leaves the float range are
refused by the statistics. The run is still saved: `summary.significance_error` or
Expand Down
27 changes: 17 additions & 10 deletions dokimi/dokimi/eval/interactions.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
_describe_deltas,
_finalize,
)
from dokimi.eval.sign_flip import pair_magnitudes
from dokimi.treatments.specs import split_combined_name


Expand Down Expand Up @@ -91,11 +92,15 @@ def _interaction_contrast(
if len(rows) < 2:
return None

col_ab = [ab for ab, _, _, _ in rows]
col_a = [a for _, a, _, _ in rows]
col_b = [b for _, _, b, _ in rows]
bases = [base for _, _, _, base in rows]
diffs_a = _deltas([a for _, a, _, _ in rows], bases)
diffs_b = _deltas([b for _, _, b, _ in rows], bases)
diffs_ab = _deltas([ab for ab, _, _, _ in rows], bases)
diffs_a = _deltas(col_a, bases)
diffs_b = _deltas(col_b, bases)
diffs_ab = _deltas(col_ab, bases)
contrasts: List[float] = []
magnitudes = [max(abs(x) for x in row) for row in rows] # float noise of the contrast follows the largest score
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
for row in rows:
ab, a, b, base = row
contrast = ab - a - b + base
Expand All @@ -107,9 +112,9 @@ def _interaction_contrast(
if not math.isfinite(contrast) and finite_row:
raise OverflowError("deltas too large: a contrast of the four scores leaves the float range")
contrasts.append(contrast)
res = _describe_deltas(contrasts, confidence=confidence, seed=seed)
delta_a = _arm_delta(diffs_a, confidence, seed)
delta_b = _arm_delta(diffs_b, confidence, seed)
res = _describe_deltas(contrasts, confidence=confidence, seed=seed, magnitudes=magnitudes)
delta_a = _arm_delta(diffs_a, confidence, seed, pair_magnitudes(col_a, bases))
delta_b = _arm_delta(diffs_b, confidence, seed, pair_magnitudes(col_b, bases))
res.update(
{
"arm_a": arm_a,
Expand All @@ -118,7 +123,7 @@ def _interaction_contrast(
"baseline_arm": baseline_arm,
"delta_a": delta_a,
"delta_b": delta_b,
"delta_combined": _arm_delta(diffs_ab, confidence, seed),
"delta_combined": _arm_delta(diffs_ab, confidence, seed, pair_magnitudes(col_ab, bases)),
"interaction": res["delta_mean"],
# the sum of the two printed deltas, so the row adds up; round() only drops float noise
"additive_expectation": round(delta_a + delta_b, 12) + 0.0,
Expand All @@ -127,9 +132,9 @@ def _interaction_contrast(
return res


def _arm_delta(deltas: List[float], confidence: float, seed: int) -> float:
def _arm_delta(deltas: List[float], confidence: float, seed: int, magnitudes: List[float]) -> float:
"""The ``delta_mean`` the arms table prints for these deltas: rounded and held inside their own interval."""
return _describe_deltas(deltas, confidence=confidence, seed=seed)["delta_mean"]
return _describe_deltas(deltas, confidence=confidence, seed=seed, magnitudes=magnitudes)["delta_mean"]


def interaction_test(
Expand All @@ -156,7 +161,9 @@ def interaction_test(
interval inverts it, so it excludes 0 exactly when ``p < 0.05``. It assumes
the contrast symmetric about 0 under no interaction, and its level is 5%
only then (module docstring). ``significant`` is ``None`` with no p or under
the floor rule (:func:`dokimi.eval.stats._describe_deltas`).
the floor rule (:func:`dokimi.eval.stats._describe_deltas`). The tie width of
a contrast is ``EPS`` times the largest of its four scores when that is
below 1, and each arm delta ties by the larger score of its pair.

Only questions where all four arms produced a score contribute. Returns
``None`` if fewer than two such questions exist.
Expand Down
74 changes: 74 additions & 0 deletions dokimi/tests/test_interactions_tie_width.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""The sign-flip tie width of an interaction row follows the scores of the row (issue #821).

The rule is the one of docs/benchmarking-and-comparison.md ("A delta within 1e-9 of 0 is a tie ... unless the values
of its pair are below 1"), applied to the contrast ``AB - A - B + baseline`` with the largest of the four scores as
the scale of its float noise, and to each arm delta with the larger score of its pair.
"""

from __future__ import annotations

import pytest

from dokimi.eval.interactions import interaction_test
from dokimi.eval.stats import paired_test

G = [2, 4, 3, 5, 6, 9, 7, 8]
ARMS = {"arm_a": "a", "arm_b": "b", "combined_arm": "a+b", "baseline_arm": "baseline"}


def _ev(qid: str, ab: float, a: float, b: float, base: float) -> dict:
scores = {"a+b": ab, "a": a, "b": b, "baseline": base}
return {"question_id": qid, "scores": {k: {"aggregate": v} for k, v in scores.items()}}


def _gains(scale: float, arm: str) -> list[dict]:
rows = []
for i, g in enumerate(G):
vals = {"a+b": 0.0, "a": 0.0, "b": 0.0, "baseline": 0.0}
vals[arm] = g * scale
rows.append(_ev(f"q{i}", vals["a+b"], vals["a"], vals["b"], vals["baseline"]))
return rows


@pytest.mark.parametrize("scale", [1e-3, 1e-300])
def test_a_contrast_of_tiny_scores_is_a_change_with_a_p_at_any_scale(scale):
res = interaction_test(_gains(scale, "a+b"), **ARMS)
assert res is not None
assert res["n_changed"] == 8
assert res["p"] == 0.0078
assert res["significant"] is True


def test_the_tie_width_of_a_row_follows_its_largest_score():
# 1e-14 on scores of 1e-3 is float noise (a tie), 1e-6 on scores of 100 is a change
tie_row = _ev("t", 1e-3 + 1e-14, 1e-3, 1e-3, 1e-3)
change_row = _ev("c", 100 + 1e-6, 100.0, 100.0, 100.0)
big = [_ev(f"q{i}", g * 1e-3, 0.0, 0.0, 0.0) for i, g in enumerate(G)]
tie = interaction_test([*big, tie_row], **ARMS)
change = interaction_test([*big, change_row], **ARMS)
assert tie is not None and change is not None
assert tie["n_changed"] == 8
assert change["n_changed"] == 9

Comment thread
antonwolfy marked this conversation as resolved.
Resolved

def test_the_scale_of_a_contrast_is_the_largest_of_all_four_scores_not_of_ab_and_baseline():
# the contrast is -7e-13 and `a` is the largest score: a tie at 1e-12 (EPS times a); EPS times max(ab, base) = 5e-13 would count a change
row = _ev("x", 5e-4, 1e-3 + 7e-13, 0.0, 5e-4)
big = [_ev(f"q{i}", g * 1e-3, 0.0, 0.0, 0.0) for i, g in enumerate(G)]
res = interaction_test([*big, row], **ARMS)
assert res is not None
assert res["n_changed"] == 8


@pytest.mark.parametrize("scale", [1e-3, 1e-300])
@pytest.mark.parametrize("arm", ["a", "b", "a+b"])
def test_an_arm_delta_of_tiny_scores_is_read_like_the_arms_table_reads_it(arm, scale):
res = interaction_test(_gains(scale, arm), **ARMS)
assert res is not None
treatment = [g * scale for g in G]
expected = paired_test(treatment, [0.0] * len(G))["delta_mean"]
key = {"a": "delta_a", "b": "delta_b", "a+b": "delta_combined"}[arm]
assert res[key] == expected
Loading