Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **The trust gate reads a `score_cv` or `multirun_variance` whose spread leaves the float range as not measured.** `evaluate_trust`, and with it `ReportGenerator.generate_report`, raised `OverflowError: integer division result too large for a float` for context-arm scores such as 1.7e308, -1.7e308 and 1.7e308, or per-run averages like them. Each check now has `value` null, `measured` false and a "Not measured" message, and the verdict lists it under `not_measured` (#870).
2 changes: 1 addition & 1 deletion dokimi/docs/methodology.md
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ Every serious run should answer "Can we trust this run?" before interpreting del
- retrieval hit rate — the share of questions where at least one document reached the `with_docs` prompt, from the per-evaluation `retrieval_hit` the judge and panel record; eval files judged before that field existed show it as not measured, and so do files where fewer than 80% of evaluations record it (`min_retrieval_coverage`), since the rate would then describe a subset,
- multi-run variance when `--multi-run N` is used.

A check whose input the run did not produce passes without blocking or warning, but is shown as not measured, never as a pass: the verdict line reads e.g. `PASS (1 of 7 not measured)`, and the JSON report lists those checks under `trust.not_measured`.
A check whose input the run did not produce passes without blocking or warning, but is shown as not measured, never as a pass: the verdict line reads e.g. `PASS (1 of 7 not measured)`, and the JSON report lists those checks under `trust.not_measured`. The same holds for the average delta, the score coefficient of variation and the multi-run variance when the arm means differ, or the scores or per-run averages spread, by more than the float range (scores of about 1e308 in both signs).

For reproducibility:

Expand Down
69 changes: 46 additions & 23 deletions dokimi/dokimi/gate/trust_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,16 +205,28 @@ def evaluate_trust(
))

# ── 5. Score coefficient of variation (stability) ─────────────────────
with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0
cv = (with_std / with_avg) if with_avg > 0 else 1.0
checks.append(Check(
name="score_cv",
passed=cv <= cfg["max_cv"],
value=round(cv, 3),
threshold=cfg["max_cv"],
message=f"Score CV = {cv:.2f} std={with_std:.1f} (max CV {cfg['max_cv']})",
severity="warn",
))
try:
with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0
except OverflowError: # finite scores whose variance leaves the float range
checks.append(Check(
name="score_cv",
passed=True,
value=None,
threshold=cfg["max_cv"],
message="Not measured: the spread of the context-arm scores is beyond the float range",
severity="warn",
measured=False,
))
else:
cv = (with_std / with_avg) if with_avg > 0 else 1.0
checks.append(Check(
name="score_cv",
passed=cv <= cfg["max_cv"],
value=round(cv, 3),
threshold=cfg["max_cv"],
message=f"Score CV = {cv:.2f} std={with_std:.1f} (max CV {cfg['max_cv']})",
severity="warn",
))

# ── 6. Inter-rater agreement (panel judge) ────────────────────────────
agreement_scores = [a for r in valid if (a := r.scores[CONTEXT].agreement) is not None]
Expand All @@ -233,20 +245,31 @@ def evaluate_trust(
checks.append(_retrieval_check(every, cfg))

# ── 8. Multi-run variance (optional) ─────────────────────────────────
multirun_std = None
if multirun_with_averages and len(multirun_with_averages) >= 2:
multirun_std = statistics.stdev(multirun_with_averages)
checks.append(Check(
name="multirun_variance",
passed=multirun_std <= cfg["max_multirun_std"],
value=round(multirun_std, 2),
threshold=cfg["max_multirun_std"],
message=(
f"Multi-run context-arm std = {multirun_std:.2f} across {len(multirun_with_averages)} runs "
f"(max {cfg['max_multirun_std']})"
),
severity="fail",
))
try:
multirun_std = statistics.stdev(multirun_with_averages)
except OverflowError: # finite averages whose variance leaves the float range
checks.append(Check(
name="multirun_variance",
passed=True,
value=None,
threshold=cfg["max_multirun_std"],
message="Not measured: the spread of the per-run context-arm averages is beyond the float range",
severity="fail",
measured=False,
))
else:
checks.append(Check(
name="multirun_variance",
passed=multirun_std <= cfg["max_multirun_std"],
value=round(multirun_std, 2),
threshold=cfg["max_multirun_std"],
message=(
f"Multi-run context-arm std = {multirun_std:.2f} across {len(multirun_with_averages)} runs "
f"(max {cfg['max_multirun_std']})"
),
severity="fail",
))

return Verdict(checks=checks)

Expand Down
39 changes: 39 additions & 0 deletions dokimi/tests/test_trust_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,45 @@ def test_an_all_zero_context_arm_reads_as_cv_one(self):
assert (check.value, check.passed) == (1.0, False)


class TestASpreadBeyondTheFloatRange:
"""Finite scores whose variance leaves the float range: the check is not measured, the gate still answers."""

BIG = [1.7e308, -1.7e308, 1.7e308]

def test_score_cv_is_not_measured_and_the_report_generates(self):
import json

from dokimi.report.generator import ReportGenerator

evals = [_row(s, 1.0) for s in self.BIG]
v = evaluate_trust(evals, thresholds={"min_questions": 1})
check = _check(v, "score_cv")
assert (check.measured, check.passed, check.value) == (False, True, None)
assert check.message == "Not measured: the spread of the context-arm scores is beyond the float range"
assert "score_cv" in v.not_measured

report = json.loads(ReportGenerator().generate_report(
{"evaluations": evals}, {"questions": []}, output_format="json",
))
(cv,) = [c for c in report["trust"]["checks"] if c["name"] == "score_cv"]
assert (cv["measured"], cv["value"]) == (False, None)
assert "score_cv" in report["trust"]["not_measured"]

def test_the_block_renders_the_unmeasured_cv_like_any_other(self):
md = format_trust_block(evaluate_trust([_row(s, 1.0) for s in self.BIG],
thresholds={"min_questions": 1}))
assert "| score_cv | ➖ | None | 0.6 | Not measured: the spread of the context-arm scores" in md
assert "not measured)" in md.splitlines()[2]

def test_multirun_variance_is_not_measured(self):
v = evaluate_trust(_evals(10), multirun_with_averages=self.BIG)
check = _check(v, "multirun_variance")
assert (check.measured, check.passed, check.value) == (False, True, None)
assert check.message == (
"Not measured: the spread of the per-run context-arm averages is beyond the float range")
assert "multirun_variance" in v.not_measured


def test_a_not_measured_retrieval_check_never_warns():
"""Under the coverage floor the rate is not judged: it passes, is not a
warning, and serializes ``measured`` as false (the report JSON reads it)."""
Expand Down
Loading