From cc571a6ce76dcdb1fb398e237ff4edc40b4e90f3 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 20:35:50 +0000 Subject: [PATCH] fix(gate): trust gate reads a spread beyond the float range as not measured (#870) evaluate_trust called statistics.stdev on the context-arm scores (score_cv) and on the per-run averages (multirun_variance). For finite values such as 1.7e308, -1.7e308 and 1.7e308 the variance leaves the float range and stdev raised OverflowError, which stopped ReportGenerator.generate_report. Both checks now catch OverflowError and report a Check with value None, passed True and measured False and a "Not measured" message, as min_delta does for an overflowing delta. The verdict lists them under not_measured; every finite spread keeps its value and message. Signed-off-by: Anton Volkov --- ...0261117_150000_trust-spread-float-range.md | 8 +++ dokimi/docs/methodology.md | 2 +- dokimi/dokimi/gate/trust_gate.py | 69 ++++++++++++------- dokimi/tests/test_trust_gate.py | 39 +++++++++++ 4 files changed, 94 insertions(+), 24 deletions(-) create mode 100644 dokimi/changelog.d/20261117_150000_trust-spread-float-range.md diff --git a/dokimi/changelog.d/20261117_150000_trust-spread-float-range.md b/dokimi/changelog.d/20261117_150000_trust-spread-float-range.md new file mode 100644 index 000000000..33d9c05c0 --- /dev/null +++ b/dokimi/changelog.d/20261117_150000_trust-spread-float-range.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **The trust gate reads a `score_cv` or `multirun_variance` whose spread leaves the float range as not measured.** `evaluate_trust`, and with it `ReportGenerator.generate_report`, raised `OverflowError: integer division result too large for a float` for context-arm scores such as 1.7e308, -1.7e308 and 1.7e308, or per-run averages like them. Each check now has `value` null, `measured` false and a "Not measured" message, and the verdict lists it under `not_measured` (#870). diff --git a/dokimi/docs/methodology.md b/dokimi/docs/methodology.md index 4669fbdf7..2f547cc83 100644 --- a/dokimi/docs/methodology.md +++ b/dokimi/docs/methodology.md @@ -213,7 +213,7 @@ Every serious run should answer "Can we trust this run?" before interpreting del - retrieval hit rate — the share of questions where at least one document reached the `with_docs` prompt, from the per-evaluation `retrieval_hit` the judge and panel record; eval files judged before that field existed show it as not measured, and so do files where fewer than 80% of evaluations record it (`min_retrieval_coverage`), since the rate would then describe a subset, - multi-run variance when `--multi-run N` is used. -A check whose input the run did not produce passes without blocking or warning, but is shown as not measured, never as a pass: the verdict line reads e.g. `PASS (1 of 7 not measured)`, and the JSON report lists those checks under `trust.not_measured`. +A check whose input the run did not produce passes without blocking or warning, but is shown as not measured, never as a pass: the verdict line reads e.g. `PASS (1 of 7 not measured)`, and the JSON report lists those checks under `trust.not_measured`. The same holds for the average delta, the score coefficient of variation and the multi-run variance when the arm means differ, or the scores or per-run averages spread, by more than the float range (scores of about 1e308 in both signs). For reproducibility: diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py index 273506259..98bdea150 100644 --- a/dokimi/dokimi/gate/trust_gate.py +++ b/dokimi/dokimi/gate/trust_gate.py @@ -205,16 +205,28 @@ def evaluate_trust( )) # ── 5. Score coefficient of variation (stability) ───────────────────── - with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0 - cv = (with_std / with_avg) if with_avg > 0 else 1.0 - checks.append(Check( - name="score_cv", - passed=cv <= cfg["max_cv"], - value=round(cv, 3), - threshold=cfg["max_cv"], - message=f"Score CV = {cv:.2f} std={with_std:.1f} (max CV {cfg['max_cv']})", - severity="warn", - )) + try: + with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0 + except OverflowError: # finite scores whose variance leaves the float range + checks.append(Check( + name="score_cv", + passed=True, + value=None, + threshold=cfg["max_cv"], + message="Not measured: the spread of the context-arm scores is beyond the float range", + severity="warn", + measured=False, + )) + else: + cv = (with_std / with_avg) if with_avg > 0 else 1.0 + checks.append(Check( + name="score_cv", + passed=cv <= cfg["max_cv"], + value=round(cv, 3), + threshold=cfg["max_cv"], + message=f"Score CV = {cv:.2f} std={with_std:.1f} (max CV {cfg['max_cv']})", + severity="warn", + )) # ── 6. Inter-rater agreement (panel judge) ──────────────────────────── agreement_scores = [a for r in valid if (a := r.scores[CONTEXT].agreement) is not None] @@ -233,20 +245,31 @@ def evaluate_trust( checks.append(_retrieval_check(every, cfg)) # ── 8. Multi-run variance (optional) ───────────────────────────────── - multirun_std = None if multirun_with_averages and len(multirun_with_averages) >= 2: - multirun_std = statistics.stdev(multirun_with_averages) - checks.append(Check( - name="multirun_variance", - passed=multirun_std <= cfg["max_multirun_std"], - value=round(multirun_std, 2), - threshold=cfg["max_multirun_std"], - message=( - f"Multi-run context-arm std = {multirun_std:.2f} across {len(multirun_with_averages)} runs " - f"(max {cfg['max_multirun_std']})" - ), - severity="fail", - )) + try: + multirun_std = statistics.stdev(multirun_with_averages) + except OverflowError: # finite averages whose variance leaves the float range + checks.append(Check( + name="multirun_variance", + passed=True, + value=None, + threshold=cfg["max_multirun_std"], + message="Not measured: the spread of the per-run context-arm averages is beyond the float range", + severity="fail", + measured=False, + )) + else: + checks.append(Check( + name="multirun_variance", + passed=multirun_std <= cfg["max_multirun_std"], + value=round(multirun_std, 2), + threshold=cfg["max_multirun_std"], + message=( + f"Multi-run context-arm std = {multirun_std:.2f} across {len(multirun_with_averages)} runs " + f"(max {cfg['max_multirun_std']})" + ), + severity="fail", + )) return Verdict(checks=checks) diff --git a/dokimi/tests/test_trust_gate.py b/dokimi/tests/test_trust_gate.py index 389192e45..b68875d3b 100644 --- a/dokimi/tests/test_trust_gate.py +++ b/dokimi/tests/test_trust_gate.py @@ -404,6 +404,45 @@ def test_an_all_zero_context_arm_reads_as_cv_one(self): assert (check.value, check.passed) == (1.0, False) +class TestASpreadBeyondTheFloatRange: + """Finite scores whose variance leaves the float range: the check is not measured, the gate still answers.""" + + BIG = [1.7e308, -1.7e308, 1.7e308] + + def test_score_cv_is_not_measured_and_the_report_generates(self): + import json + + from dokimi.report.generator import ReportGenerator + + evals = [_row(s, 1.0) for s in self.BIG] + v = evaluate_trust(evals, thresholds={"min_questions": 1}) + check = _check(v, "score_cv") + assert (check.measured, check.passed, check.value) == (False, True, None) + assert check.message == "Not measured: the spread of the context-arm scores is beyond the float range" + assert "score_cv" in v.not_measured + + report = json.loads(ReportGenerator().generate_report( + {"evaluations": evals}, {"questions": []}, output_format="json", + )) + (cv,) = [c for c in report["trust"]["checks"] if c["name"] == "score_cv"] + assert (cv["measured"], cv["value"]) == (False, None) + assert "score_cv" in report["trust"]["not_measured"] + + def test_the_block_renders_the_unmeasured_cv_like_any_other(self): + md = format_trust_block(evaluate_trust([_row(s, 1.0) for s in self.BIG], + thresholds={"min_questions": 1})) + assert "| score_cv | ➖ | None | 0.6 | Not measured: the spread of the context-arm scores" in md + assert "not measured)" in md.splitlines()[2] + + def test_multirun_variance_is_not_measured(self): + v = evaluate_trust(_evals(10), multirun_with_averages=self.BIG) + check = _check(v, "multirun_variance") + assert (check.measured, check.passed, check.value) == (False, True, None) + assert check.message == ( + "Not measured: the spread of the per-run context-arm averages is beyond the float range") + assert "multirun_variance" in v.not_measured + + def test_a_not_measured_retrieval_check_never_warns(): """Under the coverage floor the rate is not judged: it passes, is not a warning, and serializes ``measured`` as false (the report JSON reads it)."""