From 4f29307c8fedf047b4be620f8040595c80bf06d1 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 11:16:56 +0000 Subject: [PATCH 1/5] fix(eval): paired_summary returns no delta where a paired difference leaves the float range (#830) paired_summary formed its deltas through checked_delta, which raises OverflowError for two finite scores such as 1.7e308 and -1.7e308. The evaluate pipeline stopped after saving the evaluations and before the report, and `report generate` and `report eval` failed. paired_summary now returns the arm means, ranges and counts with delta_mean, delta_min and delta_max set to None, and delta_overflow counts the pairs without a delta. A pair without a delta is counted as an improvement or a degradation by comparing its two scores. The pipeline prints delta=none, `orchestrate` prints an em dash, the reports print an em dash with a one-line note, and `report eval` states that its significance test was refused instead of stopping. paired_test, paired_delta and the arms path still raise as before. Test-narrowed: test_paired_summary_paired_delta_and_arm_significance_refuse_the_same_deltas pinned the raise of paired_summary that this change removes on purpose; it is replaced by test_paired_delta_and_arm_significance_refuse_deltas_beyond_the_range_while_paired_summary_has_none, which keeps the two raises that stay and asserts paired_summary returns no delta. Signed-off-by: Anton Volkov --- ...20261117_120000_paired-summary-overflow.md | 8 ++ dokimi/dokimi/commands/orchestrate.py | 3 +- dokimi/dokimi/eval/stats.py | 40 +++++-- dokimi/dokimi/orchestrator/pipeline.py | 6 +- dokimi/dokimi/report/eval_report.py | 21 +++- dokimi/dokimi/report/generator.py | 21 ++-- dokimi/tests/test_paired_summary_overflow.py | 110 ++++++++++++++++++ dokimi/tests/test_stats_float_range.py | 5 +- 8 files changed, 188 insertions(+), 26 deletions(-) create mode 100644 dokimi/changelog.d/20261117_120000_paired-summary-overflow.md create mode 100644 dokimi/tests/test_paired_summary_overflow.py diff --git a/dokimi/changelog.d/20261117_120000_paired-summary-overflow.md b/dokimi/changelog.d/20261117_120000_paired-summary-overflow.md new file mode 100644 index 000000000..b0c2b64cd --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_paired-summary-overflow.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **The evaluate pipeline and the reports finish when a paired score difference leaves the float range.** `paired_summary` raised `OverflowError` for scores such as 1.7e308 and -1.7e308 in one pair, which ended `dokimi orchestrate` after the evaluations were saved and failed `report generate` and `report eval`. It now returns the arm means, ranges and counts with no delta (`delta_mean`, `delta_min` and `delta_max` are `None`, `delta_overflow` counts the pairs); improvements and degradations of such a pair are counted by comparing its two scores. The pipeline prints `delta=none`, the reports print an em dash with a one-line note, and `report eval` names the refused significance test instead of stopping (#830). diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index a3ae425d9..981f02bdd 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -102,7 +102,8 @@ def cmd_evaluate(args: argparse.Namespace) -> None: print("\nResults:") print(f" Context avg: {summary['with_avg']}") print(f" Baseline avg: {summary['without_avg']}") - print(f" Delta: {summary['delta_avg']:+.1f}") + print(" Delta: " + ("— (a paired difference leaves the float range)" + if summary["delta_avg"] is None else f"{summary['delta_avg']:+.1f}")) print(f"\nšŸ“Š View full report: cat {results['steps']['report']['path']}") diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index d0a58becb..6fa61bc04 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -469,16 +469,25 @@ class PairedSummary: unpaired: int with_mean: float without_mean: float - delta_mean: float + delta_mean: Optional[float] with_min: float with_max: float without_min: float without_max: float - delta_min: float - delta_max: float + delta_min: Optional[float] + delta_max: Optional[float] improvements: int degradations: int neutral: int + delta_overflow: int = 0 + + +def _pair_delta(treatment: float, control: float) -> Optional[float]: + """``treatment - control``, or ``None`` where the difference of two finite scores leaves the float range.""" + try: + return checked_delta(treatment, control) + except OverflowError: + return None def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary: @@ -488,27 +497,36 @@ def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary: per-question deltas (``context - baseline``). Averaging each arm over *every* scored question instead gives two means over different questions, whose difference is not the delta the page prints beside them. An empty input gives zeros throughout. + + A pair whose two finite scores differ by more than the float range has no delta: the + delta mean, min and max are then ``None`` (``delta_overflow`` counts those pairs) and the + arm means, ranges and counts are kept. Such a pair is counted as an improvement or a + degradation by comparing its two scores, not their difference. """ + # https://github.com/intel/dokimi/issues/830 recs = records(evaluations, source="evaluations", kind="evaluations") with_s, without_s = paired_vectors(recs, CONTEXT, BASELINE) - deltas = _deltas(with_s, without_s) - paired = len(deltas) + rows = [(t, c, _pair_delta(t, c)) for t, c in zip(with_s, without_s, strict=True)] + deltas = [d for _, _, d in rows if d is not None] + overflow = len(rows) - len(deltas) + paired = len(rows) return PairedSummary( total=len(recs), paired=paired, unpaired=len(recs) - paired, with_mean=_mean(with_s) if paired else 0.0, without_mean=_mean(without_s) if paired else 0.0, - delta_mean=_mean(deltas) if paired else 0.0, + delta_mean=None if overflow else _mean(deltas) if paired else 0.0, with_min=min(with_s, default=0.0), with_max=max(with_s, default=0.0), without_min=min(without_s, default=0.0), without_max=max(without_s, default=0.0), - delta_min=min(deltas, default=0.0), - delta_max=max(deltas, default=0.0), - improvements=sum(1 for d in deltas if d > 0), - degradations=sum(1 for d in deltas if d < 0), - neutral=sum(1 for d in deltas if d == 0), + delta_min=None if overflow else min(deltas, default=0.0), + delta_max=None if overflow else max(deltas, default=0.0), + improvements=sum(1 for t, c, d in rows if (t > c if d is None else d > 0)), + degradations=sum(1 for t, c, d in rows if (t < c if d is None else d < 0)), + neutral=sum(1 for _, _, d in rows if d == 0), + delta_overflow=overflow, ) diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index a5d57ceed..afd257a97 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -427,12 +427,14 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: results["steps"]["evaluation"]["summary"] = { "with_avg": round(summary.with_mean, 1), "without_avg": round(summary.without_mean, 1), - "delta_avg": round(summary.delta_mean, 1) + "delta_avg": None if summary.delta_mean is None else round(summary.delta_mean, 1) } + delta_avg = results["steps"]["evaluation"]["summary"]["delta_avg"] print(f"āœ“ Evaluated {len(evaluations)} answers: " f"context={results['steps']['evaluation']['summary']['with_avg']}, " f"baseline={results['steps']['evaluation']['summary']['without_avg']}, " - f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}" + + ("delta=none (a paired difference leaves the float range)" if delta_avg is None + else f"delta={delta_avg:+.1f}") + (f" (over the {summary.paired} scored in both arms; {summary.unpaired} left out)" if summary.unpaired else "")) else: diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py index c64df4722..3b0d1e558 100755 --- a/dokimi/dokimi/report/eval_report.py +++ b/dokimi/dokimi/report/eval_report.py @@ -60,6 +60,14 @@ def significance_test(evals): return result +def _significance_or_refusal(evals) -> tuple[dict | None, list[str]]: + """The significance test, or the lines that say a paired difference beyond the float range refused it.""" + try: + return significance_test(evals), [] + except OverflowError as exc: + return None, ["## Statistical Significance", f"_Not computed: {exc}._", ""] + + def has_scores(e: Record) -> bool: return e.score(CONTEXT) is not None and e.score(BASELINE) is not None @@ -89,7 +97,7 @@ def stats(qs): "unpaired": s.unpaired, "with_avg": round(s.with_mean, 1), "without_avg": round(s.without_mean, 1), - "delta_avg": round(s.delta_mean, 1), + "delta_avg": None if s.delta_mean is None else round(s.delta_mean, 1), "improvements": s.improvements, "degradations": s.degradations, "neutral": s.neutral, @@ -118,6 +126,8 @@ def _summary_cells(st: dict) -> tuple[str, str, str]: """The context, baseline and delta cells of a Summary row: an em dash when no question is paired.""" if not st["valid"]: return "—", "—", "—" + if st["delta_avg"] is None: + return str(st["with_avg"]), str(st["without_avg"]), "—" return str(st["with_avg"]), str(st["without_avg"]), f'**{st["delta_avg"]:+}**' @@ -125,6 +135,8 @@ def _overall_line(st: dict) -> str: """The Overall delta bullet, or a note that no question is paired.""" if not st["valid"]: return "- **Overall delta:** none — no complete pairs (a question scored in both arms), so there is no delta." + if st["delta_avg"] is None: + return "- **Overall delta:** none — a paired difference leaves the float range, so there is no delta." verdict = "context arm is helping." if st["delta_avg"] > 2 else "context arm is neutral or slightly harmful." return f'- **Overall delta: {st["delta_avg"]:+}** — {verdict}' @@ -133,6 +145,8 @@ def _agreement_line(dyn: dict, static: dict) -> str: """The Dynamic vs Static bullet; no agreement is claimed when either set has no complete pair.""" if not (dyn["valid"] and static["valid"]): return "- **Dynamic vs Static agreement:** none — a set has no complete pairs, so there is no delta to compare." + if dyn["delta_avg"] is None or static["delta_avg"] is None: + return "- **Dynamic vs Static agreement:** none — a set has no delta (a paired difference leaves the float range)." gap = abs(dyn["delta_avg"] - static["delta_avg"]) verdict = "good consistency, benchmark is stable." if gap < 3 else "notable gap — investigate question quality." return f'- **Dynamic vs Static agreement:** {dyn["delta_avg"]:+} vs {static["delta_avg"]:+} — {verdict}' @@ -141,6 +155,8 @@ def _agreement_line(dyn: dict, static: dict) -> str: def _delta_summary(st: dict) -> str: if not st["valid"]: return " Delta: none (no complete pairs)" + if st["delta_avg"] is None: + return f" Delta: none (a paired difference leaves the float range; with={st['with_avg']}, without={st['without_avg']})" return f" Delta: {st['delta_avg']:+} (with={st['with_avg']}, without={st['without_avg']})" @@ -321,7 +337,8 @@ def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = Non lines += _unpaired_note(all_stats) # Statistical significance - sig = significance_test(evals) + sig, refusal = _significance_or_refusal(evals) + lines += refusal if sig: sig_label = ("āœ… **Statistically significant** (p < 0.05)" if sig.get("significant") else "āŒ **Not statistically significant** (p ≄ 0.05)" if sig.get("significant") is False diff --git a/dokimi/dokimi/report/generator.py b/dokimi/dokimi/report/generator.py index 532d0bf5f..f1bbb0972 100644 --- a/dokimi/dokimi/report/generator.py +++ b/dokimi/dokimi/report/generator.py @@ -158,6 +158,7 @@ def _compute_stats(self, evaluations: Sequence[Record]) -> Dict[str, Any]: "total_questions": s.total, "paired_questions": s.paired, "unpaired_questions": s.unpaired, + "delta_overflow_questions": s.delta_overflow, "with_docs": _stat_block(s, s.with_mean, s.with_min, s.with_max), "without_docs": _stat_block(s, s.without_mean, s.without_min, s.without_max), "delta": _stat_block(s, s.delta_mean, s.delta_min, s.delta_max), @@ -501,10 +502,12 @@ def _format_markdown( return "\n".join(lines) -def _stat_block(s: Any, mean: float, low: float, high: float) -> Dict[str, Optional[float]]: - """avg, min and max of one statistic, rounded; all None when no question is a complete pair.""" - if not s.paired: - return {"avg": None, "min": None, "max": None} # no complete pair: no value, not a zero +def _stat_block( + s: Any, mean: Optional[float], low: Optional[float], high: Optional[float] +) -> Dict[str, Optional[float]]: + """avg, min and max of one statistic, rounded; all None when no question is a complete pair or the statistic has no value.""" + if not s.paired or mean is None or low is None or high is None: + return {"avg": None, "min": None, "max": None} # no value, not a zero return {"avg": round(mean, 1), "min": round(low, 1), "max": round(high, 1)} @@ -519,6 +522,8 @@ def _overall_advice(stats: Dict[str, Any]) -> List[str]: """The Overall recommendation lines; no conclusion is drawn without a complete pair.""" if not stats["paired_questions"]: return ["⚪ **Overall:** no complete pairs, so no conclusion can be drawn."] + if stats["delta"]["avg"] is None: + return ["⚪ **Overall:** a paired difference leaves the float range, so there is no delta and no conclusion."] if stats["delta"]["avg"] < 0: return ["āš ļø **Overall:** Documentation is hurting answer quality. Consider:", "- Improving retrieval relevance", @@ -541,7 +546,7 @@ def _arm_means(evaluations: Sequence[Record]) -> Dict[str, Any]: "unpaired": s.unpaired, "with_avg": None if not s.paired else round(s.with_mean, 1), "without_avg": None if not s.paired else round(s.without_mean, 1), - "delta_avg": None if not s.paired else round(s.delta_mean, 1), + "delta_avg": None if s.delta_mean is None or not s.paired else round(s.delta_mean, 1), } @@ -557,8 +562,10 @@ def _complete_pairs(evaluations: Sequence[Record]) -> List[Record]: def _unpaired_note(stats: Dict[str, Any]) -> List[str]: """A line saying what the overall means leave out; nothing when every question is paired.""" + overflow = stats.get("delta_overflow_questions", 0) + note = [f"_No delta: {overflow} complete pairs differ by more than the float range._", ""] if overflow else [] if not stats["unpaired_questions"]: - return [] + return note if not stats["paired_questions"]: return [f"_No complete pairs: {stats['unpaired_questions']} unpaired questions, " "so no arm mean or delta is given._", ""] @@ -566,4 +573,4 @@ def _unpaired_note(stats: Dict[str, Any]) -> List[str]: f"_Averages, min/max, delta and the top/bottom tables are over the {stats['paired_questions']} complete pairs; " f"{stats['unpaired_questions']} unpaired (scored in one arm only) left out._", "", - ] + ] + note diff --git a/dokimi/tests/test_paired_summary_overflow.py b/dokimi/tests/test_paired_summary_overflow.py new file mode 100644 index 000000000..50334a80e --- /dev/null +++ b/dokimi/tests/test_paired_summary_overflow.py @@ -0,0 +1,110 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A paired difference beyond the float range leaves ``paired_summary`` without a delta, not without a result. + +https://github.com/intel/dokimi/issues/830 +""" + +from __future__ import annotations + +import json +import statistics + +import pytest + +from dokimi.eval import stats +from dokimi.orchestrator.pipeline import EvaluationPipeline +from dokimi.report.eval_report import generate_report +from dokimi.report.generator import ReportGenerator + +BIG = 1.7e308 + + +def _row(qid, with_score, without_score): + return {"question_id": qid, "question_text": f"Question {qid}?", + "with_docs": {"aggregate": with_score}, "without_docs": {"aggregate": without_score}} + + +# The issue's input: one pair far apart, one pair equal. +ISSUE_INPUT = [_row("q1", BIG, -BIG), _row("q2", 0.0, 0.0)] + + +def test_paired_summary_of_the_issue_input_has_no_delta_and_keeps_the_rest(): + s = stats.paired_summary(ISSUE_INPUT) + assert (s.delta_mean, s.delta_min, s.delta_max) == (None, None, None) + assert s.delta_overflow == 1 + + +def test_paired_summary_keeps_the_arm_means_ranges_and_counts_of_an_overflowing_pair(): + s = stats.paired_summary(ISSUE_INPUT + [_row("q3", 10.0, 4.0), {"question_id": "q4", "with_docs": {"aggregate": 1.0}}]) + assert (s.total, s.paired, s.unpaired) == (4, 3, 1) + assert (s.with_mean, s.without_mean) == (statistics.mean([BIG, 0.0, 10.0]), statistics.mean([-BIG, 0.0, 4.0])) + assert (s.with_min, s.with_max, s.without_min, s.without_max) == (0.0, BIG, -BIG, 4.0) + + +def test_paired_summary_counts_an_overflowing_pair_by_comparing_its_scores(): + s = stats.paired_summary([_row("q1", BIG, -BIG), _row("q2", -BIG, BIG), _row("q3", 5.0, 5.0), _row("q4", 6.0, 5.0)]) + assert (s.improvements, s.degradations, s.neutral) == (2, 1, 1) + + +def test_paired_summary_of_pairs_inside_the_range_still_has_a_delta(): + s = stats.paired_summary([_row("q1", BIG, BIG - 1e300), _row("q2", 80.0, 60.0)]) + assert s.delta_max == pytest.approx(1e300, rel=1e-9) + assert s.delta_min == 20.0 + + +def _run_pipeline(tmp_path, monkeypatch, evaluations): + monkeypatch.setattr(EvaluationPipeline, "_generate_answers", lambda self, *a, **k: []) + monkeypatch.setattr(EvaluationPipeline, "_evaluate_answers", lambda self, *a, **k: evaluations) + monkeypatch.setattr(EvaluationPipeline, "_generate_report", lambda self, *a, **k: "report") + output_dir = tmp_path / "run" + (output_dir / "personas").mkdir(parents=True) + (output_dir / "questions").mkdir() + (output_dir / "personas" / "oneTBB.json").write_text(json.dumps({"personas": []}), encoding="utf-8") + questions = [{"id": e["question_id"], "question": e["question_text"], "source_type": "manual"} for e in evaluations] + (output_dir / "questions" / "oneTBB.json").write_text(json.dumps({"questions": questions}), encoding="utf-8") + pipeline = EvaluationPipeline(product="oneTBB", output_dir=output_dir, description="Threading Building Blocks") + return pipeline.run(concurrency=1) + + +def test_pipeline_reaches_the_report_step_with_the_arm_averages_and_no_delta(tmp_path, monkeypatch, capsys): + result = _run_pipeline(tmp_path, monkeypatch, ISSUE_INPUT) + assert result["status"] == "success" + step = result["steps"]["evaluation"] + assert step["summary"]["delta_avg"] is None + assert (step["paired"], step["unpaired"]) == (2, 0) + assert "delta=none (a paired difference leaves the float range)" in capsys.readouterr().out + assert "report" in result["steps"] + + +def test_report_generator_prints_an_em_dash_and_a_note_for_the_missing_delta(): + data = {"evaluations": ISSUE_INPUT} + md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") + assert "| **—** |" in md + assert "_No delta: 1 complete pairs differ by more than the float range._" in md + assert "no delta and no conclusion" in md + out = json.loads(ReportGenerator().generate_report(data, {"questions": []}, "json"))["stats"] + assert out["delta"] == {"avg": None, "min": None, "max": None} + assert out["with_docs"]["avg"] == pytest.approx(BIG / 2, rel=1e-12) + assert (out["doc_improvements"], out["doc_degradations"]) == (1, 0) + + +def test_report_generator_group_tables_give_a_group_with_an_overflowing_pair_no_delta(): + questions = {"questions": [{"id": "q1", "question": "Q1?", "source_type": "manual", "topic": "t"}, + {"id": "q2", "question": "Q2?", "source_type": "manual", "topic": "t"}]} + out = json.loads(ReportGenerator().generate_report({"evaluations": ISSUE_INPUT}, questions, "json")) + assert out["clusters"][0]["delta_avg"] is None + assert out["clusters"][0]["count"] == 2 + assert out["sources_summary"][0]["delta_avg"] is None + + +def test_eval_report_prints_an_em_dash_for_the_missing_delta(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": ISSUE_INPUT}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + text = md.read_text(encoding="utf-8") + assert "a paired difference leaves the float range, so there is no delta" in text + assert "_Not computed: the difference of 1.7e+308 and -1.7e+308 leaves the float range._" in text diff --git a/dokimi/tests/test_stats_float_range.py b/dokimi/tests/test_stats_float_range.py index 9a67f0308..f78e7e8bc 100644 --- a/dokimi/tests/test_stats_float_range.py +++ b/dokimi/tests/test_stats_float_range.py @@ -75,10 +75,9 @@ def test_paired_test_keeps_the_meaning_of_a_non_finite_input(bad): assert result["significant"] is None -def test_paired_summary_paired_delta_and_arm_significance_refuse_the_same_deltas(): +def test_paired_delta_and_arm_significance_refuse_deltas_beyond_the_range_while_paired_summary_has_none(): evals = _evals(BIG, -BIG) - with pytest.raises(OverflowError): - stats.paired_summary(evals) + assert stats.paired_summary(evals).delta_mean is None # no raise: #830 with pytest.raises(OverflowError): stats.paired_delta(evals, "without_docs", "with_docs") with pytest.raises(OverflowError): From 72eff4ad831870477c507638b8748590dd548175 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 11:33:33 +0000 Subject: [PATCH 2/5] test(eval): pin the Delta line of cmd_evaluate and the agreement line of the eval report for a summary with no delta (#830) The `Delta:` print of `cmd_evaluate` for a summary whose `delta_avg` is None and the `Dynamic vs Static agreement` bullet when one set has no delta had no test. One test each now: `cmd_evaluate` with a mocked pipeline result, and `generate_report` on an evaluations file with a golden set and a generated set where the generated one overflows. Signed-off-by: Anton Volkov --- dokimi/tests/test_paired_summary_overflow.py | 38 ++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/dokimi/tests/test_paired_summary_overflow.py b/dokimi/tests/test_paired_summary_overflow.py index 50334a80e..3f5c5ddf9 100644 --- a/dokimi/tests/test_paired_summary_overflow.py +++ b/dokimi/tests/test_paired_summary_overflow.py @@ -99,6 +99,44 @@ def test_report_generator_group_tables_give_a_group_with_an_overflowing_pair_no_ assert out["sources_summary"][0]["delta_avg"] is None +def test_eval_report_agreement_line_says_none_when_the_dynamic_set_has_no_delta(tmp_path): + rows = [_row("onetbb-Q001", 80.0, 60.0), _row("onetbb-Q002", 70.0, 50.0), _row("gen-1", BIG, -BIG), _row("gen-2", 0.0, 0.0)] + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": rows}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + text = md.read_text(encoding="utf-8") + assert "- **Dynamic vs Static agreement:** none — a set has no delta (a paired difference leaves the float range)." in text + + +def test_cmd_evaluate_prints_a_dash_for_the_delta_of_a_summary_without_one(monkeypatch, capsys): + import argparse + + from dokimi import orchestrator + from dokimi.commands import orchestrate + + result = {"steps": {"personas": {"skipped": True}, "questions_merged": {"path": "q"}, "answers": {"path": "a"}, + "report": {"path": "r"}, + "evaluation": {"path": "e", "summary": {"with_avg": 1.0, "without_avg": 2.0, "delta_avg": None}}}} + + class FakePipeline: + def __init__(self, **kwargs): + pass + + def run(self): + return result + + monkeypatch.setattr(orchestrator, "EvaluationPipeline", FakePipeline) + args = argparse.Namespace(repo="owner/repo", description=None, product="demo", output_dir="out", custom_questions=None, + provider="openai", model="m", judge_provider="openai", judge_model="j", judge_temperature=None, + personas_count=1, questions_per_topic=1, top_k=None, rerank_threshold=0.3, debug_retrieval=False) + orchestrate.cmd_evaluate(args) + out = capsys.readouterr().out + assert " Delta: — (a paired difference leaves the float range)" in out + assert "Context avg: 1.0" in out + + def test_eval_report_prints_an_em_dash_for_the_missing_delta(tmp_path): path = tmp_path / "eval" / "lib.json" path.parent.mkdir() From c97bcee813406d9130ccd67d926e895bf9180d2b Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 12:34:27 +0000 Subject: [PATCH 3/5] fix(eval): the trust gate reads a min_delta beyond the float range as not measured Settles the review point that evaluate_trust still subtracted the arm means directly, so a report whose paired delta is unavailable (three pairs of 1.7e308 and -1.7e308) wrote Infinity as the min_delta value and marked it passed. The check now has value None, measured False and a "Not measured" message when the difference of the means is not finite, so the JSON stays standard and the verdict lists min_delta under not_measured. A regression test parses the report JSON with non-finite constants rejected. Signed-off-by: Anton Volkov --- ...1117_130000_trust-min-delta-float-range.md | 8 +++++ dokimi/dokimi/gate/trust_gate.py | 29 ++++++++++++++----- dokimi/tests/test_paired_summary_overflow.py | 16 ++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md diff --git a/dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md b/dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md new file mode 100644 index 000000000..ccc5c2d8e --- /dev/null +++ b/dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **The trust gate reads a `min_delta` whose arm means differ by more than the float range as not measured.** `report generate --format json` wrote `Infinity` (not standard JSON) as that check's value and marked it passed for scores such as 1.7e308 and -1.7e308. The check now has `value` null, `measured` false and a "Not measured" message, and the verdict lists `min_delta` under `not_measured` (#830). diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py index 634585657..273506259 100644 --- a/dokimi/dokimi/gate/trust_gate.py +++ b/dokimi/dokimi/gate/trust_gate.py @@ -11,6 +11,7 @@ from __future__ import annotations +import math import statistics from typing import Any, Dict, Iterable, List, Optional @@ -182,14 +183,26 @@ def evaluate_trust( # ── 4. Minimum delta (context arm minus baseline) ───────────────────── delta_avg = statistics.mean(with_scores) - statistics.mean(without_scores) - checks.append(Check( - name="min_delta", - passed=delta_avg >= cfg["min_delta"], - value=round(delta_avg, 1), - threshold=cfg["min_delta"], - message=f"Avg delta (context arm minus baseline) = {delta_avg:+.1f} (min {cfg['min_delta']:+})", - severity="warn", - )) + if math.isfinite(delta_avg): + checks.append(Check( + name="min_delta", + passed=delta_avg >= cfg["min_delta"], + value=round(delta_avg, 1), + threshold=cfg["min_delta"], + message=f"Avg delta (context arm minus baseline) = {delta_avg:+.1f} (min {cfg['min_delta']:+})", + severity="warn", + )) + else: + # The arm means differ by more than the float range: no delta to judge, as in the stats. + checks.append(Check( + name="min_delta", + passed=True, + value=None, + threshold=cfg["min_delta"], + message="Not measured: the arm means differ by more than the float range", + severity="warn", + measured=False, + )) # ── 5. Score coefficient of variation (stability) ───────────────────── with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0 diff --git a/dokimi/tests/test_paired_summary_overflow.py b/dokimi/tests/test_paired_summary_overflow.py index 3f5c5ddf9..a8f0a43f7 100644 --- a/dokimi/tests/test_paired_summary_overflow.py +++ b/dokimi/tests/test_paired_summary_overflow.py @@ -14,6 +14,7 @@ import pytest from dokimi.eval import stats +from dokimi.gate.trust_gate import evaluate_trust from dokimi.orchestrator.pipeline import EvaluationPipeline from dokimi.report.eval_report import generate_report from dokimi.report.generator import ReportGenerator @@ -90,6 +91,21 @@ def test_report_generator_prints_an_em_dash_and_a_note_for_the_missing_delta(): assert (out["doc_improvements"], out["doc_degradations"]) == (1, 0) +def _reject_constant(name): + raise AssertionError(f"non-standard JSON constant {name}") + + +def test_report_json_trust_reports_min_delta_not_measured_and_is_strict_json(): + rows = [_row(f"q{i}", BIG, -BIG) for i in range(3)] + text = ReportGenerator().generate_report({"evaluations": rows}, {"questions": []}, "json") + out = json.loads(text, parse_constant=_reject_constant) + check = next(c for c in out["trust"]["checks"] if c["name"] == "min_delta") + assert (check["value"], check["measured"], check["passed"]) == (None, False, True) + assert "Not measured" in check["message"] + assert "min_delta" in evaluate_trust(rows).not_measured + assert out["trust"]["not_measured"] == evaluate_trust(rows).not_measured + + def test_report_generator_group_tables_give_a_group_with_an_overflowing_pair_no_delta(): questions = {"questions": [{"id": "q1", "question": "Q1?", "source_type": "manual", "topic": "t"}, {"id": "q2", "question": "Q2?", "source_type": "manual", "topic": "t"}]} From 6462ca493a21d5226c0fb1fa91ec6f21227846ad Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 12:43:55 +0000 Subject: [PATCH 4/5] fix(eval): the no-delta note agrees in number with the pairs it counts Settles review thread 4599606890: the report note read "1 complete pairs differ" for the issue's own input. It now reads "1 complete pair differs" for one pair and "N complete pairs differ" otherwise, and the tests pin both. Signed-off-by: Anton Volkov --- dokimi/dokimi/report/generator.py | 3 ++- dokimi/tests/test_paired_summary_overflow.py | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/report/generator.py b/dokimi/dokimi/report/generator.py index f1bbb0972..10f899369 100644 --- a/dokimi/dokimi/report/generator.py +++ b/dokimi/dokimi/report/generator.py @@ -563,7 +563,8 @@ def _complete_pairs(evaluations: Sequence[Record]) -> List[Record]: def _unpaired_note(stats: Dict[str, Any]) -> List[str]: """A line saying what the overall means leave out; nothing when every question is paired.""" overflow = stats.get("delta_overflow_questions", 0) - note = [f"_No delta: {overflow} complete pairs differ by more than the float range._", ""] if overflow else [] + pairs = "pair differs" if overflow == 1 else "pairs differ" + note = [f"_No delta: {overflow} complete {pairs} by more than the float range._", ""] if overflow else [] if not stats["unpaired_questions"]: return note if not stats["paired_questions"]: diff --git a/dokimi/tests/test_paired_summary_overflow.py b/dokimi/tests/test_paired_summary_overflow.py index a8f0a43f7..ad7453f6d 100644 --- a/dokimi/tests/test_paired_summary_overflow.py +++ b/dokimi/tests/test_paired_summary_overflow.py @@ -83,7 +83,7 @@ def test_report_generator_prints_an_em_dash_and_a_note_for_the_missing_delta(): data = {"evaluations": ISSUE_INPUT} md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") assert "| **—** |" in md - assert "_No delta: 1 complete pairs differ by more than the float range._" in md + assert "_No delta: 1 complete pair differs by more than the float range._" in md assert "no delta and no conclusion" in md out = json.loads(ReportGenerator().generate_report(data, {"questions": []}, "json"))["stats"] assert out["delta"] == {"avg": None, "min": None, "max": None} @@ -91,6 +91,12 @@ def test_report_generator_prints_an_em_dash_and_a_note_for_the_missing_delta(): assert (out["doc_improvements"], out["doc_degradations"]) == (1, 0) +def test_report_generator_note_counts_several_overflowing_pairs_in_the_plural(): + data = {"evaluations": [_row("q1", BIG, -BIG), _row("q2", -BIG, BIG), _row("q3", 1.0, 1.0)]} + md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") + assert "_No delta: 2 complete pairs differ by more than the float range._" in md + + def _reject_constant(name): raise AssertionError(f"non-standard JSON constant {name}") From 4b9e0c6b1aef3b3d3058a1db1ebd19274d2a482a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 13:25:25 +0000 Subject: [PATCH 5/5] test(eval): pin the branch of the Overall advice that has a delta next to the one that has none (#830) The codecov concern on generator.py:526 stays open because the PR's own test file only ever takes the no-delta branch of `_overall_advice`; the other branch was reached by unrelated test files only. The new test draws the hurting and minimal-impact conclusions from a delta that exists and asserts the float-range line is absent, so both branches of the `delta.avg is None` check are pinned by tests/test_paired_summary_overflow.py itself. Signed-off-by: Anton Volkov --- dokimi/tests/test_paired_summary_overflow.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dokimi/tests/test_paired_summary_overflow.py b/dokimi/tests/test_paired_summary_overflow.py index ad7453f6d..b359e5bd3 100644 --- a/dokimi/tests/test_paired_summary_overflow.py +++ b/dokimi/tests/test_paired_summary_overflow.py @@ -91,6 +91,15 @@ def test_report_generator_prints_an_em_dash_and_a_note_for_the_missing_delta(): assert (out["doc_improvements"], out["doc_degradations"]) == (1, 0) +def test_report_generator_draws_the_conclusion_from_a_delta_that_exists(): + hurting = ReportGenerator().generate_report({"evaluations": [_row("q1", 50.0, 70.0)]}, {"questions": []}, "markdown") + assert "Documentation is hurting answer quality" in hurting + assert "no delta and no conclusion" not in hurting + zero = ReportGenerator().generate_report({"evaluations": [_row("q1", 60.0, 60.0)]}, {"questions": []}, "markdown") + assert "Documentation has minimal impact" in zero + assert "leaves the float range" not in zero + + def test_report_generator_note_counts_several_overflowing_pairs_in_the_plural(): data = {"evaluations": [_row("q1", BIG, -BIG), _row("q2", -BIG, BIG), _row("q3", 1.0, 1.0)]} md = ReportGenerator().generate_report(data, {"questions": []}, "markdown")