diff --git a/dokimi/changelog.d/20261117_120000_paired-summary-overflow.md b/dokimi/changelog.d/20261117_120000_paired-summary-overflow.md new file mode 100644 index 000000000..b0c2b64cd --- /dev/null +++ b/dokimi/changelog.d/20261117_120000_paired-summary-overflow.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **The evaluate pipeline and the reports finish when a paired score difference leaves the float range.** `paired_summary` raised `OverflowError` for scores such as 1.7e308 and -1.7e308 in one pair, which ended `dokimi orchestrate` after the evaluations were saved and failed `report generate` and `report eval`. It now returns the arm means, ranges and counts with no delta (`delta_mean`, `delta_min` and `delta_max` are `None`, `delta_overflow` counts the pairs); improvements and degradations of such a pair are counted by comparing its two scores. The pipeline prints `delta=none`, the reports print an em dash with a one-line note, and `report eval` names the refused significance test instead of stopping (#830). diff --git a/dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md b/dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md new file mode 100644 index 000000000..ccc5c2d8e --- /dev/null +++ b/dokimi/changelog.d/20261117_130000_trust-min-delta-float-range.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **The trust gate reads a `min_delta` whose arm means differ by more than the float range as not measured.** `report generate --format json` wrote `Infinity` (not standard JSON) as that check's value and marked it passed for scores such as 1.7e308 and -1.7e308. The check now has `value` null, `measured` false and a "Not measured" message, and the verdict lists `min_delta` under `not_measured` (#830). diff --git a/dokimi/dokimi/commands/orchestrate.py b/dokimi/dokimi/commands/orchestrate.py index a3ae425d9..981f02bdd 100644 --- a/dokimi/dokimi/commands/orchestrate.py +++ b/dokimi/dokimi/commands/orchestrate.py @@ -102,7 +102,8 @@ def cmd_evaluate(args: argparse.Namespace) -> None: print("\nResults:") print(f" Context avg: {summary['with_avg']}") print(f" Baseline avg: {summary['without_avg']}") - print(f" Delta: {summary['delta_avg']:+.1f}") + print(" Delta: " + ("— (a paired difference leaves the float range)" + if summary["delta_avg"] is None else f"{summary['delta_avg']:+.1f}")) print(f"\nšŸ“Š View full report: cat {results['steps']['report']['path']}") diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index d0a58becb..6fa61bc04 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -469,16 +469,25 @@ class PairedSummary: unpaired: int with_mean: float without_mean: float - delta_mean: float + delta_mean: Optional[float] with_min: float with_max: float without_min: float without_max: float - delta_min: float - delta_max: float + delta_min: Optional[float] + delta_max: Optional[float] improvements: int degradations: int neutral: int + delta_overflow: int = 0 + + +def _pair_delta(treatment: float, control: float) -> Optional[float]: + """``treatment - control``, or ``None`` where the difference of two finite scores leaves the float range.""" + try: + return checked_delta(treatment, control) + except OverflowError: + return None def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary: @@ -488,27 +497,36 @@ def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary: per-question deltas (``context - baseline``). Averaging each arm over *every* scored question instead gives two means over different questions, whose difference is not the delta the page prints beside them. An empty input gives zeros throughout. + + A pair whose two finite scores differ by more than the float range has no delta: the + delta mean, min and max are then ``None`` (``delta_overflow`` counts those pairs) and the + arm means, ranges and counts are kept. Such a pair is counted as an improvement or a + degradation by comparing its two scores, not their difference. """ + # https://github.com/intel/dokimi/issues/830 recs = records(evaluations, source="evaluations", kind="evaluations") with_s, without_s = paired_vectors(recs, CONTEXT, BASELINE) - deltas = _deltas(with_s, without_s) - paired = len(deltas) + rows = [(t, c, _pair_delta(t, c)) for t, c in zip(with_s, without_s, strict=True)] + deltas = [d for _, _, d in rows if d is not None] + overflow = len(rows) - len(deltas) + paired = len(rows) return PairedSummary( total=len(recs), paired=paired, unpaired=len(recs) - paired, with_mean=_mean(with_s) if paired else 0.0, without_mean=_mean(without_s) if paired else 0.0, - delta_mean=_mean(deltas) if paired else 0.0, + delta_mean=None if overflow else _mean(deltas) if paired else 0.0, with_min=min(with_s, default=0.0), with_max=max(with_s, default=0.0), without_min=min(without_s, default=0.0), without_max=max(without_s, default=0.0), - delta_min=min(deltas, default=0.0), - delta_max=max(deltas, default=0.0), - improvements=sum(1 for d in deltas if d > 0), - degradations=sum(1 for d in deltas if d < 0), - neutral=sum(1 for d in deltas if d == 0), + delta_min=None if overflow else min(deltas, default=0.0), + delta_max=None if overflow else max(deltas, default=0.0), + improvements=sum(1 for t, c, d in rows if (t > c if d is None else d > 0)), + degradations=sum(1 for t, c, d in rows if (t < c if d is None else d < 0)), + neutral=sum(1 for _, _, d in rows if d == 0), + delta_overflow=overflow, ) diff --git a/dokimi/dokimi/gate/trust_gate.py b/dokimi/dokimi/gate/trust_gate.py index 634585657..273506259 100644 --- a/dokimi/dokimi/gate/trust_gate.py +++ b/dokimi/dokimi/gate/trust_gate.py @@ -11,6 +11,7 @@ from __future__ import annotations +import math import statistics from typing import Any, Dict, Iterable, List, Optional @@ -182,14 +183,26 @@ def evaluate_trust( # ── 4. Minimum delta (context arm minus baseline) ───────────────────── delta_avg = statistics.mean(with_scores) - statistics.mean(without_scores) - checks.append(Check( - name="min_delta", - passed=delta_avg >= cfg["min_delta"], - value=round(delta_avg, 1), - threshold=cfg["min_delta"], - message=f"Avg delta (context arm minus baseline) = {delta_avg:+.1f} (min {cfg['min_delta']:+})", - severity="warn", - )) + if math.isfinite(delta_avg): + checks.append(Check( + name="min_delta", + passed=delta_avg >= cfg["min_delta"], + value=round(delta_avg, 1), + threshold=cfg["min_delta"], + message=f"Avg delta (context arm minus baseline) = {delta_avg:+.1f} (min {cfg['min_delta']:+})", + severity="warn", + )) + else: + # The arm means differ by more than the float range: no delta to judge, as in the stats. + checks.append(Check( + name="min_delta", + passed=True, + value=None, + threshold=cfg["min_delta"], + message="Not measured: the arm means differ by more than the float range", + severity="warn", + measured=False, + )) # ── 5. Score coefficient of variation (stability) ───────────────────── with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0 diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index a5d57ceed..afd257a97 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -427,12 +427,14 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]: results["steps"]["evaluation"]["summary"] = { "with_avg": round(summary.with_mean, 1), "without_avg": round(summary.without_mean, 1), - "delta_avg": round(summary.delta_mean, 1) + "delta_avg": None if summary.delta_mean is None else round(summary.delta_mean, 1) } + delta_avg = results["steps"]["evaluation"]["summary"]["delta_avg"] print(f"āœ“ Evaluated {len(evaluations)} answers: " f"context={results['steps']['evaluation']['summary']['with_avg']}, " f"baseline={results['steps']['evaluation']['summary']['without_avg']}, " - f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}" + + ("delta=none (a paired difference leaves the float range)" if delta_avg is None + else f"delta={delta_avg:+.1f}") + (f" (over the {summary.paired} scored in both arms; {summary.unpaired} left out)" if summary.unpaired else "")) else: diff --git a/dokimi/dokimi/report/eval_report.py b/dokimi/dokimi/report/eval_report.py index c64df4722..3b0d1e558 100755 --- a/dokimi/dokimi/report/eval_report.py +++ b/dokimi/dokimi/report/eval_report.py @@ -60,6 +60,14 @@ def significance_test(evals): return result +def _significance_or_refusal(evals) -> tuple[dict | None, list[str]]: + """The significance test, or the lines that say a paired difference beyond the float range refused it.""" + try: + return significance_test(evals), [] + except OverflowError as exc: + return None, ["## Statistical Significance", f"_Not computed: {exc}._", ""] + + def has_scores(e: Record) -> bool: return e.score(CONTEXT) is not None and e.score(BASELINE) is not None @@ -89,7 +97,7 @@ def stats(qs): "unpaired": s.unpaired, "with_avg": round(s.with_mean, 1), "without_avg": round(s.without_mean, 1), - "delta_avg": round(s.delta_mean, 1), + "delta_avg": None if s.delta_mean is None else round(s.delta_mean, 1), "improvements": s.improvements, "degradations": s.degradations, "neutral": s.neutral, @@ -118,6 +126,8 @@ def _summary_cells(st: dict) -> tuple[str, str, str]: """The context, baseline and delta cells of a Summary row: an em dash when no question is paired.""" if not st["valid"]: return "—", "—", "—" + if st["delta_avg"] is None: + return str(st["with_avg"]), str(st["without_avg"]), "—" return str(st["with_avg"]), str(st["without_avg"]), f'**{st["delta_avg"]:+}**' @@ -125,6 +135,8 @@ def _overall_line(st: dict) -> str: """The Overall delta bullet, or a note that no question is paired.""" if not st["valid"]: return "- **Overall delta:** none — no complete pairs (a question scored in both arms), so there is no delta." + if st["delta_avg"] is None: + return "- **Overall delta:** none — a paired difference leaves the float range, so there is no delta." verdict = "context arm is helping." if st["delta_avg"] > 2 else "context arm is neutral or slightly harmful." return f'- **Overall delta: {st["delta_avg"]:+}** — {verdict}' @@ -133,6 +145,8 @@ def _agreement_line(dyn: dict, static: dict) -> str: """The Dynamic vs Static bullet; no agreement is claimed when either set has no complete pair.""" if not (dyn["valid"] and static["valid"]): return "- **Dynamic vs Static agreement:** none — a set has no complete pairs, so there is no delta to compare." + if dyn["delta_avg"] is None or static["delta_avg"] is None: + return "- **Dynamic vs Static agreement:** none — a set has no delta (a paired difference leaves the float range)." gap = abs(dyn["delta_avg"] - static["delta_avg"]) verdict = "good consistency, benchmark is stable." if gap < 3 else "notable gap — investigate question quality." return f'- **Dynamic vs Static agreement:** {dyn["delta_avg"]:+} vs {static["delta_avg"]:+} — {verdict}' @@ -141,6 +155,8 @@ def _agreement_line(dyn: dict, static: dict) -> str: def _delta_summary(st: dict) -> str: if not st["valid"]: return " Delta: none (no complete pairs)" + if st["delta_avg"] is None: + return f" Delta: none (a paired difference leaves the float range; with={st['with_avg']}, without={st['without_avg']})" return f" Delta: {st['delta_avg']:+} (with={st['with_avg']}, without={st['without_avg']})" @@ -321,7 +337,8 @@ def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = Non lines += _unpaired_note(all_stats) # Statistical significance - sig = significance_test(evals) + sig, refusal = _significance_or_refusal(evals) + lines += refusal if sig: sig_label = ("āœ… **Statistically significant** (p < 0.05)" if sig.get("significant") else "āŒ **Not statistically significant** (p ≄ 0.05)" if sig.get("significant") is False diff --git a/dokimi/dokimi/report/generator.py b/dokimi/dokimi/report/generator.py index 532d0bf5f..10f899369 100644 --- a/dokimi/dokimi/report/generator.py +++ b/dokimi/dokimi/report/generator.py @@ -158,6 +158,7 @@ def _compute_stats(self, evaluations: Sequence[Record]) -> Dict[str, Any]: "total_questions": s.total, "paired_questions": s.paired, "unpaired_questions": s.unpaired, + "delta_overflow_questions": s.delta_overflow, "with_docs": _stat_block(s, s.with_mean, s.with_min, s.with_max), "without_docs": _stat_block(s, s.without_mean, s.without_min, s.without_max), "delta": _stat_block(s, s.delta_mean, s.delta_min, s.delta_max), @@ -501,10 +502,12 @@ def _format_markdown( return "\n".join(lines) -def _stat_block(s: Any, mean: float, low: float, high: float) -> Dict[str, Optional[float]]: - """avg, min and max of one statistic, rounded; all None when no question is a complete pair.""" - if not s.paired: - return {"avg": None, "min": None, "max": None} # no complete pair: no value, not a zero +def _stat_block( + s: Any, mean: Optional[float], low: Optional[float], high: Optional[float] +) -> Dict[str, Optional[float]]: + """avg, min and max of one statistic, rounded; all None when no question is a complete pair or the statistic has no value.""" + if not s.paired or mean is None or low is None or high is None: + return {"avg": None, "min": None, "max": None} # no value, not a zero return {"avg": round(mean, 1), "min": round(low, 1), "max": round(high, 1)} @@ -519,6 +522,8 @@ def _overall_advice(stats: Dict[str, Any]) -> List[str]: """The Overall recommendation lines; no conclusion is drawn without a complete pair.""" if not stats["paired_questions"]: return ["⚪ **Overall:** no complete pairs, so no conclusion can be drawn."] + if stats["delta"]["avg"] is None: + return ["⚪ **Overall:** a paired difference leaves the float range, so there is no delta and no conclusion."] if stats["delta"]["avg"] < 0: return ["āš ļø **Overall:** Documentation is hurting answer quality. Consider:", "- Improving retrieval relevance", @@ -541,7 +546,7 @@ def _arm_means(evaluations: Sequence[Record]) -> Dict[str, Any]: "unpaired": s.unpaired, "with_avg": None if not s.paired else round(s.with_mean, 1), "without_avg": None if not s.paired else round(s.without_mean, 1), - "delta_avg": None if not s.paired else round(s.delta_mean, 1), + "delta_avg": None if s.delta_mean is None or not s.paired else round(s.delta_mean, 1), } @@ -557,8 +562,11 @@ def _complete_pairs(evaluations: Sequence[Record]) -> List[Record]: def _unpaired_note(stats: Dict[str, Any]) -> List[str]: """A line saying what the overall means leave out; nothing when every question is paired.""" + overflow = stats.get("delta_overflow_questions", 0) + pairs = "pair differs" if overflow == 1 else "pairs differ" + note = [f"_No delta: {overflow} complete {pairs} by more than the float range._", ""] if overflow else [] if not stats["unpaired_questions"]: - return [] + return note if not stats["paired_questions"]: return [f"_No complete pairs: {stats['unpaired_questions']} unpaired questions, " "so no arm mean or delta is given._", ""] @@ -566,4 +574,4 @@ def _unpaired_note(stats: Dict[str, Any]) -> List[str]: f"_Averages, min/max, delta and the top/bottom tables are over the {stats['paired_questions']} complete pairs; " f"{stats['unpaired_questions']} unpaired (scored in one arm only) left out._", "", - ] + ] + note diff --git a/dokimi/tests/test_paired_summary_overflow.py b/dokimi/tests/test_paired_summary_overflow.py new file mode 100644 index 000000000..b359e5bd3 --- /dev/null +++ b/dokimi/tests/test_paired_summary_overflow.py @@ -0,0 +1,179 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A paired difference beyond the float range leaves ``paired_summary`` without a delta, not without a result. + +https://github.com/intel/dokimi/issues/830 +""" + +from __future__ import annotations + +import json +import statistics + +import pytest + +from dokimi.eval import stats +from dokimi.gate.trust_gate import evaluate_trust +from dokimi.orchestrator.pipeline import EvaluationPipeline +from dokimi.report.eval_report import generate_report +from dokimi.report.generator import ReportGenerator + +BIG = 1.7e308 + + +def _row(qid, with_score, without_score): + return {"question_id": qid, "question_text": f"Question {qid}?", + "with_docs": {"aggregate": with_score}, "without_docs": {"aggregate": without_score}} + + +# The issue's input: one pair far apart, one pair equal. +ISSUE_INPUT = [_row("q1", BIG, -BIG), _row("q2", 0.0, 0.0)] + + +def test_paired_summary_of_the_issue_input_has_no_delta_and_keeps_the_rest(): + s = stats.paired_summary(ISSUE_INPUT) + assert (s.delta_mean, s.delta_min, s.delta_max) == (None, None, None) + assert s.delta_overflow == 1 + + +def test_paired_summary_keeps_the_arm_means_ranges_and_counts_of_an_overflowing_pair(): + s = stats.paired_summary(ISSUE_INPUT + [_row("q3", 10.0, 4.0), {"question_id": "q4", "with_docs": {"aggregate": 1.0}}]) + assert (s.total, s.paired, s.unpaired) == (4, 3, 1) + assert (s.with_mean, s.without_mean) == (statistics.mean([BIG, 0.0, 10.0]), statistics.mean([-BIG, 0.0, 4.0])) + assert (s.with_min, s.with_max, s.without_min, s.without_max) == (0.0, BIG, -BIG, 4.0) + + +def test_paired_summary_counts_an_overflowing_pair_by_comparing_its_scores(): + s = stats.paired_summary([_row("q1", BIG, -BIG), _row("q2", -BIG, BIG), _row("q3", 5.0, 5.0), _row("q4", 6.0, 5.0)]) + assert (s.improvements, s.degradations, s.neutral) == (2, 1, 1) + + +def test_paired_summary_of_pairs_inside_the_range_still_has_a_delta(): + s = stats.paired_summary([_row("q1", BIG, BIG - 1e300), _row("q2", 80.0, 60.0)]) + assert s.delta_max == pytest.approx(1e300, rel=1e-9) + assert s.delta_min == 20.0 + + +def _run_pipeline(tmp_path, monkeypatch, evaluations): + monkeypatch.setattr(EvaluationPipeline, "_generate_answers", lambda self, *a, **k: []) + monkeypatch.setattr(EvaluationPipeline, "_evaluate_answers", lambda self, *a, **k: evaluations) + monkeypatch.setattr(EvaluationPipeline, "_generate_report", lambda self, *a, **k: "report") + output_dir = tmp_path / "run" + (output_dir / "personas").mkdir(parents=True) + (output_dir / "questions").mkdir() + (output_dir / "personas" / "oneTBB.json").write_text(json.dumps({"personas": []}), encoding="utf-8") + questions = [{"id": e["question_id"], "question": e["question_text"], "source_type": "manual"} for e in evaluations] + (output_dir / "questions" / "oneTBB.json").write_text(json.dumps({"questions": questions}), encoding="utf-8") + pipeline = EvaluationPipeline(product="oneTBB", output_dir=output_dir, description="Threading Building Blocks") + return pipeline.run(concurrency=1) + + +def test_pipeline_reaches_the_report_step_with_the_arm_averages_and_no_delta(tmp_path, monkeypatch, capsys): + result = _run_pipeline(tmp_path, monkeypatch, ISSUE_INPUT) + assert result["status"] == "success" + step = result["steps"]["evaluation"] + assert step["summary"]["delta_avg"] is None + assert (step["paired"], step["unpaired"]) == (2, 0) + assert "delta=none (a paired difference leaves the float range)" in capsys.readouterr().out + assert "report" in result["steps"] + + +def test_report_generator_prints_an_em_dash_and_a_note_for_the_missing_delta(): + data = {"evaluations": ISSUE_INPUT} + md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") + assert "| **—** |" in md + assert "_No delta: 1 complete pair differs by more than the float range._" in md + assert "no delta and no conclusion" in md + out = json.loads(ReportGenerator().generate_report(data, {"questions": []}, "json"))["stats"] + assert out["delta"] == {"avg": None, "min": None, "max": None} + assert out["with_docs"]["avg"] == pytest.approx(BIG / 2, rel=1e-12) + assert (out["doc_improvements"], out["doc_degradations"]) == (1, 0) + + +def test_report_generator_draws_the_conclusion_from_a_delta_that_exists(): + hurting = ReportGenerator().generate_report({"evaluations": [_row("q1", 50.0, 70.0)]}, {"questions": []}, "markdown") + assert "Documentation is hurting answer quality" in hurting + assert "no delta and no conclusion" not in hurting + zero = ReportGenerator().generate_report({"evaluations": [_row("q1", 60.0, 60.0)]}, {"questions": []}, "markdown") + assert "Documentation has minimal impact" in zero + assert "leaves the float range" not in zero + + +def test_report_generator_note_counts_several_overflowing_pairs_in_the_plural(): + data = {"evaluations": [_row("q1", BIG, -BIG), _row("q2", -BIG, BIG), _row("q3", 1.0, 1.0)]} + md = ReportGenerator().generate_report(data, {"questions": []}, "markdown") + assert "_No delta: 2 complete pairs differ by more than the float range._" in md + + +def _reject_constant(name): + raise AssertionError(f"non-standard JSON constant {name}") + + +def test_report_json_trust_reports_min_delta_not_measured_and_is_strict_json(): + rows = [_row(f"q{i}", BIG, -BIG) for i in range(3)] + text = ReportGenerator().generate_report({"evaluations": rows}, {"questions": []}, "json") + out = json.loads(text, parse_constant=_reject_constant) + check = next(c for c in out["trust"]["checks"] if c["name"] == "min_delta") + assert (check["value"], check["measured"], check["passed"]) == (None, False, True) + assert "Not measured" in check["message"] + assert "min_delta" in evaluate_trust(rows).not_measured + assert out["trust"]["not_measured"] == evaluate_trust(rows).not_measured + + +def test_report_generator_group_tables_give_a_group_with_an_overflowing_pair_no_delta(): + questions = {"questions": [{"id": "q1", "question": "Q1?", "source_type": "manual", "topic": "t"}, + {"id": "q2", "question": "Q2?", "source_type": "manual", "topic": "t"}]} + out = json.loads(ReportGenerator().generate_report({"evaluations": ISSUE_INPUT}, questions, "json")) + assert out["clusters"][0]["delta_avg"] is None + assert out["clusters"][0]["count"] == 2 + assert out["sources_summary"][0]["delta_avg"] is None + + +def test_eval_report_agreement_line_says_none_when_the_dynamic_set_has_no_delta(tmp_path): + rows = [_row("onetbb-Q001", 80.0, 60.0), _row("onetbb-Q002", 70.0, 50.0), _row("gen-1", BIG, -BIG), _row("gen-2", 0.0, 0.0)] + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": rows}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + text = md.read_text(encoding="utf-8") + assert "- **Dynamic vs Static agreement:** none — a set has no delta (a paired difference leaves the float range)." in text + + +def test_cmd_evaluate_prints_a_dash_for_the_delta_of_a_summary_without_one(monkeypatch, capsys): + import argparse + + from dokimi import orchestrator + from dokimi.commands import orchestrate + + result = {"steps": {"personas": {"skipped": True}, "questions_merged": {"path": "q"}, "answers": {"path": "a"}, + "report": {"path": "r"}, + "evaluation": {"path": "e", "summary": {"with_avg": 1.0, "without_avg": 2.0, "delta_avg": None}}}} + + class FakePipeline: + def __init__(self, **kwargs): + pass + + def run(self): + return result + + monkeypatch.setattr(orchestrator, "EvaluationPipeline", FakePipeline) + args = argparse.Namespace(repo="owner/repo", description=None, product="demo", output_dir="out", custom_questions=None, + provider="openai", model="m", judge_provider="openai", judge_model="j", judge_temperature=None, + personas_count=1, questions_per_topic=1, top_k=None, rerank_threshold=0.3, debug_retrieval=False) + orchestrate.cmd_evaluate(args) + out = capsys.readouterr().out + assert " Delta: — (a paired difference leaves the float range)" in out + assert "Context avg: 1.0" in out + + +def test_eval_report_prints_an_em_dash_for_the_missing_delta(tmp_path): + path = tmp_path / "eval" / "lib.json" + path.parent.mkdir() + path.write_text(json.dumps({"evaluations": ISSUE_INPUT}), encoding="utf-8") + md = tmp_path / "lib.md" + generate_report(str(path), str(md), str(tmp_path / "lib.qa.json")) + text = md.read_text(encoding="utf-8") + assert "a paired difference leaves the float range, so there is no delta" in text + assert "_Not computed: the difference of 1.7e+308 and -1.7e+308 leaves the float range._" in text diff --git a/dokimi/tests/test_stats_float_range.py b/dokimi/tests/test_stats_float_range.py index 9a67f0308..f78e7e8bc 100644 --- a/dokimi/tests/test_stats_float_range.py +++ b/dokimi/tests/test_stats_float_range.py @@ -75,10 +75,9 @@ def test_paired_test_keeps_the_meaning_of_a_non_finite_input(bad): assert result["significant"] is None -def test_paired_summary_paired_delta_and_arm_significance_refuse_the_same_deltas(): +def test_paired_delta_and_arm_significance_refuse_deltas_beyond_the_range_while_paired_summary_has_none(): evals = _evals(BIG, -BIG) - with pytest.raises(OverflowError): - stats.paired_summary(evals) + assert stats.paired_summary(evals).delta_mean is None # no raise: #830 with pytest.raises(OverflowError): stats.paired_delta(evals, "without_docs", "with_docs") with pytest.raises(OverflowError):