Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **The evaluate pipeline and the reports finish when a paired score difference leaves the float range.** `paired_summary` raised `OverflowError` for scores such as 1.7e308 and -1.7e308 in one pair, which ended `dokimi orchestrate` after the evaluations were saved and failed `report generate` and `report eval`. It now returns the arm means, ranges and counts with no delta (`delta_mean`, `delta_min` and `delta_max` are `None`, `delta_overflow` counts the pairs); improvements and degradations of such a pair are counted by comparing its two scores. The pipeline prints `delta=none`, the reports print an em dash with a one-line note, and `report eval` names the refused significance test instead of stopping (#830).
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **The trust gate reads a `min_delta` whose arm means differ by more than the float range as not measured.** `report generate --format json` wrote `Infinity` (not standard JSON) as that check's value and marked it passed for scores such as 1.7e308 and -1.7e308. The check now has `value` null, `measured` false and a "Not measured" message, and the verdict lists `min_delta` under `not_measured` (#830).
3 changes: 2 additions & 1 deletion dokimi/dokimi/commands/orchestrate.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ def cmd_evaluate(args: argparse.Namespace) -> None:
print("\nResults:")
print(f" Context avg: {summary['with_avg']}")
print(f" Baseline avg: {summary['without_avg']}")
print(f" Delta: {summary['delta_avg']:+.1f}")
print(" Delta: " + ("— (a paired difference leaves the float range)"
if summary["delta_avg"] is None else f"{summary['delta_avg']:+.1f}"))

print(f"\n📊 View full report: cat {results['steps']['report']['path']}")

Expand Down
40 changes: 29 additions & 11 deletions dokimi/dokimi/eval/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -469,16 +469,25 @@ class PairedSummary:
unpaired: int
with_mean: float
without_mean: float
delta_mean: float
delta_mean: Optional[float]
with_min: float
with_max: float
without_min: float
without_max: float
delta_min: float
delta_max: float
delta_min: Optional[float]
delta_max: Optional[float]
improvements: int
degradations: int
neutral: int
delta_overflow: int = 0


def _pair_delta(treatment: float, control: float) -> Optional[float]:
"""``treatment - control``, or ``None`` where the difference of two finite scores leaves the float range."""
try:
return checked_delta(treatment, control)
except OverflowError:
return None


def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary:
Expand All @@ -488,27 +497,36 @@ def paired_summary(evaluations: Iterable[RecordLike]) -> PairedSummary:
per-question deltas (``context - baseline``). Averaging each arm over *every* scored
question instead gives two means over different questions, whose difference is not
the delta the page prints beside them. An empty input gives zeros throughout.

A pair whose two finite scores differ by more than the float range has no delta: the
delta mean, min and max are then ``None`` (``delta_overflow`` counts those pairs) and the
arm means, ranges and counts are kept. Such a pair is counted as an improvement or a
degradation by comparing its two scores, not their difference.
"""
# https://github.com/intel/dokimi/issues/830
recs = records(evaluations, source="evaluations", kind="evaluations")
with_s, without_s = paired_vectors(recs, CONTEXT, BASELINE)
deltas = _deltas(with_s, without_s)
paired = len(deltas)
rows = [(t, c, _pair_delta(t, c)) for t, c in zip(with_s, without_s, strict=True)]
deltas = [d for _, _, d in rows if d is not None]
overflow = len(rows) - len(deltas)
paired = len(rows)
return PairedSummary(
total=len(recs),
paired=paired,
unpaired=len(recs) - paired,
with_mean=_mean(with_s) if paired else 0.0,
without_mean=_mean(without_s) if paired else 0.0,
delta_mean=_mean(deltas) if paired else 0.0,
delta_mean=None if overflow else _mean(deltas) if paired else 0.0,
with_min=min(with_s, default=0.0),
with_max=max(with_s, default=0.0),
without_min=min(without_s, default=0.0),
without_max=max(without_s, default=0.0),
delta_min=min(deltas, default=0.0),
delta_max=max(deltas, default=0.0),
improvements=sum(1 for d in deltas if d > 0),
degradations=sum(1 for d in deltas if d < 0),
neutral=sum(1 for d in deltas if d == 0),
delta_min=None if overflow else min(deltas, default=0.0),
delta_max=None if overflow else max(deltas, default=0.0),
improvements=sum(1 for t, c, d in rows if (t > c if d is None else d > 0)),
degradations=sum(1 for t, c, d in rows if (t < c if d is None else d < 0)),
neutral=sum(1 for _, _, d in rows if d == 0),
delta_overflow=overflow,
)


Expand Down
29 changes: 21 additions & 8 deletions dokimi/dokimi/gate/trust_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from __future__ import annotations

import math
import statistics
from typing import Any, Dict, Iterable, List, Optional

Expand Down Expand Up @@ -182,14 +183,26 @@ def evaluate_trust(

# ── 4. Minimum delta (context arm minus baseline) ─────────────────────
delta_avg = statistics.mean(with_scores) - statistics.mean(without_scores)
checks.append(Check(
name="min_delta",
passed=delta_avg >= cfg["min_delta"],
value=round(delta_avg, 1),
threshold=cfg["min_delta"],
message=f"Avg delta (context arm minus baseline) = {delta_avg:+.1f} (min {cfg['min_delta']:+})",
severity="warn",
))
if math.isfinite(delta_avg):
checks.append(Check(
name="min_delta",
passed=delta_avg >= cfg["min_delta"],
value=round(delta_avg, 1),
threshold=cfg["min_delta"],
message=f"Avg delta (context arm minus baseline) = {delta_avg:+.1f} (min {cfg['min_delta']:+})",
severity="warn",
))
else:
# The arm means differ by more than the float range: no delta to judge, as in the stats.
checks.append(Check(
name="min_delta",
passed=True,
value=None,
threshold=cfg["min_delta"],
message="Not measured: the arm means differ by more than the float range",
severity="warn",
measured=False,
))

# ── 5. Score coefficient of variation (stability) ─────────────────────
with_std = statistics.stdev(with_scores) if len(with_scores) > 1 else 0.0
Expand Down
6 changes: 4 additions & 2 deletions dokimi/dokimi/orchestrator/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,12 +427,14 @@ def run(self, concurrency: int = 5) -> Dict[str, Any]:
results["steps"]["evaluation"]["summary"] = {
"with_avg": round(summary.with_mean, 1),
"without_avg": round(summary.without_mean, 1),
"delta_avg": round(summary.delta_mean, 1)
"delta_avg": None if summary.delta_mean is None else round(summary.delta_mean, 1)
}
delta_avg = results["steps"]["evaluation"]["summary"]["delta_avg"]
print(f"✓ Evaluated {len(evaluations)} answers: "
f"context={results['steps']['evaluation']['summary']['with_avg']}, "
f"baseline={results['steps']['evaluation']['summary']['without_avg']}, "
f"delta={results['steps']['evaluation']['summary']['delta_avg']:+.1f}"
+ ("delta=none (a paired difference leaves the float range)" if delta_avg is None
else f"delta={delta_avg:+.1f}")
+ (f" (over the {summary.paired} scored in both arms; {summary.unpaired} left out)"
if summary.unpaired else ""))
else:
Expand Down
21 changes: 19 additions & 2 deletions dokimi/dokimi/report/eval_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ def significance_test(evals):
return result


def _significance_or_refusal(evals) -> tuple[dict | None, list[str]]:
"""The significance test, or the lines that say a paired difference beyond the float range refused it."""
try:
return significance_test(evals), []
except OverflowError as exc:
return None, ["## Statistical Significance", f"_Not computed: {exc}._", ""]


def has_scores(e: Record) -> bool:
return e.score(CONTEXT) is not None and e.score(BASELINE) is not None

Expand Down Expand Up @@ -89,7 +97,7 @@ def stats(qs):
"unpaired": s.unpaired,
"with_avg": round(s.with_mean, 1),
"without_avg": round(s.without_mean, 1),
"delta_avg": round(s.delta_mean, 1),
"delta_avg": None if s.delta_mean is None else round(s.delta_mean, 1),
"improvements": s.improvements,
"degradations": s.degradations,
"neutral": s.neutral,
Expand Down Expand Up @@ -118,13 +126,17 @@ def _summary_cells(st: dict) -> tuple[str, str, str]:
"""The context, baseline and delta cells of a Summary row: an em dash when no question is paired."""
if not st["valid"]:
return "—", "—", "—"
if st["delta_avg"] is None:
return str(st["with_avg"]), str(st["without_avg"]), "—"
return str(st["with_avg"]), str(st["without_avg"]), f'**{st["delta_avg"]:+}**'


def _overall_line(st: dict) -> str:
"""The Overall delta bullet, or a note that no question is paired."""
if not st["valid"]:
return "- **Overall delta:** none — no complete pairs (a question scored in both arms), so there is no delta."
if st["delta_avg"] is None:
return "- **Overall delta:** none — a paired difference leaves the float range, so there is no delta."
verdict = "context arm is helping." if st["delta_avg"] > 2 else "context arm is neutral or slightly harmful."
return f'- **Overall delta: {st["delta_avg"]:+}** — {verdict}'

Expand All @@ -133,6 +145,8 @@ def _agreement_line(dyn: dict, static: dict) -> str:
"""The Dynamic vs Static bullet; no agreement is claimed when either set has no complete pair."""
if not (dyn["valid"] and static["valid"]):
return "- **Dynamic vs Static agreement:** none — a set has no complete pairs, so there is no delta to compare."
if dyn["delta_avg"] is None or static["delta_avg"] is None:
return "- **Dynamic vs Static agreement:** none — a set has no delta (a paired difference leaves the float range)."
gap = abs(dyn["delta_avg"] - static["delta_avg"])
verdict = "good consistency, benchmark is stable." if gap < 3 else "notable gap — investigate question quality."
return f'- **Dynamic vs Static agreement:** {dyn["delta_avg"]:+} vs {static["delta_avg"]:+} — {verdict}'
Expand All @@ -141,6 +155,8 @@ def _agreement_line(dyn: dict, static: dict) -> str:
def _delta_summary(st: dict) -> str:
if not st["valid"]:
return " Delta: none (no complete pairs)"
if st["delta_avg"] is None:
return f" Delta: none (a paired difference leaves the float range; with={st['with_avg']}, without={st['without_avg']})"
return f" Delta: {st['delta_avg']:+} (with={st['with_avg']}, without={st['without_avg']})"


Expand Down Expand Up @@ -321,7 +337,8 @@ def generate_report(eval_path: str, out_path: str, qa_json_out: str | None = Non
lines += _unpaired_note(all_stats)

# Statistical significance
sig = significance_test(evals)
sig, refusal = _significance_or_refusal(evals)
lines += refusal
if sig:
sig_label = ("✅ **Statistically significant** (p < 0.05)" if sig.get("significant")
else "❌ **Not statistically significant** (p ≥ 0.05)" if sig.get("significant") is False
Expand Down
22 changes: 15 additions & 7 deletions dokimi/dokimi/report/generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ def _compute_stats(self, evaluations: Sequence[Record]) -> Dict[str, Any]:
"total_questions": s.total,
"paired_questions": s.paired,
"unpaired_questions": s.unpaired,
"delta_overflow_questions": s.delta_overflow,
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
"with_docs": _stat_block(s, s.with_mean, s.with_min, s.with_max),
"without_docs": _stat_block(s, s.without_mean, s.without_min, s.without_max),
"delta": _stat_block(s, s.delta_mean, s.delta_min, s.delta_max),
Expand Down Expand Up @@ -501,10 +502,12 @@ def _format_markdown(
return "\n".join(lines)


def _stat_block(s: Any, mean: float, low: float, high: float) -> Dict[str, Optional[float]]:
"""avg, min and max of one statistic, rounded; all None when no question is a complete pair."""
if not s.paired:
return {"avg": None, "min": None, "max": None} # no complete pair: no value, not a zero
def _stat_block(
s: Any, mean: Optional[float], low: Optional[float], high: Optional[float]
) -> Dict[str, Optional[float]]:
"""avg, min and max of one statistic, rounded; all None when no question is a complete pair or the statistic has no value."""
if not s.paired or mean is None or low is None or high is None:
return {"avg": None, "min": None, "max": None} # no value, not a zero
return {"avg": round(mean, 1), "min": round(low, 1), "max": round(high, 1)}


Expand All @@ -519,6 +522,8 @@ def _overall_advice(stats: Dict[str, Any]) -> List[str]:
"""The Overall recommendation lines; no conclusion is drawn without a complete pair."""
if not stats["paired_questions"]:
return ["⚪ **Overall:** no complete pairs, so no conclusion can be drawn."]
if stats["delta"]["avg"] is None:
return ["⚪ **Overall:** a paired difference leaves the float range, so there is no delta and no conclusion."]
if stats["delta"]["avg"] < 0:
return ["⚠️ **Overall:** Documentation is hurting answer quality. Consider:",
"- Improving retrieval relevance",
Expand All @@ -541,7 +546,7 @@ def _arm_means(evaluations: Sequence[Record]) -> Dict[str, Any]:
"unpaired": s.unpaired,
"with_avg": None if not s.paired else round(s.with_mean, 1),
"without_avg": None if not s.paired else round(s.without_mean, 1),
"delta_avg": None if not s.paired else round(s.delta_mean, 1),
"delta_avg": None if s.delta_mean is None or not s.paired else round(s.delta_mean, 1),
}


Expand All @@ -557,13 +562,16 @@ def _complete_pairs(evaluations: Sequence[Record]) -> List[Record]:

def _unpaired_note(stats: Dict[str, Any]) -> List[str]:
"""A line saying what the overall means leave out; nothing when every question is paired."""
overflow = stats.get("delta_overflow_questions", 0)
pairs = "pair differs" if overflow == 1 else "pairs differ"
note = [f"_No delta: {overflow} complete {pairs} by more than the float range._", ""] if overflow else []
if not stats["unpaired_questions"]:
return []
return note
if not stats["paired_questions"]:
return [f"_No complete pairs: {stats['unpaired_questions']} unpaired questions, "
"so no arm mean or delta is given._", ""]
return [
f"_Averages, min/max, delta and the top/bottom tables are over the {stats['paired_questions']} complete pairs; "
f"{stats['unpaired_questions']} unpaired (scored in one arm only) left out._",
"",
]
] + note
Loading
Loading