Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions dokimi/changelog.d/20261116_150000_arms-overflow-save.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<!--
Copyright (C) 2026 Intel Corporation
SPDX-License-Identifier: Apache-2.0
-->

### Fixed

- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores are averaged exactly, or refused when a single difference leaves the range, so the saved file holds no `Infinity`; `arms run` prints a refused delta in its console summary. A per-question delta whose scores differ by more than the float range is left out of `evaluations[].deltas_vs_baseline` and named in that row's `delta_errors`.
13 changes: 13 additions & 0 deletions dokimi/docs/benchmarking-and-comparison.md
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,19 @@ is what tells you whether combining two treatments beats the sum of their
separate effects. Running the arms separately and subtracting run averages does
not answer that question — the baselines drift between runs.

Scores whose paired difference or interaction contrast leaves the float range are
refused by the statistics. The run is still saved: `summary.significance_error` or
`summary.interactions_error` holds the refusal, the section it names is absent, and
the report prints it in place of the table. An arm whose own paired differences leave
the range has a null `delta_vs_baseline` and says why in `per_arm.<arm>.delta_error`,
which the report prints as an arm-labelled note under the significance table (or its
heading, when no arm has a row); the significance of the other arms stays when the
family test itself succeeds. A mean of finite scores or of finite paired differences
whose sum would pass the float range is computed exactly (`statistics.mean`), so the
per-arm `avg_aggregate` and `delta_vs_baseline` are refused only when a single
difference leaves the range. The console summary of `arms run` prints
`(delta not computed: <the refusal>)` for such an arm.

Everything above assumes the judge's continuous 0–100 score. The
execution-verified task track (`cli.py tasks run`) has a binary outcome per task
and uses McNemar's exact test instead — see
Expand Down
2 changes: 2 additions & 0 deletions dokimi/dokimi/commands/arms.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ def cmd_arms_run(args: argparse.Namespace) -> None:
delta = stats.get("delta_vs_baseline")
avg_s = "n/a" if avg is None else f"{avg:.1f}"
delta_s = "" if (delta is None or arm == args.baseline_arm) else f" (delta {delta:+.1f})"
if stats.get("delta_error") and arm != args.baseline_arm:
delta_s = f" (delta not computed: {stats['delta_error']})"
err_s = f" ({stats['errors']} errors)" if stats.get("errors") else ""
print(f" {arm:<24} {avg_s}{delta_s}{err_s}")

Expand Down
40 changes: 18 additions & 22 deletions dokimi/dokimi/eval/arm_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from functools import partial
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence

Expand All @@ -26,7 +27,8 @@
from dokimi.eval import stats as arm_stats
from dokimi.eval.interactions import interaction_significance
from dokimi.eval.arm_summary import (
cell_values, cost_summary, empty_reply_cell, summarize_agentic_usage, summarize_skill_selection, unjudged_cell,
cell_values, cost_summary, empty_reply_cell, mean_in_range, question_deltas, refusal_kept,
summarize_agentic_usage, summarize_skill_selection, unjudged_cell,
)
from dokimi.eval.cells import Cell
from dokimi.eval.question_text import question_text_of, require_question_texts
Expand Down Expand Up @@ -646,22 +648,18 @@ def _score(rec: Record) -> Dict[str, Any]:
score["judge_context_sha256"] = digest
scores[arm_name] = score

deltas = {}
base = scores.get(baseline_arm)
base_agg = base.get("aggregate") if isinstance(base, dict) else None
if base_agg is not None:
for arm_name, s in scores.items():
if arm_name != baseline_arm and isinstance(s, dict) and s.get("aggregate") is not None:
deltas[arm_name] = round(s["aggregate"] - base_agg, 1)

return {
deltas, delta_errors = question_deltas(scores, baseline_arm)
row: Dict[str, Any] = {
"question_id": rec.question_id,
"question_text": rec.question_text,
"category": rec.category,
"difficulty": rec.difficulty,
"scores": scores,
"deltas_vs_baseline": deltas,
}
if delta_errors:
row["delta_errors"] = delta_errors
return row

recs = as_records(records, source="answers", kind="answers")
# Before any judge call: refuse, do not grade against "".
Expand Down Expand Up @@ -762,31 +760,29 @@ def _summarize(

summary: Dict[str, Any] = {"per_arm": {}}
for arm_name, vals in per_arm.items():
avg = round(sum(vals) / len(vals), 1) if vals else None
avg = round(mean_in_range(vals), 1) if vals else None
summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals), "errors": errors.get(arm_name, 0)}
# Paired over the questions both arms scored; the unpaired means above are still reported.
# Always present: null for the baseline, and for an arm with no question both arms scored.
# Paired over the questions both arms scored. Null for the baseline, an arm with no such
# question, and an arm whose score range is refused, which says why in its own delta_error (#750).
for arm_name, stats in summary["per_arm"].items():
stats["delta_vs_baseline"] = (
arm_stats.paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None
stats["delta_vs_baseline"] = None if arm_name == baseline_arm else refusal_kept(
stats, "delta_error", partial(arm_stats.paired_delta, evaluations, baseline_arm, arm_name)
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
)
Comment thread
antonwolfy marked this conversation as resolved.
Resolved

# Paired statistics. Every arm is tested against the *same* baseline
# sample question-by-question, so the arms are mutually comparable and
# p-values get corrected across the family. Unpaired means above are
# kept as-is for artifact compatibility.
arm_names = list(summary["per_arm"].keys())
significance = arm_stats.arm_significance(
evaluations, baseline_arm, arm_names, correction=self.stats_correction
)
for arm_name, sig in significance.items():
significance = refusal_kept(summary, "significance_error", lambda: arm_stats.arm_significance(
evaluations, baseline_arm, arm_names, correction=self.stats_correction))
for arm_name, sig in (significance or {}).items():
summary["per_arm"].setdefault(arm_name, {})["significance"] = sig

# Interaction contrasts are their own hypothesis family, corrected
# among themselves rather than pooled with the arm comparisons above.
interactions = interaction_significance(
evaluations, arm_names, baseline_arm, correction=self.stats_correction
)
interactions = refusal_kept(summary, "interactions_error", lambda: interaction_significance(
evaluations, arm_names, baseline_arm, correction=self.stats_correction))
if interactions:
summary["interactions"] = interactions

Expand Down
50 changes: 49 additions & 1 deletion dokimi/dokimi/eval/arm_summary.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,17 @@

from __future__ import annotations

import logging
import math
import statistics
import time
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple

from dokimi.records import Answer, RecordLike
from dokimi.records import records as as_records

logger = logging.getLogger(__name__)

#: Stamp on an agentic answer's ``tool_usage["counts"]``: its tool counts skip
#: calls that failed (``error`` in the transcript). Unstamped answers counted
#: every attempt, so their tool-use rates do not pair with stamped ones.
Expand All @@ -37,6 +42,49 @@ def _tool_counts_stamp(stamped: int, n: int) -> str | None:
return TOOL_COUNTS if stamped == n else TOOL_COUNTS_MIXED


def refusal_kept(holder: Dict[str, Any], key: str, compute: Callable[[], Any]) -> Any:
"""``compute()``, or None with the refusal in ``holder[key]`` (the first one stays) when it raises OverflowError.

A score range the statistics refuse (#599, #638) must not drop the run's answers (#750).
"""
try:
return compute()
except OverflowError as exc:
logger.warning("%s, the run is kept without it: %s", key, exc)
Comment thread
antonwolfy marked this conversation as resolved.
Resolved
holder.setdefault(key, str(exc))
return None


def question_deltas(scores: Mapping[str, Any], baseline_arm: str) -> Tuple[Dict[str, float], Dict[str, str]]:
"""Each scored arm's rounded aggregate minus the baseline's, and the arms whose difference leaves the float range.

Infinity is not JSON: such a delta is refused by name, not stored (#750).
"""
deltas: Dict[str, float] = {}
errors: Dict[str, str] = {}
base = scores.get(baseline_arm)
base_agg = base.get("aggregate") if isinstance(base, dict) else None
if base_agg is None:
return deltas, errors
for arm_name, s in scores.items():
if arm_name == baseline_arm or not isinstance(s, dict) or s.get("aggregate") is None:
continue
delta = round(s["aggregate"] - base_agg, 1)
if math.isfinite(delta):
deltas[arm_name] = delta
else:
errors[arm_name] = "the difference of the scores leaves the float range"
return deltas, errors


def mean_in_range(values: Sequence[float]) -> float:
"""``sum(values) / len(values)``, averaging exactly (``statistics.mean``) when the sum of finite values leaves the float range."""
mean = sum(values) / len(values)
if math.isfinite(mean) or not all(math.isfinite(v) for v in values):
return mean
return statistics.mean(values)


def summarize_agentic_usage(records: Iterable[RecordLike]) -> Dict[str, Any]:
"""Aggregate tool-use/discoverability signals from answer records.

Expand Down
3 changes: 2 additions & 1 deletion dokimi/dokimi/eval/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
from dataclasses import dataclass
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple

from dokimi.eval.arm_summary import mean_in_range
from dokimi.eval.finite_delta import check_sum_range, checked_delta
from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, round_mean, sign_flip_row, within
from dokimi.records import BASELINE, CONTEXT, RecordLike, records
Expand Down Expand Up @@ -441,7 +442,7 @@ def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) -
"""
arm_vals, base_vals = paired_vectors(evaluations, arm_name, baseline_arm)
diffs = _deltas(arm_vals, base_vals)
return round(sum(diffs) / len(diffs), 1) if diffs else None
return round(mean_in_range(diffs), 1) if diffs else None


@dataclass(frozen=True)
Expand Down
17 changes: 15 additions & 2 deletions dokimi/dokimi/report/arms_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,22 @@ def _significance_lines(
summary: Dict[str, Any], arms: List[str], baseline: str
) -> List[str]:
"""Paired significance table: every arm vs the one shared baseline."""
refused = [
f"_`{arm}` not computed: {error}_"
for arm in arms
if arm != baseline and (error := summary.get("per_arm", {}).get(arm, {}).get("delta_error"))
]
notes = [line for note in refused for line in (note, "")]
if error := summary.get("significance_error"):
return ["## Paired significance vs baseline", "", f"_Not computed: {error}_", "", *notes]
rows = [
(arm, summary["per_arm"][arm]["significance"])
for arm in arms
if arm != baseline
and isinstance(summary.get("per_arm", {}).get(arm, {}).get("significance"), dict)
]
if not rows:
return []
return ["## Paired significance vs baseline", "", *notes] if refused else []

correction = next(
(sig.get("correction") for _, sig in rows if sig.get("correction")), "holm"
Expand Down Expand Up @@ -126,11 +134,16 @@ def _significance_lines(
)
lines += _ttest_note([sig for _, sig in rows])
lines.append("")
return lines
return lines + notes


def _interaction_lines(summary: Dict[str, Any]) -> List[str]:
"""Factorial interaction: does combining two treatments beat their sum?"""
if error := summary.get("interactions_error"):
return [
"## Factorial interaction", "",
f"_Not computed: {error}_", "",
]
interactions = summary.get("interactions") or []
if not interactions:
return []
Expand Down
12 changes: 12 additions & 0 deletions dokimi/dokimi/schemas/arms.v1.json
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@
"null"
]
},
"delta_error": {
"description": "Why delta_vs_baseline is null for this arm: its paired differences leave the float range. Present only on such a refusal.",
"type": "string"
},
"significance": {
"description": "Paired test of this arm against the shared baseline arm. Absent on the baseline arm itself, and on arms with too few complete pairs to describe.",
"type": "object",
Expand Down Expand Up @@ -255,6 +259,14 @@
}
}
},
"significance_error": {
"description": "Why per_arm.<arm>.significance is absent from every arm: the paired tests of the arm family refused the scores (a difference or a sum of differences beyond the float range). Present only on such a refusal; an arm with too few pairs for the family leaves the others' significance in place.",
"type": "string"
},
"interactions_error": {
"description": "Why interactions is absent: an interaction contrast was beyond the float range. Present only on such a refusal.",
"type": "string"
},
"interactions": {
"description": "One entry per detected 2x2 factorial cell, i.e. per combined arm whose two sub-treatments are also present as standalone arms. Absent when no such cell exists.",
"type": "array",
Expand Down
1 change: 0 additions & 1 deletion dokimi/scripts/growth_baseline.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
"dokimi/context/trace.py::_apply": {"complexity": 18},
"dokimi/eval/agent_runner.py::_summarize_tool_usage": {"complexity": 11},
"dokimi/eval/agent_runner.py::run_agent_loop": {"complexity": 14},
"dokimi/eval/arm_runner.py::ArmRunner.judge": {"complexity": 11},
"dokimi/eval/checklist.py::ChecklistReport.format_markdown": {"complexity": 12},
"dokimi/eval/degrade.py::build_pairs": {"complexity": 17, "rationale": "grew on main in #73 (judge-validity --context stored) from 10 to 17 before the ratchet landed; split it in its own PR"},
"dokimi/eval/judge_sequential.py::run_sequential": {"complexity": 12, "rationale": "grew on main in #73 (judge-validity --context stored) from 6 to 12 before the ratchet landed; split it in its own PR"},
Expand Down
Loading
Loading