From ab883ea19e77c90498b5d95ca25337846cfb817a Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 18:20:52 +0000 Subject: [PATCH 1/5] fix(eval): keep an arms run whose scores overflow the significance tests (#750) ArmRunner._summarize computed the per-arm paired delta, arm_significance and interaction_significance after the arms had run and before the save. Scores whose paired difference or interaction contrast leave the float range raise OverflowError there (#599, #638), the exception left ArmRunner.run, and the run's answers and evaluations, which cost model calls, were never written. Each of the three computations is now guarded (refusal_kept in eval/arm_summary.py): on OverflowError it logs a warning, leaves the result out (delta_vs_baseline null, no per-arm significance, no interactions) and records the exception text in summary.significance_error or summary.interactions_error. The two families are guarded separately, so an interaction refusal keeps the arm significance. The arms report prints the recorded refusal where the table would be, and arms.v1.json describes the two keys (a widening). An input without overflow gives the same summary as before. stats.py and interactions.py still raise. Signed-off-by: Anton Volkov --- .../20261116_150000_arms-overflow-save.md | 8 ++ dokimi/docs/benchmarking-and-comparison.md | 5 + dokimi/dokimi/eval/arm_runner.py | 24 ++-- dokimi/dokimi/eval/arm_summary.py | 18 ++- dokimi/dokimi/report/arms_report.py | 10 ++ dokimi/dokimi/schemas/arms.v1.json | 8 ++ dokimi/tests/test_arm_runner_float_range.py | 105 ++++++++++++++++++ 7 files changed, 165 insertions(+), 13 deletions(-) create mode 100644 dokimi/changelog.d/20261116_150000_arms-overflow-save.md create mode 100644 dokimi/tests/test_arm_runner_float_range.py diff --git a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md new file mode 100644 index 000000000..97f00ffd3 --- /dev/null +++ b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error` or `summary.interactions_error` says why (#750). diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4bc8639e5..429defcd6 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -430,6 +430,11 @@ is what tells you whether combining two treatments beats the sum of their separate effects. Running the arms separately and subtracting run averages does not answer that question — the baselines drift between runs. +Scores whose paired difference or interaction contrast leaves the float range are +refused by the statistics. The run is still saved: `summary.significance_error` or +`summary.interactions_error` holds the refusal, the section it names is absent, and +the report prints it in place of the table. + Everything above assumes the judge's continuous 0–100 score. The execution-verified task track (`cli.py tasks run`) has a binary outcome per task and uses McNemar's exact test instead — see diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index cf642893d..7d44664d7 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -15,6 +15,7 @@ from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone +from functools import partial from pathlib import Path from typing import Any, Dict, List, Optional, Sequence @@ -26,7 +27,8 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, empty_reply_cell, summarize_agentic_usage, summarize_skill_selection, unjudged_cell, + cell_values, cost_summary, empty_reply_cell, refusal_kept, summarize_agentic_usage, summarize_skill_selection, + unjudged_cell, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -764,11 +766,11 @@ def _summarize( for arm_name, vals in per_arm.items(): avg = round(sum(vals) / len(vals), 1) if vals else None summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals), "errors": errors.get(arm_name, 0)} - # Paired over the questions both arms scored; the unpaired means above are still reported. - # Always present: null for the baseline, and for an arm with no question both arms scored. + # Paired over the questions both arms scored. Null for the baseline, an arm with no such + # question, and a refused score range (#750). for arm_name, stats in summary["per_arm"].items(): - stats["delta_vs_baseline"] = ( - arm_stats.paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None + stats["delta_vs_baseline"] = None if arm_name == baseline_arm else refusal_kept( + summary, "significance_error", partial(arm_stats.paired_delta, evaluations, baseline_arm, arm_name) ) # Paired statistics. Every arm is tested against the *same* baseline @@ -776,17 +778,15 @@ def _summarize( # p-values get corrected across the family. Unpaired means above are # kept as-is for artifact compatibility. arm_names = list(summary["per_arm"].keys()) - significance = arm_stats.arm_significance( - evaluations, baseline_arm, arm_names, correction=self.stats_correction - ) - for arm_name, sig in significance.items(): + significance = refusal_kept(summary, "significance_error", lambda: arm_stats.arm_significance( + evaluations, baseline_arm, arm_names, correction=self.stats_correction)) + for arm_name, sig in (significance or {}).items(): summary["per_arm"].setdefault(arm_name, {})["significance"] = sig # Interaction contrasts are their own hypothesis family, corrected # among themselves rather than pooled with the arm comparisons above. - interactions = interaction_significance( - evaluations, arm_names, baseline_arm, correction=self.stats_correction - ) + interactions = refusal_kept(summary, "interactions_error", lambda: interaction_significance( + evaluations, arm_names, baseline_arm, correction=self.stats_correction)) if interactions: summary["interactions"] = interactions diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 874c3d4a0..c40f38470 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -13,12 +13,15 @@ from __future__ import annotations +import logging import time -from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from dokimi.records import Answer, RecordLike from dokimi.records import records as as_records +logger = logging.getLogger(__name__) + #: Stamp on an agentic answer's ``tool_usage["counts"]``: its tool counts skip #: calls that failed (``error`` in the transcript). Unstamped answers counted #: every attempt, so their tool-use rates do not pair with stamped ones. @@ -37,6 +40,19 @@ def _tool_counts_stamp(stamped: int, n: int) -> str | None: return TOOL_COUNTS if stamped == n else TOOL_COUNTS_MIXED +def refusal_kept(summary: Dict[str, Any], key: str, compute: Callable[[], Any]) -> Any: + """``compute()``, or None with the refusal in ``summary[key]`` (the first one stays) when it raises OverflowError. + + A score range the statistics refuse (#599, #638) must not drop the run's answers (#750). + """ + try: + return compute() + except OverflowError as exc: + logger.warning("%s, the run is kept without it: %s", key, exc) + summary.setdefault(key, str(exc)) + return None + + def summarize_agentic_usage(records: Iterable[RecordLike]) -> Dict[str, Any]: """Aggregate tool-use/discoverability signals from answer records. diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index edfa91540..4e0842b2b 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -89,6 +89,11 @@ def _significance_lines( summary: Dict[str, Any], arms: List[str], baseline: str ) -> List[str]: """Paired significance table: every arm vs the one shared baseline.""" + if error := summary.get("significance_error"): + return [ + "## Paired significance vs baseline", "", + f"_Not computed: {error}_", "", + ] rows = [ (arm, summary["per_arm"][arm]["significance"]) for arm in arms @@ -131,6 +136,11 @@ def _significance_lines( def _interaction_lines(summary: Dict[str, Any]) -> List[str]: """Factorial interaction: does combining two treatments beat their sum?""" + if error := summary.get("interactions_error"): + return [ + "## Factorial interaction", "", + f"_Not computed: {error}_", "", + ] interactions = summary.get("interactions") or [] if not interactions: return [] diff --git a/dokimi/dokimi/schemas/arms.v1.json b/dokimi/dokimi/schemas/arms.v1.json index 9e3d6d1c7..c14ce5c4e 100644 --- a/dokimi/dokimi/schemas/arms.v1.json +++ b/dokimi/dokimi/schemas/arms.v1.json @@ -255,6 +255,14 @@ } } }, + "significance_error": { + "description": "Why per_arm..significance is absent from every arm: the paired test refused the scores (a difference beyond the float range). Present only on such a refusal.", + "type": "string" + }, + "interactions_error": { + "description": "Why interactions is absent: an interaction contrast was beyond the float range. Present only on such a refusal.", + "type": "string" + }, "interactions": { "description": "One entry per detected 2x2 factorial cell, i.e. per combined arm whose two sub-treatments are also present as standalone arms. Absent when no such cell exists.", "type": "array", diff --git a/dokimi/tests/test_arm_runner_float_range.py b/dokimi/tests/test_arm_runner_float_range.py new file mode 100644 index 000000000..682ace997 --- /dev/null +++ b/dokimi/tests/test_arm_runner_float_range.py @@ -0,0 +1,105 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A score range the statistics refuse (#599, #638) must not lose the arms run (#750).""" + +import json + +import dokimi.eval.arm_runner as arm_runner_mod +from dokimi.eval.arm_runner import ArmRunner +from dokimi.report.arms_report import render_arms_report +from dokimi.treatments import BaselineTreatment + +BIG = 1.7e308 +QUESTIONS = [ + {"id": "q1", "question": "How do I parallelize a loop?"}, + {"id": "q2", "question": "How do I sum an array in parallel?"}, +] + + +def _fake_call(prompt, model, provider="openai", api_key=None, system=None, **kw): + return "an answer", {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + + +class _Judge: + model = "fake-judge" + provider = "fake-provider" + + +def _ev(qid, scores): + return {"question_id": qid, "scores": {k: {"aggregate": v} for k, v in scores.items()}} + + +def _run(monkeypatch, arms, evaluations): + monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", _fake_call) + runner = ArmRunner([BaselineTreatment(name) for name in arms]) + records = runner.run("oneTBB", QUESTIONS, concurrency=1) + return runner, records, runner.build_output("oneTBB", records, evaluations, judge=_Judge()) + + +def _saved(runner, output, tmp_path): + path = tmp_path / "arms.json" + runner.save(output, path) + return json.loads(path.read_text(encoding="utf-8")) + + +def test_overflowing_paired_difference_keeps_the_run(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": -BIG, "a": BIG}), + _ev("q2", {"baseline": 0.0, "a": 0.0}), + ] + runner, records, output = _run(monkeypatch, ["baseline", "a"], evaluations) + + assert output["answers"] == records + assert output["evaluations"] == evaluations + summary = output["summary"] + assert "float range" in summary["significance_error"] + assert "significance" not in summary["per_arm"]["a"] + assert summary["per_arm"]["a"]["delta_vs_baseline"] is None + assert "interactions_error" not in summary + + saved = _saved(runner, output, tmp_path) + assert saved["evaluations"] == evaluations + assert saved["summary"]["significance_error"] == summary["significance_error"] + + +def test_overflowing_interaction_contrast_keeps_the_arm_significance(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": 0.0, "a": -BIG, "b": 0.0, "a+b": BIG}), + _ev("q2", {"baseline": 0.0, "a": 0.0, "b": 0.0, "a+b": 0.0}), + ] + runner, records, output = _run(monkeypatch, ["baseline", "a", "b", "a+b"], evaluations) + + assert output["answers"] == records + summary = output["summary"] + assert "float range" in summary["interactions_error"] + assert "interactions" not in summary + assert "significance_error" not in summary + assert summary["per_arm"]["a"]["significance"]["n"] == 2 + + saved = _saved(runner, output, tmp_path) + assert saved["summary"]["interactions_error"] == summary["interactions_error"] + assert saved["summary"]["per_arm"]["a"]["significance"]["n"] == 2 + + +def test_report_states_why_the_sections_are_missing(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": -BIG, "a": BIG}), + _ev("q2", {"baseline": 0.0, "a": 0.0}), + ] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + output["summary"]["interactions_error"] = "contrast too large" + text = render_arms_report(_saved(runner, output, tmp_path)) + + assert "Not computed: " + output["summary"]["significance_error"] in text + assert "Not computed: contrast too large" in text + + +def test_no_refusal_key_without_overflow(monkeypatch): + evaluations = [ + _ev("q1", {"baseline": 50.0, "a": 60.0}), + _ev("q2", {"baseline": 52.0, "a": 55.0}), + ] + _, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + + assert not {"significance_error", "interactions_error"} & set(output["summary"]) From d66c8a59d434d9a56e0d064b9a3a875e862272d6 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 19:40:24 +0000 Subject: [PATCH 2/5] fix(eval): keep arms averages finite and scope a refused delta to its arm Settles the two review threads on #831 about the kept overflow run. The per-arm average of finite extreme scores (-1.7e308, 1.7e308) summed to inf and was saved as Infinity, which strict JSON readers reject. mean_in_range divides first when the plain mean leaves the float range. paired_delta refuses a delta whose sum leaves the range, so delta_vs_baseline is null instead of Infinity. A paired delta that overflows no longer sets summary.significance_error, which means the whole arm family was refused. It is recorded as per_arm..delta_error, so an arm that the family skips (one pair) does not hide another arm's valid significance table. significance_error stays for a refused family. Signed-off-by: Anton Volkov --- .../20261116_150000_arms-overflow-save.md | 2 +- dokimi/docs/benchmarking-and-comparison.md | 5 +- dokimi/dokimi/eval/arm_runner.py | 10 ++-- dokimi/dokimi/eval/arm_summary.py | 15 ++++- dokimi/dokimi/eval/stats.py | 1 + dokimi/dokimi/schemas/arms.v1.json | 6 +- dokimi/tests/test_arm_runner_float_range.py | 57 ++++++++++++++++++- 7 files changed, 83 insertions(+), 13 deletions(-) diff --git a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md index 97f00ffd3..b8555e4e4 100644 --- a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md +++ b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed -- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error` or `summary.interactions_error` says why (#750). +- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why (#750). The per-arm averages and deltas of such scores stay finite or are refused, so the saved file holds no `Infinity`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 429defcd6..a8ea648b1 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -433,7 +433,10 @@ not answer that question — the baselines drift between runs. Scores whose paired difference or interaction contrast leaves the float range are refused by the statistics. The run is still saved: `summary.significance_error` or `summary.interactions_error` holds the refusal, the section it names is absent, and -the report prints it in place of the table. +the report prints it in place of the table. An arm whose own paired differences leave +the range has a null `delta_vs_baseline` and says why in `per_arm..delta_error`; +the significance of the other arms stays when the family test itself succeeds. A mean +of finite scores that would pass the float range is computed by dividing first. Everything above assumes the judge's continuous 0–100 score. The execution-verified task track (`cli.py tasks run`) has a binary outcome per task diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 7d44664d7..163adf9eb 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -27,8 +27,8 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, empty_reply_cell, refusal_kept, summarize_agentic_usage, summarize_skill_selection, - unjudged_cell, + cell_values, cost_summary, empty_reply_cell, mean_in_range, refusal_kept, summarize_agentic_usage, + summarize_skill_selection, unjudged_cell, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -764,13 +764,13 @@ def _summarize( summary: Dict[str, Any] = {"per_arm": {}} for arm_name, vals in per_arm.items(): - avg = round(sum(vals) / len(vals), 1) if vals else None + avg = round(mean_in_range(vals), 1) if vals else None summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals), "errors": errors.get(arm_name, 0)} # Paired over the questions both arms scored. Null for the baseline, an arm with no such - # question, and a refused score range (#750). + # question, and an arm whose score range is refused, which says why in its own delta_error (#750). for arm_name, stats in summary["per_arm"].items(): stats["delta_vs_baseline"] = None if arm_name == baseline_arm else refusal_kept( - summary, "significance_error", partial(arm_stats.paired_delta, evaluations, baseline_arm, arm_name) + stats, "delta_error", partial(arm_stats.paired_delta, evaluations, baseline_arm, arm_name) ) # Paired statistics. Every arm is tested against the *same* baseline diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index c40f38470..0c236bb77 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -14,6 +14,7 @@ from __future__ import annotations import logging +import math import time from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -40,8 +41,8 @@ def _tool_counts_stamp(stamped: int, n: int) -> str | None: return TOOL_COUNTS if stamped == n else TOOL_COUNTS_MIXED -def refusal_kept(summary: Dict[str, Any], key: str, compute: Callable[[], Any]) -> Any: - """``compute()``, or None with the refusal in ``summary[key]`` (the first one stays) when it raises OverflowError. +def refusal_kept(holder: Dict[str, Any], key: str, compute: Callable[[], Any]) -> Any: + """``compute()``, or None with the refusal in ``holder[key]`` (the first one stays) when it raises OverflowError. A score range the statistics refuse (#599, #638) must not drop the run's answers (#750). """ @@ -49,10 +50,18 @@ def refusal_kept(summary: Dict[str, Any], key: str, compute: Callable[[], Any]) return compute() except OverflowError as exc: logger.warning("%s, the run is kept without it: %s", key, exc) - summary.setdefault(key, str(exc)) + holder.setdefault(key, str(exc)) return None +def mean_in_range(values: Sequence[float]) -> float: + """``sum(values) / len(values)``, dividing first when the sum of finite values leaves the float range.""" + mean = sum(values) / len(values) + if math.isfinite(mean) or not all(math.isfinite(v) for v in values): + return mean + return sum(v / len(values) for v in values) + + def summarize_agentic_usage(records: Iterable[RecordLike]) -> Dict[str, Any]: """Aggregate tool-use/discoverability signals from answer records. diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index ddc217e33..913e9accb 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -441,6 +441,7 @@ def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) - """ arm_vals, base_vals = paired_vectors(evaluations, arm_name, baseline_arm) diffs = _deltas(arm_vals, base_vals) + check_sum_range(diffs) return round(sum(diffs) / len(diffs), 1) if diffs else None diff --git a/dokimi/dokimi/schemas/arms.v1.json b/dokimi/dokimi/schemas/arms.v1.json index c14ce5c4e..ca58c79f8 100644 --- a/dokimi/dokimi/schemas/arms.v1.json +++ b/dokimi/dokimi/schemas/arms.v1.json @@ -125,6 +125,10 @@ "null" ] }, + "delta_error": { + "description": "Why delta_vs_baseline is null for this arm: its paired differences leave the float range. Present only on such a refusal.", + "type": "string" + }, "significance": { "description": "Paired test of this arm against the shared baseline arm. Absent on the baseline arm itself, and on arms with too few complete pairs to describe.", "type": "object", @@ -256,7 +260,7 @@ } }, "significance_error": { - "description": "Why per_arm..significance is absent from every arm: the paired test refused the scores (a difference beyond the float range). Present only on such a refusal.", + "description": "Why per_arm..significance is absent from every arm: the paired tests of the arm family refused the scores (a difference or a sum of differences beyond the float range). Present only on such a refusal; an arm with too few pairs for the family leaves the others' significance in place.", "type": "string" }, "interactions_error": { diff --git a/dokimi/tests/test_arm_runner_float_range.py b/dokimi/tests/test_arm_runner_float_range.py index 682ace997..47bf3d147 100644 --- a/dokimi/tests/test_arm_runner_float_range.py +++ b/dokimi/tests/test_arm_runner_float_range.py @@ -30,10 +30,10 @@ def _ev(qid, scores): return {"question_id": qid, "scores": {k: {"aggregate": v} for k, v in scores.items()}} -def _run(monkeypatch, arms, evaluations): +def _run(monkeypatch, arms, evaluations, questions=QUESTIONS): monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", _fake_call) runner = ArmRunner([BaselineTreatment(name) for name in arms]) - records = runner.run("oneTBB", QUESTIONS, concurrency=1) + records = runner.run("oneTBB", questions, concurrency=1) return runner, records, runner.build_output("oneTBB", records, evaluations, judge=_Judge()) @@ -56,6 +56,7 @@ def test_overflowing_paired_difference_keeps_the_run(monkeypatch, tmp_path): assert "float range" in summary["significance_error"] assert "significance" not in summary["per_arm"]["a"] assert summary["per_arm"]["a"]["delta_vs_baseline"] is None + assert "float range" in summary["per_arm"]["a"]["delta_error"] assert "interactions_error" not in summary saved = _saved(runner, output, tmp_path) @@ -95,6 +96,58 @@ def test_report_states_why_the_sections_are_missing(monkeypatch, tmp_path): assert "Not computed: contrast too large" in text +def _strict_json(path): + def refuse(constant): + raise AssertionError(f"{constant} is not JSON") + + return json.loads(path.read_text(encoding="utf-8"), parse_constant=refuse) + + +def test_saved_averages_of_extreme_scores_are_finite_json(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": -BIG, "a": BIG}), + _ev("q2", {"baseline": -BIG, "a": BIG}), + ] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + path = tmp_path / "arms.json" + runner.save(output, path) + + per_arm = _strict_json(path)["summary"]["per_arm"] + assert per_arm["baseline"]["avg_aggregate"] == -BIG + assert per_arm["a"]["avg_aggregate"] == BIG + + +def test_delta_whose_sum_leaves_the_range_is_refused_not_infinite(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": 0.0, "a": 1.0e308}), + _ev("q2", {"baseline": 0.0, "a": 1.0e308}), + ] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + path = tmp_path / "arms.json" + runner.save(output, path) + + arm = _strict_json(path)["summary"]["per_arm"]["a"] + assert arm["delta_vs_baseline"] is None + assert "float range" in arm["delta_error"] + + +def test_refused_delta_of_a_thin_arm_keeps_another_arms_significance(monkeypatch, tmp_path): + questions = [{"id": f"q{i}", "question": f"question {i}"} for i in range(1, 9)] + evaluations = [_ev("q1", {"baseline": -BIG, "a": BIG, "b": -BIG})] + [ + _ev(f"q{i}", {"baseline": 0.0, "b": 1.0}) for i in range(2, 9) + ] + runner, _, output = _run(monkeypatch, ["baseline", "a", "b"], evaluations, questions) + summary = output["summary"] + + assert "float range" in summary["per_arm"]["a"]["delta_error"] + assert summary["per_arm"]["a"]["delta_vs_baseline"] is None + assert "significance_error" not in summary + assert summary["per_arm"]["b"]["significance"]["n"] == 8 + text = render_arms_report(_saved(runner, output, tmp_path)) + assert "Not computed" not in text + assert "`b`" in text.split("## Paired significance vs baseline")[1] + + def test_no_refusal_key_without_overflow(monkeypatch): evaluations = [ _ev("q1", {"baseline": 50.0, "a": 60.0}), From 6ff3ab5fd0b94430080a9950474d0594d375c921 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 20:22:46 +0000 Subject: [PATCH 3/5] fix(eval): print an arm's delta refusal in the arms report; average maximum scores exactly Settles two review points on #831. The arms report never read `per_arm..delta_error`, so an arm whose delta was refused showed a dash and no reason when another arm's family test succeeded; the significance section now prints an arm-labelled `not computed` note beside the table, or under its heading when no arm has a row. `mean_in_range` divided each score first, and three `sys.float_info.max` scores still summed to `inf` after rounding; it now falls back to `statistics.mean`, as `stats._mean` does. Tests: the thin-arm test asserts the rendered note, and a three-question maximum-score run is saved and parsed strictly. Signed-off-by: Anton Volkov --- .../20261116_150000_arms-overflow-save.md | 2 +- dokimi/docs/benchmarking-and-comparison.md | 8 +++++--- dokimi/dokimi/eval/arm_summary.py | 5 +++-- dokimi/dokimi/report/arms_report.py | 15 ++++++++------ dokimi/tests/test_arm_runner_float_range.py | 20 +++++++++++++++++-- 5 files changed, 36 insertions(+), 14 deletions(-) diff --git a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md index b8555e4e4..a0786fb41 100644 --- a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md +++ b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed -- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why (#750). The per-arm averages and deltas of such scores stay finite or are refused, so the saved file holds no `Infinity`. +- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores stay finite or are refused, so the saved file holds no `Infinity`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index a8ea648b1..728b1fbbc 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -434,9 +434,11 @@ Scores whose paired difference or interaction contrast leaves the float range ar refused by the statistics. The run is still saved: `summary.significance_error` or `summary.interactions_error` holds the refusal, the section it names is absent, and the report prints it in place of the table. An arm whose own paired differences leave -the range has a null `delta_vs_baseline` and says why in `per_arm..delta_error`; -the significance of the other arms stays when the family test itself succeeds. A mean -of finite scores that would pass the float range is computed by dividing first. +the range has a null `delta_vs_baseline` and says why in `per_arm..delta_error`, +which the report prints as an arm-labelled note under the significance table (or its +heading, when no arm has a row); the significance of the other arms stays when the +family test itself succeeds. A mean of finite scores whose sum would pass the float +range is computed exactly (`statistics.mean`). Everything above assumes the judge's continuous 0–100 score. The execution-verified task track (`cli.py tasks run`) has a binary outcome per task diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 0c236bb77..317300cac 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -15,6 +15,7 @@ import logging import math +import statistics import time from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple @@ -55,11 +56,11 @@ def refusal_kept(holder: Dict[str, Any], key: str, compute: Callable[[], Any]) - def mean_in_range(values: Sequence[float]) -> float: - """``sum(values) / len(values)``, dividing first when the sum of finite values leaves the float range.""" + """``sum(values) / len(values)``, averaging exactly (``statistics.mean``) when the sum of finite values leaves the float range.""" mean = sum(values) / len(values) if math.isfinite(mean) or not all(math.isfinite(v) for v in values): return mean - return sum(v / len(values) for v in values) + return statistics.mean(values) def summarize_agentic_usage(records: Iterable[RecordLike]) -> Dict[str, Any]: diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index 4e0842b2b..079b2bf6b 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -89,11 +89,14 @@ def _significance_lines( summary: Dict[str, Any], arms: List[str], baseline: str ) -> List[str]: """Paired significance table: every arm vs the one shared baseline.""" + refused = [ + f"_`{arm}` not computed: {error}_" + for arm in arms + if arm != baseline and (error := summary.get("per_arm", {}).get(arm, {}).get("delta_error")) + ] + notes = [line for note in refused for line in (note, "")] if error := summary.get("significance_error"): - return [ - "## Paired significance vs baseline", "", - f"_Not computed: {error}_", "", - ] + return ["## Paired significance vs baseline", "", f"_Not computed: {error}_", "", *notes] rows = [ (arm, summary["per_arm"][arm]["significance"]) for arm in arms @@ -101,7 +104,7 @@ def _significance_lines( and isinstance(summary.get("per_arm", {}).get(arm, {}).get("significance"), dict) ] if not rows: - return [] + return ["## Paired significance vs baseline", "", *notes] if refused else [] correction = next( (sig.get("correction") for _, sig in rows if sig.get("correction")), "holm" @@ -131,7 +134,7 @@ def _significance_lines( ) lines += _ttest_note([sig for _, sig in rows]) lines.append("") - return lines + return lines + notes def _interaction_lines(summary: Dict[str, Any]) -> List[str]: diff --git a/dokimi/tests/test_arm_runner_float_range.py b/dokimi/tests/test_arm_runner_float_range.py index 47bf3d147..d4cc1933a 100644 --- a/dokimi/tests/test_arm_runner_float_range.py +++ b/dokimi/tests/test_arm_runner_float_range.py @@ -4,6 +4,7 @@ """A score range the statistics refuse (#599, #638) must not lose the arms run (#750).""" import json +import sys import dokimi.eval.arm_runner as arm_runner_mod from dokimi.eval.arm_runner import ArmRunner @@ -117,6 +118,19 @@ def test_saved_averages_of_extreme_scores_are_finite_json(monkeypatch, tmp_path) assert per_arm["a"]["avg_aggregate"] == BIG +def test_saved_average_of_three_maximum_scores_is_the_maximum(monkeypatch, tmp_path): + top = sys.float_info.max + evaluations = [_ev(f"q{i}", {"baseline": top, "a": top}) for i in range(1, 4)] + questions = [{"id": f"q{i}", "question": f"question {i}"} for i in range(1, 4)] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations, questions) + path = tmp_path / "arms.json" + runner.save(output, path) + + per_arm = _strict_json(path)["summary"]["per_arm"] + assert per_arm["baseline"]["avg_aggregate"] == sys.float_info.max + assert per_arm["a"]["avg_aggregate"] == sys.float_info.max + + def test_delta_whose_sum_leaves_the_range_is_refused_not_infinite(monkeypatch, tmp_path): evaluations = [ _ev("q1", {"baseline": 0.0, "a": 1.0e308}), @@ -144,8 +158,10 @@ def test_refused_delta_of_a_thin_arm_keeps_another_arms_significance(monkeypatch assert "significance_error" not in summary assert summary["per_arm"]["b"]["significance"]["n"] == 8 text = render_arms_report(_saved(runner, output, tmp_path)) - assert "Not computed" not in text - assert "`b`" in text.split("## Paired significance vs baseline")[1] + assert "Not computed:" not in text + table = text.split("## Paired significance vs baseline")[1] + assert "| `b` |" in table + assert f"`a` not computed: {summary['per_arm']['a']['delta_error']}" in table def test_no_refusal_key_without_overflow(monkeypatch): From 99683e4a01e70fe1af08d2134f682c5cd7b2373d Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 23:07:00 +0000 Subject: [PATCH 4/5] fix(eval): average a delta exactly, print a refused delta on the console, test the warning Settles the review of 6ff3ab5. `paired_delta` averaged the differences with `mean_in_range`, as the per-arm `avg_aggregate` does, instead of refusing by the sum of magnitudes: `+1e308, -1e308, 0` is `0.0` and `1e308` twice is `1e308`; only a single difference that leaves the range is refused. `arms run` prints `(delta not computed: )` for an arm with a `delta_error`. `test_overflowing_paired_difference_keeps_the_run` checks the logged warning. The docs and the changelog fragment say so. Signed-off-by: Anton Volkov --- .../20261116_150000_arms-overflow-save.md | 2 +- dokimi/docs/benchmarking-and-comparison.md | 7 ++- dokimi/dokimi/commands/arms.py | 2 + dokimi/dokimi/eval/stats.py | 4 +- dokimi/tests/test_arm_runner_float_range.py | 49 +++++++++++++++++-- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md index a0786fb41..97a3d8aee 100644 --- a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md +++ b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed -- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores stay finite or are refused, so the saved file holds no `Infinity`. +- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores are averaged exactly, or refused when a single difference leaves the range, so the saved file holds no `Infinity`; `arms run` prints a refused delta in its console summary. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 728b1fbbc..df15b4ee2 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -437,8 +437,11 @@ the report prints it in place of the table. An arm whose own paired differences the range has a null `delta_vs_baseline` and says why in `per_arm..delta_error`, which the report prints as an arm-labelled note under the significance table (or its heading, when no arm has a row); the significance of the other arms stays when the -family test itself succeeds. A mean of finite scores whose sum would pass the float -range is computed exactly (`statistics.mean`). +family test itself succeeds. A mean of finite scores or of finite paired differences +whose sum would pass the float range is computed exactly (`statistics.mean`), so the +per-arm `avg_aggregate` and `delta_vs_baseline` are refused only when a single +difference leaves the range. The console summary of `arms run` prints +`(delta not computed: )` for such an arm. Everything above assumes the judge's continuous 0–100 score. The execution-verified task track (`cli.py tasks run`) has a binary outcome per task diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 8047315f2..91206c5ed 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -93,6 +93,8 @@ def cmd_arms_run(args: argparse.Namespace) -> None: delta = stats.get("delta_vs_baseline") avg_s = "n/a" if avg is None else f"{avg:.1f}" delta_s = "" if (delta is None or arm == args.baseline_arm) else f" (delta {delta:+.1f})" + if stats.get("delta_error") and arm != args.baseline_arm: + delta_s = f" (delta not computed: {stats['delta_error']})" err_s = f" ({stats['errors']} errors)" if stats.get("errors") else "" print(f" {arm:<24} {avg_s}{delta_s}{err_s}") diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index 913e9accb..e8a8a3265 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -64,6 +64,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from dokimi.eval.arm_summary import mean_in_range from dokimi.eval.finite_delta import check_sum_range, checked_delta from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, round_mean, sign_flip_row, within from dokimi.records import BASELINE, CONTEXT, RecordLike, records @@ -441,8 +442,7 @@ def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) - """ arm_vals, base_vals = paired_vectors(evaluations, arm_name, baseline_arm) diffs = _deltas(arm_vals, base_vals) - check_sum_range(diffs) - return round(sum(diffs) / len(diffs), 1) if diffs else None + return round(mean_in_range(diffs), 1) if diffs else None @dataclass(frozen=True) diff --git a/dokimi/tests/test_arm_runner_float_range.py b/dokimi/tests/test_arm_runner_float_range.py index d4cc1933a..86f4dc221 100644 --- a/dokimi/tests/test_arm_runner_float_range.py +++ b/dokimi/tests/test_arm_runner_float_range.py @@ -44,7 +44,7 @@ def _saved(runner, output, tmp_path): return json.loads(path.read_text(encoding="utf-8")) -def test_overflowing_paired_difference_keeps_the_run(monkeypatch, tmp_path): +def test_overflowing_paired_difference_keeps_the_run(monkeypatch, tmp_path, caplog): evaluations = [ _ev("q1", {"baseline": -BIG, "a": BIG}), _ev("q2", {"baseline": 0.0, "a": 0.0}), @@ -53,6 +53,7 @@ def test_overflowing_paired_difference_keeps_the_run(monkeypatch, tmp_path): assert output["answers"] == records assert output["evaluations"] == evaluations + assert "significance_error, the run is kept without it" in caplog.text summary = output["summary"] assert "float range" in summary["significance_error"] assert "significance" not in summary["per_arm"]["a"] @@ -131,7 +132,7 @@ def test_saved_average_of_three_maximum_scores_is_the_maximum(monkeypatch, tmp_p assert per_arm["a"]["avg_aggregate"] == sys.float_info.max -def test_delta_whose_sum_leaves_the_range_is_refused_not_infinite(monkeypatch, tmp_path): +def test_delta_whose_sum_leaves_the_range_is_averaged_exactly(monkeypatch, tmp_path): evaluations = [ _ev("q1", {"baseline": 0.0, "a": 1.0e308}), _ev("q2", {"baseline": 0.0, "a": 1.0e308}), @@ -141,8 +142,24 @@ def test_delta_whose_sum_leaves_the_range_is_refused_not_infinite(monkeypatch, t runner.save(output, path) arm = _strict_json(path)["summary"]["per_arm"]["a"] - assert arm["delta_vs_baseline"] is None - assert "float range" in arm["delta_error"] + assert arm["delta_vs_baseline"] == 1.0e308 + assert "delta_error" not in arm + + +def test_cancelling_deltas_average_to_zero_like_the_scores(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": 0.0, "a": 1.0e308}), + _ev("q2", {"baseline": 1.0e308, "a": 0.0}), + _ev("q3", {"baseline": 0.0, "a": 0.0}), + ] + questions = [{"id": f"q{i}", "question": f"question {i}"} for i in range(1, 4)] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations, questions) + path = tmp_path / "arms.json" + runner.save(output, path) + + arm = _strict_json(path)["summary"]["per_arm"]["a"] + assert arm["delta_vs_baseline"] == 0.0 + assert "delta_error" not in arm def test_refused_delta_of_a_thin_arm_keeps_another_arms_significance(monkeypatch, tmp_path): @@ -164,6 +181,30 @@ def test_refused_delta_of_a_thin_arm_keeps_another_arms_significance(monkeypatch assert f"`a` not computed: {summary['per_arm']['a']['delta_error']}" in table +def test_console_summary_prints_an_arms_delta_refusal(monkeypatch, tmp_path, capsys): + import dokimi.eval.arms_run as arms_run + from dokimi.cli import build_parser + from dokimi.commands import arms as arms_cmd + + summary = {"per_arm": { + "baseline": {"avg_aggregate": 10.0, "n": 2, "errors": 0, "delta_vs_baseline": None}, + "a": {"avg_aggregate": 20.0, "n": 2, "errors": 0, "delta_vs_baseline": None, "delta_error": "too large"}, + "b": {"avg_aggregate": 30.0, "n": 2, "errors": 0, "delta_vs_baseline": 20.0}, + }} + monkeypatch.setattr(arms_cmd, "_load_questions", lambda path: QUESTIONS) + monkeypatch.setattr(arms_run, "prepare_arms", lambda *a, **k: None) + monkeypatch.setattr(arms_run, "run_prepared", lambda *a, **k: {"summary": summary}) + monkeypatch.setattr(arms_cmd, "_save_arms_output", lambda *a, **k: None) + args = build_parser().parse_args([ + "arms", "run", "--product", "p", "--questions", "q.json", "--arms", "baseline,a,b", + ]) + args.func(args) + + lines = capsys.readouterr().out.splitlines() + assert any(line.split()[:2] == ["a", "20.0"] and "(delta not computed: too large)" in line for line in lines) + assert any("(delta +20.0)" in line for line in lines) + + def test_no_refusal_key_without_overflow(monkeypatch): evaluations = [ _ev("q1", {"baseline": 50.0, "a": 60.0}), From daaaa62b078b56919abb2aa29522a1d1f85f4579 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 16 Nov 2026 23:59:25 +0000 Subject: [PATCH 5/5] fix(eval): refuse a per-question delta that leaves the float range at the judge ArmRunner.judge subtracted the baseline score unchecked, so scores of -1.7e308 and 1.7e308 stored inf in evaluations[].deltas_vs_baseline and the saved artifact held Infinity (review thread on test_arm_runner_float_range.py). The delta is now left out and named in the row's delta_errors; a test judges, builds and saves, then parses the whole file strictly. Signed-off-by: Anton Volkov --- .../20261116_150000_arms-overflow-save.md | 2 +- dokimi/dokimi/eval/arm_runner.py | 18 +++++------ dokimi/dokimi/eval/arm_summary.py | 22 +++++++++++++ dokimi/scripts/growth_baseline.json | 1 - dokimi/tests/test_arm_runner_float_range.py | 31 +++++++++++++++++++ 5 files changed, 61 insertions(+), 13 deletions(-) diff --git a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md index 97a3d8aee..cf121928d 100644 --- a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md +++ b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md @@ -5,4 +5,4 @@ SPDX-License-Identifier: Apache-2.0 ### Fixed -- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores are averaged exactly, or refused when a single difference leaves the range, so the saved file holds no `Infinity`; `arms run` prints a refused delta in its console summary. +- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores are averaged exactly, or refused when a single difference leaves the range, so the saved file holds no `Infinity`; `arms run` prints a refused delta in its console summary. A per-question delta whose scores differ by more than the float range is left out of `evaluations[].deltas_vs_baseline` and named in that row's `delta_errors`. diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index 163adf9eb..8ba6e7efe 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -27,8 +27,8 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, empty_reply_cell, mean_in_range, refusal_kept, summarize_agentic_usage, - summarize_skill_selection, unjudged_cell, + cell_values, cost_summary, empty_reply_cell, mean_in_range, question_deltas, refusal_kept, + summarize_agentic_usage, summarize_skill_selection, unjudged_cell, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -648,15 +648,8 @@ def _score(rec: Record) -> Dict[str, Any]: score["judge_context_sha256"] = digest scores[arm_name] = score - deltas = {} - base = scores.get(baseline_arm) - base_agg = base.get("aggregate") if isinstance(base, dict) else None - if base_agg is not None: - for arm_name, s in scores.items(): - if arm_name != baseline_arm and isinstance(s, dict) and s.get("aggregate") is not None: - deltas[arm_name] = round(s["aggregate"] - base_agg, 1) - - return { + deltas, delta_errors = question_deltas(scores, baseline_arm) + row: Dict[str, Any] = { "question_id": rec.question_id, "question_text": rec.question_text, "category": rec.category, @@ -664,6 +657,9 @@ def _score(rec: Record) -> Dict[str, Any]: "scores": scores, "deltas_vs_baseline": deltas, } + if delta_errors: + row["delta_errors"] = delta_errors + return row recs = as_records(records, source="answers", kind="answers") # Before any judge call: refuse, do not grade against "". diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 317300cac..81ea2a872 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -55,6 +55,28 @@ def refusal_kept(holder: Dict[str, Any], key: str, compute: Callable[[], Any]) - return None +def question_deltas(scores: Mapping[str, Any], baseline_arm: str) -> Tuple[Dict[str, float], Dict[str, str]]: + """Each scored arm's rounded aggregate minus the baseline's, and the arms whose difference leaves the float range. + + Infinity is not JSON: such a delta is refused by name, not stored (#750). + """ + deltas: Dict[str, float] = {} + errors: Dict[str, str] = {} + base = scores.get(baseline_arm) + base_agg = base.get("aggregate") if isinstance(base, dict) else None + if base_agg is None: + return deltas, errors + for arm_name, s in scores.items(): + if arm_name == baseline_arm or not isinstance(s, dict) or s.get("aggregate") is None: + continue + delta = round(s["aggregate"] - base_agg, 1) + if math.isfinite(delta): + deltas[arm_name] = delta + else: + errors[arm_name] = "the difference of the scores leaves the float range" + return deltas, errors + + def mean_in_range(values: Sequence[float]) -> float: """``sum(values) / len(values)``, averaging exactly (``statistics.mean``) when the sum of finite values leaves the float range.""" mean = sum(values) / len(values) diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 7a6a02baf..e31aefc5f 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -26,7 +26,6 @@ "dokimi/context/trace.py::_apply": {"complexity": 18}, "dokimi/eval/agent_runner.py::_summarize_tool_usage": {"complexity": 11}, "dokimi/eval/agent_runner.py::run_agent_loop": {"complexity": 14}, - "dokimi/eval/arm_runner.py::ArmRunner.judge": {"complexity": 11}, "dokimi/eval/checklist.py::ChecklistReport.format_markdown": {"complexity": 12}, "dokimi/eval/degrade.py::build_pairs": {"complexity": 17, "rationale": "grew on main in #73 (judge-validity --context stored) from 10 to 17 before the ratchet landed; split it in its own PR"}, "dokimi/eval/judge_sequential.py::run_sequential": {"complexity": 12, "rationale": "grew on main in #73 (judge-validity --context stored) from 6 to 12 before the ratchet landed; split it in its own PR"}, diff --git a/dokimi/tests/test_arm_runner_float_range.py b/dokimi/tests/test_arm_runner_float_range.py index 86f4dc221..c1b89b4ff 100644 --- a/dokimi/tests/test_arm_runner_float_range.py +++ b/dokimi/tests/test_arm_runner_float_range.py @@ -205,6 +205,37 @@ def test_console_summary_prints_an_arms_delta_refusal(monkeypatch, tmp_path, cap assert any("(delta +20.0)" in line for line in lines) +class _ScriptedJudge(_Judge): + """Scores the first answer of a question -BIG (the baseline) and the second BIG on the first question.""" + + def __init__(self): + self.calls = {} + + def score_answer(self, library_name, question, answer, context, ground_truth): + n = self.calls[question] = self.calls.get(question, 0) + 1 + if question != QUESTIONS[0]["question"]: + return {"aggregate": 0.0} + return {"aggregate": -BIG if n == 1 else BIG} + + +def test_judged_overflowing_delta_is_refused_in_the_saved_artifact(monkeypatch, tmp_path): + monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", _fake_call) + runner = ArmRunner([BaselineTreatment("baseline"), BaselineTreatment("a")]) + records = runner.run("oneTBB", QUESTIONS, concurrency=1) + judge = _ScriptedJudge() + evaluations = runner.judge(judge, "oneTBB", records, concurrency=1) + output = runner.build_output("oneTBB", records, evaluations, judge=judge) + path = tmp_path / "arms.json" + runner.save(output, path) + + first, second = _strict_json(path)["evaluations"] + assert first["scores"]["a"]["aggregate"] == BIG + assert first["deltas_vs_baseline"] == {} + assert "float range" in first["delta_errors"]["a"] + assert second["deltas_vs_baseline"] == {"a": 0.0} + assert "delta_errors" not in second + + def test_no_refusal_key_without_overflow(monkeypatch): evaluations = [ _ev("q1", {"baseline": 50.0, "a": 60.0}),