diff --git a/dokimi/changelog.d/20261116_150000_arms-overflow-save.md b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md new file mode 100644 index 000000000..cf121928d --- /dev/null +++ b/dokimi/changelog.d/20261116_150000_arms-overflow-save.md @@ -0,0 +1,8 @@ + + +### Fixed + +- **An arms run is saved when its scores overflow the significance tests.** Scores whose paired difference or interaction contrast leave the float range made `ArmRunner` raise `OverflowError` after the arms had run, so the answers and evaluations were never written. The run is now kept, the refused section is left out, and `summary.significance_error`, `summary.interactions_error` or the arm's `delta_error` says why, and the report prints each refusal (#750). The per-arm averages and deltas of such scores are averaged exactly, or refused when a single difference leaves the range, so the saved file holds no `Infinity`; `arms run` prints a refused delta in its console summary. A per-question delta whose scores differ by more than the float range is left out of `evaluations[].deltas_vs_baseline` and named in that row's `delta_errors`. diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 4bc8639e5..df15b4ee2 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -430,6 +430,19 @@ is what tells you whether combining two treatments beats the sum of their separate effects. Running the arms separately and subtracting run averages does not answer that question — the baselines drift between runs. +Scores whose paired difference or interaction contrast leaves the float range are +refused by the statistics. The run is still saved: `summary.significance_error` or +`summary.interactions_error` holds the refusal, the section it names is absent, and +the report prints it in place of the table. An arm whose own paired differences leave +the range has a null `delta_vs_baseline` and says why in `per_arm..delta_error`, +which the report prints as an arm-labelled note under the significance table (or its +heading, when no arm has a row); the significance of the other arms stays when the +family test itself succeeds. A mean of finite scores or of finite paired differences +whose sum would pass the float range is computed exactly (`statistics.mean`), so the +per-arm `avg_aggregate` and `delta_vs_baseline` are refused only when a single +difference leaves the range. The console summary of `arms run` prints +`(delta not computed: )` for such an arm. + Everything above assumes the judge's continuous 0–100 score. The execution-verified task track (`cli.py tasks run`) has a binary outcome per task and uses McNemar's exact test instead — see diff --git a/dokimi/dokimi/commands/arms.py b/dokimi/dokimi/commands/arms.py index 8047315f2..91206c5ed 100644 --- a/dokimi/dokimi/commands/arms.py +++ b/dokimi/dokimi/commands/arms.py @@ -93,6 +93,8 @@ def cmd_arms_run(args: argparse.Namespace) -> None: delta = stats.get("delta_vs_baseline") avg_s = "n/a" if avg is None else f"{avg:.1f}" delta_s = "" if (delta is None or arm == args.baseline_arm) else f" (delta {delta:+.1f})" + if stats.get("delta_error") and arm != args.baseline_arm: + delta_s = f" (delta not computed: {stats['delta_error']})" err_s = f" ({stats['errors']} errors)" if stats.get("errors") else "" print(f" {arm:<24} {avg_s}{delta_s}{err_s}") diff --git a/dokimi/dokimi/eval/arm_runner.py b/dokimi/dokimi/eval/arm_runner.py index cf642893d..8ba6e7efe 100644 --- a/dokimi/dokimi/eval/arm_runner.py +++ b/dokimi/dokimi/eval/arm_runner.py @@ -15,6 +15,7 @@ from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone +from functools import partial from pathlib import Path from typing import Any, Dict, List, Optional, Sequence @@ -26,7 +27,8 @@ from dokimi.eval import stats as arm_stats from dokimi.eval.interactions import interaction_significance from dokimi.eval.arm_summary import ( - cell_values, cost_summary, empty_reply_cell, summarize_agentic_usage, summarize_skill_selection, unjudged_cell, + cell_values, cost_summary, empty_reply_cell, mean_in_range, question_deltas, refusal_kept, + summarize_agentic_usage, summarize_skill_selection, unjudged_cell, ) from dokimi.eval.cells import Cell from dokimi.eval.question_text import question_text_of, require_question_texts @@ -646,15 +648,8 @@ def _score(rec: Record) -> Dict[str, Any]: score["judge_context_sha256"] = digest scores[arm_name] = score - deltas = {} - base = scores.get(baseline_arm) - base_agg = base.get("aggregate") if isinstance(base, dict) else None - if base_agg is not None: - for arm_name, s in scores.items(): - if arm_name != baseline_arm and isinstance(s, dict) and s.get("aggregate") is not None: - deltas[arm_name] = round(s["aggregate"] - base_agg, 1) - - return { + deltas, delta_errors = question_deltas(scores, baseline_arm) + row: Dict[str, Any] = { "question_id": rec.question_id, "question_text": rec.question_text, "category": rec.category, @@ -662,6 +657,9 @@ def _score(rec: Record) -> Dict[str, Any]: "scores": scores, "deltas_vs_baseline": deltas, } + if delta_errors: + row["delta_errors"] = delta_errors + return row recs = as_records(records, source="answers", kind="answers") # Before any judge call: refuse, do not grade against "". @@ -762,13 +760,13 @@ def _summarize( summary: Dict[str, Any] = {"per_arm": {}} for arm_name, vals in per_arm.items(): - avg = round(sum(vals) / len(vals), 1) if vals else None + avg = round(mean_in_range(vals), 1) if vals else None summary["per_arm"][arm_name] = {"avg_aggregate": avg, "n": len(vals), "errors": errors.get(arm_name, 0)} - # Paired over the questions both arms scored; the unpaired means above are still reported. - # Always present: null for the baseline, and for an arm with no question both arms scored. + # Paired over the questions both arms scored. Null for the baseline, an arm with no such + # question, and an arm whose score range is refused, which says why in its own delta_error (#750). for arm_name, stats in summary["per_arm"].items(): - stats["delta_vs_baseline"] = ( - arm_stats.paired_delta(evaluations, baseline_arm, arm_name) if arm_name != baseline_arm else None + stats["delta_vs_baseline"] = None if arm_name == baseline_arm else refusal_kept( + stats, "delta_error", partial(arm_stats.paired_delta, evaluations, baseline_arm, arm_name) ) # Paired statistics. Every arm is tested against the *same* baseline @@ -776,17 +774,15 @@ def _summarize( # p-values get corrected across the family. Unpaired means above are # kept as-is for artifact compatibility. arm_names = list(summary["per_arm"].keys()) - significance = arm_stats.arm_significance( - evaluations, baseline_arm, arm_names, correction=self.stats_correction - ) - for arm_name, sig in significance.items(): + significance = refusal_kept(summary, "significance_error", lambda: arm_stats.arm_significance( + evaluations, baseline_arm, arm_names, correction=self.stats_correction)) + for arm_name, sig in (significance or {}).items(): summary["per_arm"].setdefault(arm_name, {})["significance"] = sig # Interaction contrasts are their own hypothesis family, corrected # among themselves rather than pooled with the arm comparisons above. - interactions = interaction_significance( - evaluations, arm_names, baseline_arm, correction=self.stats_correction - ) + interactions = refusal_kept(summary, "interactions_error", lambda: interaction_significance( + evaluations, arm_names, baseline_arm, correction=self.stats_correction)) if interactions: summary["interactions"] = interactions diff --git a/dokimi/dokimi/eval/arm_summary.py b/dokimi/dokimi/eval/arm_summary.py index 874c3d4a0..81ea2a872 100644 --- a/dokimi/dokimi/eval/arm_summary.py +++ b/dokimi/dokimi/eval/arm_summary.py @@ -13,12 +13,17 @@ from __future__ import annotations +import logging +import math +import statistics import time -from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple from dokimi.records import Answer, RecordLike from dokimi.records import records as as_records +logger = logging.getLogger(__name__) + #: Stamp on an agentic answer's ``tool_usage["counts"]``: its tool counts skip #: calls that failed (``error`` in the transcript). Unstamped answers counted #: every attempt, so their tool-use rates do not pair with stamped ones. @@ -37,6 +42,49 @@ def _tool_counts_stamp(stamped: int, n: int) -> str | None: return TOOL_COUNTS if stamped == n else TOOL_COUNTS_MIXED +def refusal_kept(holder: Dict[str, Any], key: str, compute: Callable[[], Any]) -> Any: + """``compute()``, or None with the refusal in ``holder[key]`` (the first one stays) when it raises OverflowError. + + A score range the statistics refuse (#599, #638) must not drop the run's answers (#750). + """ + try: + return compute() + except OverflowError as exc: + logger.warning("%s, the run is kept without it: %s", key, exc) + holder.setdefault(key, str(exc)) + return None + + +def question_deltas(scores: Mapping[str, Any], baseline_arm: str) -> Tuple[Dict[str, float], Dict[str, str]]: + """Each scored arm's rounded aggregate minus the baseline's, and the arms whose difference leaves the float range. + + Infinity is not JSON: such a delta is refused by name, not stored (#750). + """ + deltas: Dict[str, float] = {} + errors: Dict[str, str] = {} + base = scores.get(baseline_arm) + base_agg = base.get("aggregate") if isinstance(base, dict) else None + if base_agg is None: + return deltas, errors + for arm_name, s in scores.items(): + if arm_name == baseline_arm or not isinstance(s, dict) or s.get("aggregate") is None: + continue + delta = round(s["aggregate"] - base_agg, 1) + if math.isfinite(delta): + deltas[arm_name] = delta + else: + errors[arm_name] = "the difference of the scores leaves the float range" + return deltas, errors + + +def mean_in_range(values: Sequence[float]) -> float: + """``sum(values) / len(values)``, averaging exactly (``statistics.mean``) when the sum of finite values leaves the float range.""" + mean = sum(values) / len(values) + if math.isfinite(mean) or not all(math.isfinite(v) for v in values): + return mean + return statistics.mean(values) + + def summarize_agentic_usage(records: Iterable[RecordLike]) -> Dict[str, Any]: """Aggregate tool-use/discoverability signals from answer records. diff --git a/dokimi/dokimi/eval/stats.py b/dokimi/dokimi/eval/stats.py index ddc217e33..e8a8a3265 100644 --- a/dokimi/dokimi/eval/stats.py +++ b/dokimi/dokimi/eval/stats.py @@ -64,6 +64,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from dokimi.eval.arm_summary import mean_in_range from dokimi.eval.finite_delta import check_sum_range, checked_delta from dokimi.eval.sign_flip import ALPHA, EPS, METHOD, min_sign_flip_p, round_mean, sign_flip_row, within from dokimi.records import BASELINE, CONTEXT, RecordLike, records @@ -441,7 +442,7 @@ def paired_delta(evaluations: Iterable[Any], baseline_arm: str, arm_name: str) - """ arm_vals, base_vals = paired_vectors(evaluations, arm_name, baseline_arm) diffs = _deltas(arm_vals, base_vals) - return round(sum(diffs) / len(diffs), 1) if diffs else None + return round(mean_in_range(diffs), 1) if diffs else None @dataclass(frozen=True) diff --git a/dokimi/dokimi/report/arms_report.py b/dokimi/dokimi/report/arms_report.py index edfa91540..079b2bf6b 100644 --- a/dokimi/dokimi/report/arms_report.py +++ b/dokimi/dokimi/report/arms_report.py @@ -89,6 +89,14 @@ def _significance_lines( summary: Dict[str, Any], arms: List[str], baseline: str ) -> List[str]: """Paired significance table: every arm vs the one shared baseline.""" + refused = [ + f"_`{arm}` not computed: {error}_" + for arm in arms + if arm != baseline and (error := summary.get("per_arm", {}).get(arm, {}).get("delta_error")) + ] + notes = [line for note in refused for line in (note, "")] + if error := summary.get("significance_error"): + return ["## Paired significance vs baseline", "", f"_Not computed: {error}_", "", *notes] rows = [ (arm, summary["per_arm"][arm]["significance"]) for arm in arms @@ -96,7 +104,7 @@ def _significance_lines( and isinstance(summary.get("per_arm", {}).get(arm, {}).get("significance"), dict) ] if not rows: - return [] + return ["## Paired significance vs baseline", "", *notes] if refused else [] correction = next( (sig.get("correction") for _, sig in rows if sig.get("correction")), "holm" @@ -126,11 +134,16 @@ def _significance_lines( ) lines += _ttest_note([sig for _, sig in rows]) lines.append("") - return lines + return lines + notes def _interaction_lines(summary: Dict[str, Any]) -> List[str]: """Factorial interaction: does combining two treatments beat their sum?""" + if error := summary.get("interactions_error"): + return [ + "## Factorial interaction", "", + f"_Not computed: {error}_", "", + ] interactions = summary.get("interactions") or [] if not interactions: return [] diff --git a/dokimi/dokimi/schemas/arms.v1.json b/dokimi/dokimi/schemas/arms.v1.json index 9e3d6d1c7..ca58c79f8 100644 --- a/dokimi/dokimi/schemas/arms.v1.json +++ b/dokimi/dokimi/schemas/arms.v1.json @@ -125,6 +125,10 @@ "null" ] }, + "delta_error": { + "description": "Why delta_vs_baseline is null for this arm: its paired differences leave the float range. Present only on such a refusal.", + "type": "string" + }, "significance": { "description": "Paired test of this arm against the shared baseline arm. Absent on the baseline arm itself, and on arms with too few complete pairs to describe.", "type": "object", @@ -255,6 +259,14 @@ } } }, + "significance_error": { + "description": "Why per_arm..significance is absent from every arm: the paired tests of the arm family refused the scores (a difference or a sum of differences beyond the float range). Present only on such a refusal; an arm with too few pairs for the family leaves the others' significance in place.", + "type": "string" + }, + "interactions_error": { + "description": "Why interactions is absent: an interaction contrast was beyond the float range. Present only on such a refusal.", + "type": "string" + }, "interactions": { "description": "One entry per detected 2x2 factorial cell, i.e. per combined arm whose two sub-treatments are also present as standalone arms. Absent when no such cell exists.", "type": "array", diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 7a6a02baf..e31aefc5f 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -26,7 +26,6 @@ "dokimi/context/trace.py::_apply": {"complexity": 18}, "dokimi/eval/agent_runner.py::_summarize_tool_usage": {"complexity": 11}, "dokimi/eval/agent_runner.py::run_agent_loop": {"complexity": 14}, - "dokimi/eval/arm_runner.py::ArmRunner.judge": {"complexity": 11}, "dokimi/eval/checklist.py::ChecklistReport.format_markdown": {"complexity": 12}, "dokimi/eval/degrade.py::build_pairs": {"complexity": 17, "rationale": "grew on main in #73 (judge-validity --context stored) from 10 to 17 before the ratchet landed; split it in its own PR"}, "dokimi/eval/judge_sequential.py::run_sequential": {"complexity": 12, "rationale": "grew on main in #73 (judge-validity --context stored) from 6 to 12 before the ratchet landed; split it in its own PR"}, diff --git a/dokimi/tests/test_arm_runner_float_range.py b/dokimi/tests/test_arm_runner_float_range.py new file mode 100644 index 000000000..c1b89b4ff --- /dev/null +++ b/dokimi/tests/test_arm_runner_float_range.py @@ -0,0 +1,246 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A score range the statistics refuse (#599, #638) must not lose the arms run (#750).""" + +import json +import sys + +import dokimi.eval.arm_runner as arm_runner_mod +from dokimi.eval.arm_runner import ArmRunner +from dokimi.report.arms_report import render_arms_report +from dokimi.treatments import BaselineTreatment + +BIG = 1.7e308 +QUESTIONS = [ + {"id": "q1", "question": "How do I parallelize a loop?"}, + {"id": "q2", "question": "How do I sum an array in parallel?"}, +] + + +def _fake_call(prompt, model, provider="openai", api_key=None, system=None, **kw): + return "an answer", {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2} + + +class _Judge: + model = "fake-judge" + provider = "fake-provider" + + +def _ev(qid, scores): + return {"question_id": qid, "scores": {k: {"aggregate": v} for k, v in scores.items()}} + + +def _run(monkeypatch, arms, evaluations, questions=QUESTIONS): + monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", _fake_call) + runner = ArmRunner([BaselineTreatment(name) for name in arms]) + records = runner.run("oneTBB", questions, concurrency=1) + return runner, records, runner.build_output("oneTBB", records, evaluations, judge=_Judge()) + + +def _saved(runner, output, tmp_path): + path = tmp_path / "arms.json" + runner.save(output, path) + return json.loads(path.read_text(encoding="utf-8")) + + +def test_overflowing_paired_difference_keeps_the_run(monkeypatch, tmp_path, caplog): + evaluations = [ + _ev("q1", {"baseline": -BIG, "a": BIG}), + _ev("q2", {"baseline": 0.0, "a": 0.0}), + ] + runner, records, output = _run(monkeypatch, ["baseline", "a"], evaluations) + + assert output["answers"] == records + assert output["evaluations"] == evaluations + assert "significance_error, the run is kept without it" in caplog.text + summary = output["summary"] + assert "float range" in summary["significance_error"] + assert "significance" not in summary["per_arm"]["a"] + assert summary["per_arm"]["a"]["delta_vs_baseline"] is None + assert "float range" in summary["per_arm"]["a"]["delta_error"] + assert "interactions_error" not in summary + + saved = _saved(runner, output, tmp_path) + assert saved["evaluations"] == evaluations + assert saved["summary"]["significance_error"] == summary["significance_error"] + + +def test_overflowing_interaction_contrast_keeps_the_arm_significance(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": 0.0, "a": -BIG, "b": 0.0, "a+b": BIG}), + _ev("q2", {"baseline": 0.0, "a": 0.0, "b": 0.0, "a+b": 0.0}), + ] + runner, records, output = _run(monkeypatch, ["baseline", "a", "b", "a+b"], evaluations) + + assert output["answers"] == records + summary = output["summary"] + assert "float range" in summary["interactions_error"] + assert "interactions" not in summary + assert "significance_error" not in summary + assert summary["per_arm"]["a"]["significance"]["n"] == 2 + + saved = _saved(runner, output, tmp_path) + assert saved["summary"]["interactions_error"] == summary["interactions_error"] + assert saved["summary"]["per_arm"]["a"]["significance"]["n"] == 2 + + +def test_report_states_why_the_sections_are_missing(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": -BIG, "a": BIG}), + _ev("q2", {"baseline": 0.0, "a": 0.0}), + ] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + output["summary"]["interactions_error"] = "contrast too large" + text = render_arms_report(_saved(runner, output, tmp_path)) + + assert "Not computed: " + output["summary"]["significance_error"] in text + assert "Not computed: contrast too large" in text + + +def _strict_json(path): + def refuse(constant): + raise AssertionError(f"{constant} is not JSON") + + return json.loads(path.read_text(encoding="utf-8"), parse_constant=refuse) + + +def test_saved_averages_of_extreme_scores_are_finite_json(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": -BIG, "a": BIG}), + _ev("q2", {"baseline": -BIG, "a": BIG}), + ] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + path = tmp_path / "arms.json" + runner.save(output, path) + + per_arm = _strict_json(path)["summary"]["per_arm"] + assert per_arm["baseline"]["avg_aggregate"] == -BIG + assert per_arm["a"]["avg_aggregate"] == BIG + + +def test_saved_average_of_three_maximum_scores_is_the_maximum(monkeypatch, tmp_path): + top = sys.float_info.max + evaluations = [_ev(f"q{i}", {"baseline": top, "a": top}) for i in range(1, 4)] + questions = [{"id": f"q{i}", "question": f"question {i}"} for i in range(1, 4)] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations, questions) + path = tmp_path / "arms.json" + runner.save(output, path) + + per_arm = _strict_json(path)["summary"]["per_arm"] + assert per_arm["baseline"]["avg_aggregate"] == sys.float_info.max + assert per_arm["a"]["avg_aggregate"] == sys.float_info.max + + +def test_delta_whose_sum_leaves_the_range_is_averaged_exactly(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": 0.0, "a": 1.0e308}), + _ev("q2", {"baseline": 0.0, "a": 1.0e308}), + ] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + path = tmp_path / "arms.json" + runner.save(output, path) + + arm = _strict_json(path)["summary"]["per_arm"]["a"] + assert arm["delta_vs_baseline"] == 1.0e308 + assert "delta_error" not in arm + + +def test_cancelling_deltas_average_to_zero_like_the_scores(monkeypatch, tmp_path): + evaluations = [ + _ev("q1", {"baseline": 0.0, "a": 1.0e308}), + _ev("q2", {"baseline": 1.0e308, "a": 0.0}), + _ev("q3", {"baseline": 0.0, "a": 0.0}), + ] + questions = [{"id": f"q{i}", "question": f"question {i}"} for i in range(1, 4)] + runner, _, output = _run(monkeypatch, ["baseline", "a"], evaluations, questions) + path = tmp_path / "arms.json" + runner.save(output, path) + + arm = _strict_json(path)["summary"]["per_arm"]["a"] + assert arm["delta_vs_baseline"] == 0.0 + assert "delta_error" not in arm + + +def test_refused_delta_of_a_thin_arm_keeps_another_arms_significance(monkeypatch, tmp_path): + questions = [{"id": f"q{i}", "question": f"question {i}"} for i in range(1, 9)] + evaluations = [_ev("q1", {"baseline": -BIG, "a": BIG, "b": -BIG})] + [ + _ev(f"q{i}", {"baseline": 0.0, "b": 1.0}) for i in range(2, 9) + ] + runner, _, output = _run(monkeypatch, ["baseline", "a", "b"], evaluations, questions) + summary = output["summary"] + + assert "float range" in summary["per_arm"]["a"]["delta_error"] + assert summary["per_arm"]["a"]["delta_vs_baseline"] is None + assert "significance_error" not in summary + assert summary["per_arm"]["b"]["significance"]["n"] == 8 + text = render_arms_report(_saved(runner, output, tmp_path)) + assert "Not computed:" not in text + table = text.split("## Paired significance vs baseline")[1] + assert "| `b` |" in table + assert f"`a` not computed: {summary['per_arm']['a']['delta_error']}" in table + + +def test_console_summary_prints_an_arms_delta_refusal(monkeypatch, tmp_path, capsys): + import dokimi.eval.arms_run as arms_run + from dokimi.cli import build_parser + from dokimi.commands import arms as arms_cmd + + summary = {"per_arm": { + "baseline": {"avg_aggregate": 10.0, "n": 2, "errors": 0, "delta_vs_baseline": None}, + "a": {"avg_aggregate": 20.0, "n": 2, "errors": 0, "delta_vs_baseline": None, "delta_error": "too large"}, + "b": {"avg_aggregate": 30.0, "n": 2, "errors": 0, "delta_vs_baseline": 20.0}, + }} + monkeypatch.setattr(arms_cmd, "_load_questions", lambda path: QUESTIONS) + monkeypatch.setattr(arms_run, "prepare_arms", lambda *a, **k: None) + monkeypatch.setattr(arms_run, "run_prepared", lambda *a, **k: {"summary": summary}) + monkeypatch.setattr(arms_cmd, "_save_arms_output", lambda *a, **k: None) + args = build_parser().parse_args([ + "arms", "run", "--product", "p", "--questions", "q.json", "--arms", "baseline,a,b", + ]) + args.func(args) + + lines = capsys.readouterr().out.splitlines() + assert any(line.split()[:2] == ["a", "20.0"] and "(delta not computed: too large)" in line for line in lines) + assert any("(delta +20.0)" in line for line in lines) + + +class _ScriptedJudge(_Judge): + """Scores the first answer of a question -BIG (the baseline) and the second BIG on the first question.""" + + def __init__(self): + self.calls = {} + + def score_answer(self, library_name, question, answer, context, ground_truth): + n = self.calls[question] = self.calls.get(question, 0) + 1 + if question != QUESTIONS[0]["question"]: + return {"aggregate": 0.0} + return {"aggregate": -BIG if n == 1 else BIG} + + +def test_judged_overflowing_delta_is_refused_in_the_saved_artifact(monkeypatch, tmp_path): + monkeypatch.setattr(arm_runner_mod, "llm_call_with_usage", _fake_call) + runner = ArmRunner([BaselineTreatment("baseline"), BaselineTreatment("a")]) + records = runner.run("oneTBB", QUESTIONS, concurrency=1) + judge = _ScriptedJudge() + evaluations = runner.judge(judge, "oneTBB", records, concurrency=1) + output = runner.build_output("oneTBB", records, evaluations, judge=judge) + path = tmp_path / "arms.json" + runner.save(output, path) + + first, second = _strict_json(path)["evaluations"] + assert first["scores"]["a"]["aggregate"] == BIG + assert first["deltas_vs_baseline"] == {} + assert "float range" in first["delta_errors"]["a"] + assert second["deltas_vs_baseline"] == {"a": 0.0} + assert "delta_errors" not in second + + +def test_no_refusal_key_without_overflow(monkeypatch): + evaluations = [ + _ev("q1", {"baseline": 50.0, "a": 60.0}), + _ev("q2", {"baseline": 52.0, "a": 55.0}), + ] + _, _, output = _run(monkeypatch, ["baseline", "a"], evaluations) + + assert not {"significance_error", "interactions_error"} & set(output["summary"])