From 7d9cf371b1ce623389723a498d8bd1bdd882ab2f Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 17:26:47 +0000 Subject: [PATCH 1/3] docs(repo): state the 1e150 measure bound of repo report and repo compare, and pin it at sign_flip_summary (#866) The issue reports baseline_mean and treatment_mean None for three pairs of 1.7e308 and blames an overflowing plain sum. The sum never runs: _mean reads its values through finite_values, which keeps only numbers within SCORE_BOUND (1e150, so that sums and squared deltas stay finite), so the three values are left out and the mean of nothing is None. The same rows at 1e150 give 1e+150. The bound is deliberate and no page of docs/ stated it. repo-evaluation-contract.md now says which values count as a measurement and what happens to one past the bound: a score past it is a grader failure, a cost, turn, token or score_max past it makes the result.json refused, and any other measure past it is left out of the mean. test_repo_compare_overflow.py pins that consequence at sign_flip_summary. No code changes. Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 13 +++++++++++++ dokimi/tests/test_repo_compare_overflow.py | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d68d1e783..6854f9470 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -2515,6 +2515,19 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric cannot be read (no read permission, a directory, not UTF-8 text) or parsed as JSON (a trailing comma, a BOM) is refused the same way, exit 2 before writing and naming the path, since whether it holds labels is not known. +- **The measure bound.** `repo report` and `repo compare` read a score, `--metric` + value, cost, turn count, token count or `process` count as a measurement only + when it is finite and its magnitude is at most 1e150 (`SCORE_BOUND` in + `dokimi/measure.py`), so that sums and squared deltas stay finite; a value past + it is no measurement. A `score` metric past it makes the grade a grader failure + (`grade.json has no numeric metrics. that is finite and within +-1e+150`; + a stored `result.json` whose `grade.score` is past it is read as one too), and + a `cost_usd`, `num_turns`, token count or `score_max` past it makes `repo + report` and `repo compare` refuse that `result.json`. Any other value past it + (a `--metric`, a `process` count, `first_prompt_tokens`, `first_useful_*`) is + left out of the mean and of `n`, as a value that is absent: three tasks whose + `metric:x` is 1.7e308 in both arms give a mean of `null`, not an overflow, and + the same three at 1e150 give 1e150. - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index d4db5567e..ec7f2d7c9 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -14,6 +14,7 @@ from dokimi.eval.finite_delta import checked_delta, checked_mean from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary from dokimi.repos.compare_reach import min_detectable +from dokimi.repos.grading import SCORE_BOUND def _side(value: float) -> dict: @@ -178,6 +179,17 @@ def test_sign_flip_summary_mean_is_the_exact_mean(): assert row["delta"]["mean"] == 0.5 +def test_sign_flip_summary_leaves_values_past_the_measure_bound_out_of_the_means(): + # repo-evaluation-contract.md, "The measure bound": a value past SCORE_BOUND is no measurement, so its mean is None, not an overflow (#866) + def rows(value: float) -> list[dict]: + return [{"task": f"t{i}", "baseline": value, "treatment": value, "delta": 0.0} for i in range(3)] + + past = sign_flip_summary(rows(1.7e308), "metric:x") + assert (past["baseline_mean"], past["treatment_mean"]) == (None, None) + at = sign_flip_summary(rows(SCORE_BOUND), "metric:x") + assert (at["baseline_mean"], at["treatment_mean"]) == (SCORE_BOUND, SCORE_BOUND) + + def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error(): with pytest.raises(CompareError, match="float range"): checked_delta(10**309, 0.0, CompareError) From a1c134e68c1ffbc0e3586bd0e48d1252123fd404 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 19:09:00 +0000 Subject: [PATCH 2/3] docs(repo): scope the measure-bound example to repo compare's omitted row; pin out-of-bound means Settles the two threads of the review of #880. The contract promised a JSON mean of null for three tasks at 1.7e308, but repo compare drops such a metric to absent, lists the tasks as unpaired and emits no row; the example now says that, and a test runs task_values and _compare_measure to pin it. The sign_flip_summary test also passed with the bound removed, because three 1.7e308 values sum to inf and round to None; it now uses 2 * SCORE_BOUND (a sum that stays finite) and a mix of a valid and an out-of-bound arm. Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 8 ++++--- dokimi/tests/test_repo_compare_overflow.py | 26 +++++++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 6854f9470..49a5cdccd 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -2525,9 +2525,11 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric a `cost_usd`, `num_turns`, token count or `score_max` past it makes `repo report` and `repo compare` refuse that `result.json`. Any other value past it (a `--metric`, a `process` count, `first_prompt_tokens`, `first_useful_*`) is - left out of the mean and of `n`, as a value that is absent: three tasks whose - `metric:x` is 1.7e308 in both arms give a mean of `null`, not an overflow, and - the same three at 1e150 give 1e150. + left out of the mean and of `n`, as a value that is absent. In `repo compare` + a task whose `metric:x` is absent in either arm is unpaired (`no metric:x in + baseline or treatment`), so three tasks whose `metric:x` is 1.7e308 in both + arms give no `metric:x` row at all, not an overflow, and the same three at + 1e150 give a row with means of 1e150. - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index ec7f2d7c9..c2dee0bed 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -12,7 +12,8 @@ import pytest from dokimi.eval.finite_delta import checked_delta, checked_mean -from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary +from dokimi.repos.compare import CompareError, _compare_measure, _inconclusive_note, _pair, _round, sign_flip_summary +from dokimi.repos.compare_values import task_values from dokimi.repos.compare_reach import min_detectable from dokimi.repos.grading import SCORE_BOUND @@ -184,12 +185,31 @@ def test_sign_flip_summary_leaves_values_past_the_measure_bound_out_of_the_means def rows(value: float) -> list[dict]: return [{"task": f"t{i}", "baseline": value, "treatment": value, "delta": 0.0} for i in range(3)] - past = sign_flip_summary(rows(1.7e308), "metric:x") - assert (past["baseline_mean"], past["treatment_mean"]) == (None, None) + for value in (1.7e308, 2 * SCORE_BOUND): + past = sign_flip_summary(rows(value), "metric:x") + assert (past["baseline_mean"], past["treatment_mean"]) == (None, None) + mixed = rows(1.0) + rows(2 * SCORE_BOUND) + kept = sign_flip_summary(mixed, "metric:x") + assert (kept["baseline_mean"], kept["treatment_mean"]) == (1.0, 1.0) at = sign_flip_summary(rows(SCORE_BOUND), "metric:x") assert (at["baseline_mean"], at["treatment_mean"]) == (SCORE_BOUND, SCORE_BOUND) +def test_compare_gives_no_row_for_a_metric_past_the_measure_bound_and_lists_its_tasks_unpaired(): + def arm(value: float) -> dict: + cells = [{"model": "m", "task": f"t{i}", "batch": "b", "cell": f"c{i}", "score_task": False, "agent": {}, + "metrics": {"x": value}, "grade": {"status": "ok", "success": True}} for i in range(3)] + return task_values(cells, ("x",)) + + shared = ["t0", "t1", "t2"] + unpaired: list[dict] = [] + past = _compare_measure("tr", "m", "metric:x", shared, arm(1.7e308), arm(1.7e308), 0, unpaired) + assert past is None + assert [u["reason"] for u in unpaired] == ["no metric:x in baseline or treatment"] * 3 + at = _compare_measure("tr", "m", "metric:x", shared, arm(SCORE_BOUND), arm(SCORE_BOUND), 0, []) + assert at is not None and at["baseline_mean"] == SCORE_BOUND + + def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error(): with pytest.raises(CompareError, match="float range"): checked_delta(10**309, 0.0, CompareError) From 2db449bd81d0bcc309874102519048e7e0d40891 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 22:10:15 +0000 Subject: [PATCH 3/3] docs(repo): the measure bound bullet names repo compare for --metric and states the score-as-0 mean Settles the review nit that only `repo compare` takes `--metric`: the bullet now says `repo report` reads a score, cost, turn count, token count or `process` count, and `repo compare` also reads a `--metric` value. Settles the open question on a score past the bound: the bullet states that `repo compare` counts that grader failure as a score of 0 in its mean (`task_values` in `repos/compare_values.py`). Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 49a5cdccd..db4f90fe7 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -2515,13 +2515,14 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric cannot be read (no read permission, a directory, not UTF-8 text) or parsed as JSON (a trailing comma, a BOM) is refused the same way, exit 2 before writing and naming the path, since whether it holds labels is not known. -- **The measure bound.** `repo report` and `repo compare` read a score, `--metric` - value, cost, turn count, token count or `process` count as a measurement only +- **The measure bound.** `repo report` reads a score, cost, turn count, token count + or `process` count, and `repo compare` also reads a `--metric` value, as a measurement only when it is finite and its magnitude is at most 1e150 (`SCORE_BOUND` in `dokimi/measure.py`), so that sums and squared deltas stay finite; a value past it is no measurement. A `score` metric past it makes the grade a grader failure (`grade.json has no numeric metrics. that is finite and within +-1e+150`; a stored `result.json` whose `grade.score` is past it is read as one too), and + `repo compare` counts that grader failure as a score of 0 in its mean, and a `cost_usd`, `num_turns`, token count or `score_max` past it makes `repo report` and `repo compare` refuse that `result.json`. Any other value past it (a `--metric`, a `process` count, `first_prompt_tokens`, `first_useful_*`) is