diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index d68d1e783..db4f90fe7 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -2515,6 +2515,22 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric cannot be read (no read permission, a directory, not UTF-8 text) or parsed as JSON (a trailing comma, a BOM) is refused the same way, exit 2 before writing and naming the path, since whether it holds labels is not known. +- **The measure bound.** `repo report` reads a score, cost, turn count, token count + or `process` count, and `repo compare` also reads a `--metric` value, as a measurement only + when it is finite and its magnitude is at most 1e150 (`SCORE_BOUND` in + `dokimi/measure.py`), so that sums and squared deltas stay finite; a value past + it is no measurement. A `score` metric past it makes the grade a grader failure + (`grade.json has no numeric metrics. that is finite and within +-1e+150`; + a stored `result.json` whose `grade.score` is past it is read as one too), and + `repo compare` counts that grader failure as a score of 0 in its mean, and + a `cost_usd`, `num_turns`, token count or `score_max` past it makes `repo + report` and `repo compare` refuse that `result.json`. Any other value past it + (a `--metric`, a `process` count, `first_prompt_tokens`, `first_useful_*`) is + left out of the mean and of `n`, as a value that is absent. In `repo compare` + a task whose `metric:x` is absent in either arm is unpaired (`no metric:x in + baseline or treatment`), so three tasks whose `metric:x` is 1.7e308 in both + arms give no `metric:x` row at all, not an overflow, and the same three at + 1e150 give a row with means of 1e150. - `repo compare` reduces cells to one value per (task, arm, model) first: repetitions share a task's difficulty and are not paired across arms, so the task is the unit. Per treatment arm, model and measure (gate share, score, diff --git a/dokimi/tests/test_repo_compare_overflow.py b/dokimi/tests/test_repo_compare_overflow.py index d4db5567e..c2dee0bed 100644 --- a/dokimi/tests/test_repo_compare_overflow.py +++ b/dokimi/tests/test_repo_compare_overflow.py @@ -12,8 +12,10 @@ import pytest from dokimi.eval.finite_delta import checked_delta, checked_mean -from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary +from dokimi.repos.compare import CompareError, _compare_measure, _inconclusive_note, _pair, _round, sign_flip_summary +from dokimi.repos.compare_values import task_values from dokimi.repos.compare_reach import min_detectable +from dokimi.repos.grading import SCORE_BOUND def _side(value: float) -> dict: @@ -178,6 +180,36 @@ def test_sign_flip_summary_mean_is_the_exact_mean(): assert row["delta"]["mean"] == 0.5 +def test_sign_flip_summary_leaves_values_past_the_measure_bound_out_of_the_means(): + # repo-evaluation-contract.md, "The measure bound": a value past SCORE_BOUND is no measurement, so its mean is None, not an overflow (#866) + def rows(value: float) -> list[dict]: + return [{"task": f"t{i}", "baseline": value, "treatment": value, "delta": 0.0} for i in range(3)] + + for value in (1.7e308, 2 * SCORE_BOUND): + past = sign_flip_summary(rows(value), "metric:x") + assert (past["baseline_mean"], past["treatment_mean"]) == (None, None) + mixed = rows(1.0) + rows(2 * SCORE_BOUND) + kept = sign_flip_summary(mixed, "metric:x") + assert (kept["baseline_mean"], kept["treatment_mean"]) == (1.0, 1.0) + at = sign_flip_summary(rows(SCORE_BOUND), "metric:x") + assert (at["baseline_mean"], at["treatment_mean"]) == (SCORE_BOUND, SCORE_BOUND) + + +def test_compare_gives_no_row_for_a_metric_past_the_measure_bound_and_lists_its_tasks_unpaired(): + def arm(value: float) -> dict: + cells = [{"model": "m", "task": f"t{i}", "batch": "b", "cell": f"c{i}", "score_task": False, "agent": {}, + "metrics": {"x": value}, "grade": {"status": "ok", "success": True}} for i in range(3)] + return task_values(cells, ("x",)) + + shared = ["t0", "t1", "t2"] + unpaired: list[dict] = [] + past = _compare_measure("tr", "m", "metric:x", shared, arm(1.7e308), arm(1.7e308), 0, unpaired) + assert past is None + assert [u["reason"] for u in unpaired] == ["no metric:x in baseline or treatment"] * 3 + at = _compare_measure("tr", "m", "metric:x", shared, arm(SCORE_BOUND), arm(SCORE_BOUND), 0, []) + assert at is not None and at["baseline_mean"] == SCORE_BOUND + + def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error(): with pytest.raises(CompareError, match="float range"): checked_delta(10**309, 0.0, CompareError)