Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions dokimi/docs/repo-evaluation-contract.md
Original file line number Diff line number Diff line change
Expand Up @@ -2515,6 +2515,22 @@ python cli.py repo compare repo-runs/authoring --baseline guidance:none --metric
cannot be read (no read permission, a directory, not UTF-8 text) or parsed
as JSON (a trailing comma, a BOM) is refused the same way, exit 2 before
writing and naming the path, since whether it holds labels is not known.
- **The measure bound.** `repo report` reads a score, cost, turn count, token count
or `process` count, and `repo compare` also reads a `--metric` value, as a measurement only
when it is finite and its magnitude is at most 1e150 (`SCORE_BOUND` in
`dokimi/measure.py`), so that sums and squared deltas stay finite; a value past
it is no measurement. A `score` metric past it makes the grade a grader failure
(`grade.json has no numeric metrics.<name> that is finite and within +-1e+150`;
a stored `result.json` whose `grade.score` is past it is read as one too), and
`repo compare` counts that grader failure as a score of 0 in its mean, and
a `cost_usd`, `num_turns`, token count or `score_max` past it makes `repo
report` and `repo compare` refuse that `result.json`. Any other value past it
(a `--metric`, a `process` count, `first_prompt_tokens`, `first_useful_*`) is
left out of the mean and of `n`, as a value that is absent. In `repo compare`
a task whose `metric:x` is absent in either arm is unpaired (`no metric:x in
baseline or treatment`), so three tasks whose `metric:x` is 1.7e308 in both
arms give no `metric:x` row at all, not an overflow, and the same three at
1e150 give a row with means of 1e150.
- `repo compare` reduces cells to one value per (task, arm, model) first:
repetitions share a task's difficulty and are not paired across arms, so the
task is the unit. Per treatment arm, model and measure (gate share, score,
Expand Down
34 changes: 33 additions & 1 deletion dokimi/tests/test_repo_compare_overflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
import pytest

from dokimi.eval.finite_delta import checked_delta, checked_mean
from dokimi.repos.compare import CompareError, _inconclusive_note, _pair, _round, sign_flip_summary
from dokimi.repos.compare import CompareError, _compare_measure, _inconclusive_note, _pair, _round, sign_flip_summary
from dokimi.repos.compare_values import task_values
from dokimi.repos.compare_reach import min_detectable
from dokimi.repos.grading import SCORE_BOUND


def _side(value: float) -> dict:
Expand Down Expand Up @@ -178,6 +180,36 @@ def test_sign_flip_summary_mean_is_the_exact_mean():
assert row["delta"]["mean"] == 0.5


def test_sign_flip_summary_leaves_values_past_the_measure_bound_out_of_the_means():
# repo-evaluation-contract.md, "The measure bound": a value past SCORE_BOUND is no measurement, so its mean is None, not an overflow (#866)
def rows(value: float) -> list[dict]:
return [{"task": f"t{i}", "baseline": value, "treatment": value, "delta": 0.0} for i in range(3)]

for value in (1.7e308, 2 * SCORE_BOUND):
past = sign_flip_summary(rows(value), "metric:x")
assert (past["baseline_mean"], past["treatment_mean"]) == (None, None)
mixed = rows(1.0) + rows(2 * SCORE_BOUND)
kept = sign_flip_summary(mixed, "metric:x")
assert (kept["baseline_mean"], kept["treatment_mean"]) == (1.0, 1.0)
at = sign_flip_summary(rows(SCORE_BOUND), "metric:x")
assert (at["baseline_mean"], at["treatment_mean"]) == (SCORE_BOUND, SCORE_BOUND)


def test_compare_gives_no_row_for_a_metric_past_the_measure_bound_and_lists_its_tasks_unpaired():
def arm(value: float) -> dict:
cells = [{"model": "m", "task": f"t{i}", "batch": "b", "cell": f"c{i}", "score_task": False, "agent": {},
"metrics": {"x": value}, "grade": {"status": "ok", "success": True}} for i in range(3)]
return task_values(cells, ("x",))

shared = ["t0", "t1", "t2"]
unpaired: list[dict] = []
past = _compare_measure("tr", "m", "metric:x", shared, arm(1.7e308), arm(1.7e308), 0, unpaired)
assert past is None
assert [u["reason"] for u in unpaired] == ["no metric:x in baseline or treatment"] * 3
at = _compare_measure("tr", "m", "metric:x", shared, arm(SCORE_BOUND), arm(SCORE_BOUND), 0, [])
assert at is not None and at["baseline_mean"] == SCORE_BOUND


def test_checked_delta_refuses_an_integer_beyond_the_float_range_with_the_requested_error():
with pytest.raises(CompareError, match="float range"):
checked_delta(10**309, 0.0, CompareError)
Expand Down
Loading