Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions loopx/capabilities/benchmark_toolkit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1200,6 +1200,11 @@ For a later comparison, call `experiment-board-show` with the same Goal/project
route and relevant benchmark/study filters. Reuse a current board already read
when it answers the decision; avoid rebuilding and returning it after every
single-row update. Validation and invalid-transition rejection still apply.
An initially fail-closed `runner_invalid` row may advance to `completed` when a
later deterministic terminal adjudication supplies a schema-valid completed row
for the same run identity. This promotion is one-way: completed and cancelled
rows cannot be reopened or demoted. Provider-ledger reconciliation continues to
reject conflicting terminal shards because it has no single adjudication owner.

The default ledger is locked, atomically updated, and keyed by benchmark, study,
case, and run identity. A compact row carries arm role, exact and comparison
Expand Down
5 changes: 4 additions & 1 deletion loopx/capabilities/benchmark_toolkit/experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@
"planned": _RUN_STATUSES,
"running": {"running", "completed", "runner_invalid", "cancelled"},
"completed": {"completed"},
"runner_invalid": {"runner_invalid"},
# A runner may fail closed before post-run evidence is fully reconciled.
# Permit the same stable run to advance once to a validated completed row;
# completed and cancelled rows remain immutable.
"runner_invalid": {"runner_invalid", "completed"},
"cancelled": {"cancelled"},
}
_TERMINAL_RUN_STATUSES = {"completed", "runner_invalid", "cancelled"}
Expand Down
29 changes: 29 additions & 0 deletions tests/capabilities/test_benchmark_experiment_board.py
Original file line number Diff line number Diff line change
Expand Up @@ -792,6 +792,35 @@ def test_upsert_rejects_stale_transition_after_terminal_state(tmp_path: Path) ->
assert rows[0]["status"] == "completed"


def test_upsert_promotes_runner_invalid_after_terminal_adjudication(
tmp_path: Path,
) -> None:
ledger = tmp_path / "experiment-board.jsonl"
runner_invalid = _running_baseline(
status="runner_invalid",
observed_at="2026-08-18T00:10:00+00:00",
)
completed = _baseline(observed_at="2026-08-18T00:20:00+00:00")

upsert_benchmark_experiment_board_row(ledger, runner_invalid)
receipt = upsert_benchmark_experiment_board_row(ledger, completed)
rows = read_benchmark_experiment_board_rows(ledger)

assert receipt["write"]["status"] == "updated"
assert rows == [normalize_benchmark_experiment_board_row(completed)]


def test_upsert_does_not_demote_completed_to_runner_invalid(tmp_path: Path) -> None:
ledger = tmp_path / "experiment-board.jsonl"
upsert_benchmark_experiment_board_row(ledger, _baseline())
runner_invalid = _running_baseline(status="runner_invalid")

with pytest.raises(
ValueError, match="cannot move from completed to runner_invalid"
):
upsert_benchmark_experiment_board_row(ledger, runner_invalid)


def test_reconcile_promotes_terminal_and_skips_late_nonterminal_source() -> None:
canonical = _running_baseline(observed_at="2026-08-18T00:10:00+00:00")
source_running = _running_baseline(
Expand Down
Loading