From ad2687113a6b62e9ebe6efcb6f2628ced3f48910 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Tue, 10 Nov 2026 01:41:59 +0800 Subject: [PATCH] fix(benchmark): allow adjudicated runner recovery Signed-off-by: huangruiteng --- .../capabilities/benchmark_toolkit/README.md | 5 ++++ .../benchmark_toolkit/experiment_board.py | 5 +++- .../test_benchmark_experiment_board.py | 29 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/loopx/capabilities/benchmark_toolkit/README.md b/loopx/capabilities/benchmark_toolkit/README.md index 4b998c809f..16d22b2490 100644 --- a/loopx/capabilities/benchmark_toolkit/README.md +++ b/loopx/capabilities/benchmark_toolkit/README.md @@ -1200,6 +1200,11 @@ For a later comparison, call `experiment-board-show` with the same Goal/project route and relevant benchmark/study filters. Reuse a current board already read when it answers the decision; avoid rebuilding and returning it after every single-row update. Validation and invalid-transition rejection still apply. +An initially fail-closed `runner_invalid` row may advance to `completed` when a +later deterministic terminal adjudication supplies a schema-valid completed row +for the same run identity. This promotion is one-way: completed and cancelled +rows cannot be reopened or demoted. Provider-ledger reconciliation continues to +reject conflicting terminal shards because it has no single adjudication owner. The default ledger is locked, atomically updated, and keyed by benchmark, study, case, and run identity. A compact row carries arm role, exact and comparison diff --git a/loopx/capabilities/benchmark_toolkit/experiment_board.py b/loopx/capabilities/benchmark_toolkit/experiment_board.py index 22fa8c198a..9fd3617bf5 100644 --- a/loopx/capabilities/benchmark_toolkit/experiment_board.py +++ b/loopx/capabilities/benchmark_toolkit/experiment_board.py @@ -28,7 +28,10 @@ "planned": _RUN_STATUSES, "running": {"running", "completed", "runner_invalid", "cancelled"}, "completed": {"completed"}, - "runner_invalid": {"runner_invalid"}, + # A runner may fail closed before post-run evidence is fully reconciled. + # Permit the same stable run to advance once to a validated completed row; + # completed and cancelled rows remain immutable. + "runner_invalid": {"runner_invalid", "completed"}, "cancelled": {"cancelled"}, } _TERMINAL_RUN_STATUSES = {"completed", "runner_invalid", "cancelled"} diff --git a/tests/capabilities/test_benchmark_experiment_board.py b/tests/capabilities/test_benchmark_experiment_board.py index 70c8a8c846..bb2f79d658 100644 --- a/tests/capabilities/test_benchmark_experiment_board.py +++ b/tests/capabilities/test_benchmark_experiment_board.py @@ -792,6 +792,35 @@ def test_upsert_rejects_stale_transition_after_terminal_state(tmp_path: Path) -> assert rows[0]["status"] == "completed" +def test_upsert_promotes_runner_invalid_after_terminal_adjudication( + tmp_path: Path, +) -> None: + ledger = tmp_path / "experiment-board.jsonl" + runner_invalid = _running_baseline( + status="runner_invalid", + observed_at="2026-08-18T00:10:00+00:00", + ) + completed = _baseline(observed_at="2026-08-18T00:20:00+00:00") + + upsert_benchmark_experiment_board_row(ledger, runner_invalid) + receipt = upsert_benchmark_experiment_board_row(ledger, completed) + rows = read_benchmark_experiment_board_rows(ledger) + + assert receipt["write"]["status"] == "updated" + assert rows == [normalize_benchmark_experiment_board_row(completed)] + + +def test_upsert_does_not_demote_completed_to_runner_invalid(tmp_path: Path) -> None: + ledger = tmp_path / "experiment-board.jsonl" + upsert_benchmark_experiment_board_row(ledger, _baseline()) + runner_invalid = _running_baseline(status="runner_invalid") + + with pytest.raises( + ValueError, match="cannot move from completed to runner_invalid" + ): + upsert_benchmark_experiment_board_row(ledger, runner_invalid) + + def test_reconcile_promotes_terminal_and_skips_late_nonterminal_source() -> None: canonical = _running_baseline(observed_at="2026-08-18T00:10:00+00:00") source_running = _running_baseline(