From 01f4f571067f86645030f694b7faaf4ddbc3cc16 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 4 Nov 2026 23:51:53 +0000 Subject: [PATCH 01/11] feat: restore rollouts from filesystem checkpoints An agent snapshots its prime box as model turns go upstream, and a rollout retry restores the newest snapshot in a new box and continues the trace from there instead of starting over. On by default (`[env.agent.checkpoint]`: interval, keep, after, boxless). The trace records each snapshot (`Trace.checkpoints`), rewinds to the one a retry restores (`Trace.rewind`, `Trace.generation`), and gives another process what it needs to continue it (`Trace.restorable`, `RestorePoint`, `Env.run_slot(restore=...)`). null and bash relaunch on the transcript; claude-code, codex and rlm load their ACP session from the restored box (rlm pinned to nano-rlm with session/load). A request hook that raises now ends the rollout with its error. This is the filesystem-only state of verifiers#2994 as of ebfd9f8d2, on today's main. Its env-server parts moved with the env server to prime-rl and are not part of this change. Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 6 + docs/harnesses.md | 1 + tests/unit/test_agent.py | 34 ++ tests/unit/test_checkpoint.py | 315 +++++++++++++ tests/unit/test_retry_ids.py | 4 + verifiers/v1/__init__.py | 3 + verifiers/v1/acp/__init__.py | 35 +- verifiers/v1/acp/runner.py | 44 +- verifiers/v1/agent.py | 230 ++++++++-- verifiers/v1/configs/agent.py | 42 +- verifiers/v1/env.py | 21 +- verifiers/v1/harness.py | 36 +- verifiers/v1/harnesses/bash/harness.py | 1 + verifiers/v1/harnesses/claude_code/harness.py | 9 +- verifiers/v1/harnesses/codex/harness.py | 4 +- verifiers/v1/harnesses/null/harness.py | 1 + verifiers/v1/harnesses/rlm/harness.py | 18 +- verifiers/v1/interception/server.py | 46 +- verifiers/v1/rollout.py | 428 +++++++++++++++--- verifiers/v1/runtimes/backends/prime.py | 64 ++- verifiers/v1/runtimes/base.py | 27 ++ verifiers/v1/session.py | 33 +- verifiers/v1/trace.py | 194 +++++++- 23 files changed, 1461 insertions(+), 135 deletions(-) create mode 100644 tests/unit/test_checkpoint.py diff --git a/docs/agent.md b/docs/agent.md index db057cc754..7ef3334272 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -72,3 +72,9 @@ max_turn_tokens = 4096 skip_on_truncated = true skip_on_timeout = true ``` + +## Retries and checkpoints + +`[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. + +Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box as model turns go upstream, and a retry restores the newest snapshot and continues the trace from there. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_CHECKPOINTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `interval` sets how often it snapshots, `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. diff --git a/docs/harnesses.md b/docs/harnesses.md index 54b1f418a1..e634a3e746 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,6 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). +- `SUPPORTS_FILESYSTEM_CHECKPOINTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem checkpoint, so a retry restores the newest checkpoint instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..3e4708edcd 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -18,6 +18,7 @@ class FailingSegmentRollout: closed = Rollout.closed fail = Rollout.fail step = Rollout.step + _turn = Rollout._turn timeout = Rollout.timeout @@ -39,6 +40,7 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._opened = True run._closed = False run._failed = False + run._attempt = 0 run._failure = None run._borrowed_runtime = None run.runtime = None @@ -49,6 +51,8 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._session = SimpleNamespace( request_interceptors=[], error=None, + aborted=None, + abort_event=asyncio.Event(), stopped=False, ) run.deadline_at = None @@ -99,3 +103,33 @@ async def turn(self, messages): assert trace.is_timeout and not trace.errors assert trace.stop_condition == "agent_timeout" + + +@pytest.mark.parametrize("outcome", ["keeps waiting", "dies"]) +async def test_request_hook_failure_fails_the_rollout(outcome): + """A request hook's failure is the task's verdict: it ends the segment at once and + fails the rollout, whatever the harness does meanwhile.""" + trace = make_trace() + error = vf.ProviderError("hook refused the turn", status_code=503) + cancelled = [] + + class WaitingSession: + async def turn(self, messages): + run._session.aborted = error + run._session.abort_event.set() + try: + if outcome == "keeps waiting": + await asyncio.sleep(60) + else: + raise RuntimeError("harness died") + except asyncio.CancelledError: + cancelled.append(True) + raise + + run = make_rollout(trace, WaitingSession(), time_remaining=5) + + await asyncio.wait_for(Interaction(run).turn("next"), timeout=1) + + assert [(e.type, e.stage) for e in trace.errors] == [("ProviderError", "agent")] + assert not trace.is_timeout + assert cancelled == ([True] if outcome == "keeps waiting" else []) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py new file mode 100644 index 0000000000..19b1918330 --- /dev/null +++ b/tests/unit/test_checkpoint.py @@ -0,0 +1,315 @@ +"""Rollout checkpointing: when a rollout takes checkpoints, what `rewind` restores, and +what a restored attempt must not redo.""" + +import asyncio +from types import SimpleNamespace +from typing import Literal + +import verifiers.v1 as vf +import verifiers.v1.agent as agent_module +from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.agent import RolloutCheckpointConfig +from verifiers.v1.configs.client import EvalClientConfig +from verifiers.v1.graph import MessageNode +from verifiers.v1.rollout import Rollout, kept +from verifiers.v1.session import RolloutSession +from verifiers.v1.trace import ( + Checkpoint, + ModelCall, + RestorePoint, + TimeSpan, + TraceTask, +) +from verifiers.v1.types import ( + AssistantMessage, + Request, + ToolCall, + ToolMessage, + UserMessage, +) + + +def make_trace() -> vf.Trace: + trace = vf.Trace( + agent=vf.AgentInfo(config=vf.AgentConfig()), + task=TraceTask(type="TaskData", data=vf.TaskData(idx=0, prompt="q")), + ) + trace.nodes.append(MessageNode(parent=None, message=UserMessage(content="q"))) + trace.nodes.append( + MessageNode( + parent=0, + message=AssistantMessage( + content="", + tool_calls=[ToolCall(id="c1", name="bash", arguments="{}")], + ), + sampled=True, + ) + ) + trace.calls.append(ModelCall(node=1, time=TimeSpan(start=10.0, end=40.0))) + return trace + + +def test_rewind_restores_info_with_state(): + trace = make_trace() + checkpoint = Checkpoint(id="ck", node=1, info={"setup": "nonce"}) + trace.checkpoints.append(checkpoint) + trace.info = {"setup": "nonce", "scoring_skipped": "stale"} + trace.rewind(checkpoint) + assert trace.info == {"setup": "nonce"} + + +def test_rewind_keeps_the_dropped_turns_calls(): + trace = make_trace() + checkpoint = Checkpoint(id="ck", node=0) + trace.checkpoints.append(checkpoint) + trace.rewind(checkpoint) + assert len(trace.nodes) == 1 + assert [call.node for call in trace.calls] == [None] + + +def test_restored_span_counts_only_its_own_calls(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.split_agent_time() + assert trace.timing.agent.model.duration == 5.0 + assert trace.timing.agent.harness.duration == 15.0 + + +def test_agent_time_splits_off_the_checkpoint_hold(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.checkpoints += [ + Checkpoint(id="old", captured=90.0, held=3.0), + Checkpoint(id="new", captured=106.0, held=1.5), + ] + trace.split_agent_time() + assert trace.timing.agent.checkpoint.duration == 1.5 + assert trace.timing.agent.model.duration == 3.5 + assert trace.timing.agent.harness.duration == 15.0 + + +async def test_restored_tail_is_not_intercepted_twice(): + trace = make_trace() + seen: list[str] = [] + + def tag(request: Request) -> Request: + for message in request.messages: + if isinstance(message, ToolMessage | UserMessage): + seen.append(str(message.content)) + if isinstance(message, ToolMessage): + message.content = f"{message.content} [tagged]" + return request + + session = RolloutSession( + ctx=ModelContext(model="m", client=EvalClientConfig()), + trace=trace, + request_interceptors=[tag], + ) + tail = [ + ToolMessage(tool_call_id="c1", content="out [tagged]"), + UserMessage(content="go on"), + ] + session.mark_intercepted(trace.messages, tail) + request = Request(messages=[*trace.messages, *tail]) + rewritten, records, _ = await session.rewrite_request(request) + assert rewritten.messages == request.messages + assert records == [] + assert seen == [] + + +class SlowBackend: + """Captures at once; becomes durable when the test says so.""" + + def __init__(self) -> None: + self.durable = asyncio.Event() + self.fail = False + self.taken = 0 + self.deleted: list[str] = [] + + async def checkpoint(self) -> str: + self.taken += 1 + return f"ck{self.taken}" + + async def wait_checkpoint(self, checkpoint: str) -> int | None: + await self.durable.wait() + if self.fail: + raise RuntimeError("upload failed") + return 1024 + + async def discard_checkpoints(self, ids: list[str]) -> list[str]: + self.deleted += ids + return ids + + +def make_rollout( + interval: int = 1, keep: Literal["all", "last"] | int = "last" +) -> tuple[Rollout, SlowBackend]: + backend = SlowBackend() + run = Rollout.__new__(Rollout) + run.trace = make_trace() # one turn so far + run.runtime = SimpleNamespace(backend=backend) # type: ignore[assignment] + + async def checkpoint_state() -> dict: + return {} + + run._harness_session = SimpleNamespace(checkpoint_state=checkpoint_state) # type: ignore[assignment] + run._session = SimpleNamespace(pinned_tool_results={}) # type: ignore[assignment] + run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) + run._durable = {} + run._captures = set() + run._closed = False + run._pruning = asyncio.Lock() + run.deleted_checkpoints = set() + run.rewound_checkpoints = [] + run._boxless = False + return run, backend + + +TURN = SimpleNamespace(prefix_node_ids=[0, 1], tail=[]) + + +async def test_checkpoint_interval_skips_turns(): + run, backend = make_rollout(interval=2) + assert await run._checkpoint(TURN) is None # turn 2: 1 % 2 + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + run.trace.nodes.append( + MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) + ) + assert await run._checkpoint(TURN) is not None # turn 3: 2 % 2 + assert backend.taken == 1 + + +async def test_checkpoint_becomes_durable_in_the_background(): + run, backend = make_rollout(interval=1) + checkpoint = await run._checkpoint(TURN) + assert checkpoint is not None and checkpoint.captured + assert checkpoint.time.end == 0 and checkpoint.bytes is None + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert checkpoint.time.end and checkpoint.bytes == 1024 + assert run.trace.checkpoints == [checkpoint] + + +async def test_a_checkpoint_that_never_becomes_durable_is_dropped(): + run, backend = make_rollout(interval=1) + await run._checkpoint(TURN) + backend.fail = True + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert run.trace.checkpoints == [] + + +async def test_settling_without_a_retry_stops_waiting(): + run, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint(TURN) + await asyncio.wait_for(run._settle_checkpoints(wait=False), timeout=1) + assert run._durable == {} + assert run.trace.checkpoints == [checkpoint] and checkpoint.time.end == 0 + + +async def test_a_closing_run_takes_no_checkpoint(): + run, backend = make_rollout(interval=1) + capture = asyncio.create_task(run._checkpoint(TURN)) + run._closed = True + await run._settle_checkpoints(wait=True) + assert await capture is None + assert backend.taken == 0 and run.trace.checkpoints == [] + + +def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: + return [ + Checkpoint(id=i, time=TimeSpan(start=1.0, end=2.0), deleted=i in deleted) + for i in ids + ] + + +def test_keep_policies(): + checkpoints = durable("a", "b", "c", "d", "e") + assert kept(checkpoints, "last") == {"e"} + assert kept(checkpoints, "all") == {"a", "b", "c", "d", "e"} + assert kept(checkpoints, 2) == {"a", "c", "e"} + assert kept(checkpoints, 3) == {"a", "d", "e"} + # Deleted and not yet durable snapshots are never kept. + pending = [*durable("a", "b", deleted=("a",)), Checkpoint(id="c")] + assert kept(pending, "all") == {"b"} + + +async def test_a_durable_snapshot_prunes_the_older_ones(): + run, backend = make_rollout(keep="last") + run.trace.checkpoints = durable("old1", "old2") + run.rewound_checkpoints = ["rewound"] + backend.durable.set() + newest = await run._checkpoint(TURN) + await run._settle_checkpoints(wait=True) + assert sorted(backend.deleted) == ["old1", "old2", "rewound"] + assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] + assert run.deleted_checkpoints == {"old1", "old2", "rewound"} + + +async def test_a_boxless_checkpoint_is_restorable_at_once(): + run, backend = make_rollout() + run._boxless = True + checkpoint = await run._checkpoint(TURN) + assert checkpoint is not None and not checkpoint.box + assert checkpoint.time.end and backend.taken == 0 + assert run.trace.restorable is checkpoint + + +def test_a_restore_point_round_trips(): + checkpoint = Checkpoint( + id="ck", + node=1, + tail=[ToolMessage(tool_call_id="c1", content="out")], + state=vf.State(), + info={"nonce": "n"}, + pins={("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned")}, + time=TimeSpan(start=1.0, end=2.0), + ) + wire = RestorePoint.of(checkpoint).model_dump(mode="json") + back = RestorePoint.model_validate(wire).live(vf.State) + assert back.tail == checkpoint.tail + assert back.info == {"nonce": "n"} + assert back.pins == checkpoint.pins + assert back.state == checkpoint.state + + +def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: + return vf.Agent( + vf.AgentConfig( + model="test-model", + harness={"id": harness}, + runtime=runtime, + checkpoint=RolloutCheckpointConfig(**checkpoint), + ) + ) + + +def test_checkpointing_is_on_by_default(): + assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() + assert gated("bash", vf.PrimeConfig())._checkpointing() is not None + + +def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): + monkeypatch.setattr(agent_module, "_checkpointing_warned", set()) + for _ in range(2): + assert gated("bash", vf.SubprocessConfig())._checkpointing() is None + assert gated("terminus-2", vf.PrimeConfig())._checkpointing() is None + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert warnings == [ + ( + "rollout checkpointing is off: the 'subprocess' runtime takes no " + "snapshots; a failed rollout retries from the start" + ), + ( + "rollout checkpointing is off: harness 'terminus-2' does not support it; " + "a failed rollout retries from the start" + ), + ] + + +def test_boxless_checkpointing_is_opt_in(): + assert gated("null", vf.SubprocessConfig())._checkpointing() is None + assert gated("null", vf.SubprocessConfig(), boxless=True)._checkpointing() + assert gated("bash", vf.SubprocessConfig(), boxless=True)._checkpointing() is None diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 80cb35978e..7c3d6f9613 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -33,7 +33,11 @@ async def test_agent_retries_keep_the_trace_id(monkeypatch): minted: list[vf.Trace] = [] class FakeRollout: + runtime = None + def __init__(self, *, on_trace, trace_id, attempt, **_): + self.rewound_checkpoints: list[str] = [] + self.deleted_checkpoints: set[str] = set() self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index ecaf8b4d6e..33ff9dd52b 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -119,6 +119,7 @@ AgentInfo, AgentSpan, Branch, + Checkpoint, Error, EvalRunInfo, EvalWorkInfo, @@ -209,6 +210,7 @@ "BaseConfig": "pydantic_config", "BindMount": "verifiers.v1.configs.runtime", "Branch": "verifiers.v1.trace", + "Checkpoint": "verifiers.v1.trace", "Client": "verifiers.v1.clients", "ClientConfig": "verifiers.v1.configs.client", "ComposeSpec": "verifiers.v1.runtimes.compose", @@ -430,6 +432,7 @@ "StateT", "MessageNode", "Branch", + "Checkpoint", "TurnTokens", "SamplingMask", "TopLogprobs", diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 4bed677ef5..d5fec9315b 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -14,7 +14,7 @@ from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.errors import HarnessError, InterceptionError -from verifiers.v1.harness import Harness, HarnessSession +from verifiers.v1.harness import RESTORE_PROMPT, Harness, HarnessSession from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess from verifiers.v1.semantic import ( ACP_SEMANTIC_EDGES_METADATA_KEY, @@ -22,7 +22,7 @@ ) from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace -from verifiers.v1.types import Messages +from verifiers.v1.types import Messages, UserMessage from verifiers.v1.utils.aio import run_shielded ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text() @@ -237,6 +237,10 @@ def __init__( self._responses: dict[int, asyncio.Future] = {} self._request_id = 0 self._response_task: asyncio.Task | None = None + self._session_id: str | None = None + """The agent's ACP session id, once a checkpoint asked for it.""" + self._restore_id: str | None = None + """The session the next process loads from the restored box (`restore`).""" async def _start(self) -> None: self._stderr_tail.clear() @@ -284,6 +288,28 @@ async def _request(self, operation: str, **payload: Any) -> JsonObject: if not future.done(): future.cancel() + async def checkpoint_state(self) -> dict[str, Any] | None: + # The agent keeps its session in files of the box, so a restored box can load + # it in a new process — if the agent advertises `loadSession`. + if self._session_id is None: + if self._process is None: + return None + response = await self._request("session") + result = response.get("result") + if not response.get("ok") or not isinstance(result, dict): + raise RuntimeError(response.get("error") or "ACP session query failed") + if not result.get("load_session") or result.get("session_id") is None: + return None + self._session_id = result["session_id"] + return {"acp_session_id": self._session_id} + + def restore(self, state: dict[str, Any], tail: Messages) -> Messages: + # The agent's own session files hold the conversation, the tail included as + # far as it flushed them; the restored process takes only the notice. + self._restore_id = state["acp_session_id"] + self._session_id = self._restore_id + return [UserMessage(content=RESTORE_PROMPT)] + async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed: raise RuntimeError("ACP session is closed") @@ -326,6 +352,7 @@ async def _run(self, messages: Messages | None) -> ProgramResult: ) config = { "command": self.config.command, + "load_session_id": self._restore_id, "user_contents": user_contents, "mcp_urls": self.mcp_urls, "mcp_headers": self.harness.config.resolve_mcp_headers(self.mcp_urls), @@ -354,6 +381,10 @@ async def _run(self, messages: Messages | None) -> ProgramResult: except BaseException: await run_shielded(self._stop(graceful=False)) raise + finally: + # Only the restored box's first process loads the session; later + # turns continue the live one. + self._restore_id = None turn = _turn_result(response) self.trace.root_reply = turn.reply.strip() self.trace.info["acp_stop_reason"] = turn.stop_reason diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index b19942b5b9..5a7ec185af 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -87,6 +87,8 @@ def __init__(self) -> None: self.stop_reason: str | None = None self.response_metadata: dict[str, Any] = {} self.update_metadata: list[dict[str, Any]] = [] + self.loading = False + """Set while `session/load` replays the history, which no turn reports.""" def reset(self) -> None: self.visible_reply = "" @@ -104,6 +106,8 @@ def turn_result(self) -> ACPTurn: ) async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None: + if self.loading: + return metadata = dict(kwargs) if isinstance(field_meta := getattr(update, "field_meta", None), dict): metadata.update(field_meta) @@ -267,18 +271,37 @@ async def start(self, config: dict) -> None: self.supports_steering = (initialized.field_meta or {}).get( "steering", {} ).get("supported") is True - session = await self.connection.new_session( - cwd=os.getcwd(), - mcp_servers=mcp_servers(config), - **config["session_meta"], - ) + load_id = config.get("load_session_id") + if load_id is None: + session = await self.connection.new_session( + cwd=os.getcwd(), + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) + session_id = session.session_id + else: + # A box restored from a checkpoint: the agent's session files are + # here, its process is not. Load the session in this new process. + if not (self.capabilities and self.capabilities.load_session): + raise RuntimeError("ACP agent does not support session/load") + self.client.loading = True + try: + await self.connection.load_session( + cwd=os.getcwd(), + session_id=load_id, + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) + finally: + self.client.loading = False + session_id = load_id except BaseException: with suppress(BaseException): await self.stack.aclose() self._reset() raise - self.session_id = session.session_id - self.is_new = True + self.session_id = session_id + self.is_new = load_id is None async def run(self, config: dict) -> ACPTurn: gate = config.get("tool_interception") @@ -381,6 +404,13 @@ async def handle(request: dict) -> None: result = await session.steer( request["message"], request.get("message_id") ) + elif operation == "session": + result = { + "session_id": session.session_id, + "load_session": bool( + session.capabilities and session.capabilities.load_session + ), + } elif operation == "shutdown": result = {"response_metadata": await session.close()} else: diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index ba94921c36..449ce0718d 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -25,6 +25,7 @@ ) from verifiers.v1.configs.agent import ( AgentConfig, + RolloutCheckpointConfig, TimeoutConfig, agent_config_fields, resolve_agent, @@ -34,7 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer -from verifiers.v1.rollout import Rollout, RolloutTimeouts +from verifiers.v1.rollout import Rollout, RolloutTimeouts, kept from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -44,7 +45,7 @@ ) from verifiers.v1.session import RolloutLimits from verifiers.v1.task import Task -from verifiers.v1.trace import Trace +from verifiers.v1.trace import Checkpoint, Error, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -63,6 +64,9 @@ logger = logging.getLogger(__name__) +# (harness id, runtime type) pairs already warned that they take no checkpoints. +_checkpointing_warned: set[tuple[str, str]] = set() + # Only nested runs in this scoring attempt participate in its rollback. _scoring_runs: ContextVar[list[tuple["_EpisodeAgent", Trace]] | None] = ContextVar( "scoring_runs", default=None @@ -571,6 +575,9 @@ def __init__( sampling=config.sampling, ) self._closed = False + self.pending_restore: tuple[Trace, Checkpoint] | None = None + """A restore the next `run` starts from (a trace another process ran); consumed + by that run.""" self.runtime_config: RuntimeConfig = config.runtime self.interception = interception self.limits = RolloutLimits( @@ -641,6 +648,34 @@ def _interception_for( return self._server return None + def _checkpointing(self) -> RolloutCheckpointConfig | None: + """The checkpoint config when this harness on this runtime can take and restore + checkpoints; otherwise None, with a warning once per harness and runtime.""" + config = self.config.checkpoint + if config is None: + return None + harness = self.harness + backend = backend_cls(self.config.runtime) + if harness.SUPPORTS_FILESYSTEM_CHECKPOINTS and ( + backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + or (config.boxless and not harness.EXECUTES_CODE) + ): + return config + key = (harness.config.id, self.config.runtime.type) + if key not in _checkpointing_warned: + _checkpointing_warned.add(key) + reason = ( + f"harness {harness.config.id!r} does not support it" + if not harness.SUPPORTS_FILESYSTEM_CHECKPOINTS + else f"the {self.config.runtime.type!r} runtime takes no snapshots" + ) + logger.warning( + "rollout checkpointing is off: %s; a failed rollout retries from the " + "start", + reason, + ) + return None + def _check_resume_support(self) -> None: # Multi-turn capability is derived: a harness needs transcript replay, a # native resume implementation, or a rollout-scoped session implementation. @@ -667,6 +702,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -675,49 +711,149 @@ async def run( owner, counted in the pairing check; `on_trace` observes the trace the moment it's minted, before any I/O. `collect_artifacts` captures the task's declared artifacts after its finalizer while its container runtime is still - alive. Retries whole while the trace ends with a retryable error - (`config.retries`) — never into a borrowed box; every attempt's trace keeps - the run's id with its own `attempt`, and the final trace keeps earlier - attempts' errors. `on_score(trace, runtime)` runs after task scoring in + alive. Retries while the trace ends with a retryable error (`config.retries`) + — never into a borrowed box. A retry restores the failed attempt's newest + restorable checkpoint when it has one (`config.checkpoint`, on a runtime and a + harness that support it), keeping that trace and its turns up to there; a + checkpoint whose restore failed is skipped, and without one the retry starts + over. Every attempt's trace keeps the run's id with its own `attempt`, and the + final trace keeps every attempt's errors. `on_score(trace, runtime)` runs after task scoring in the selected verifier workspace, inside its timeout and retry boundary.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") + if restore is None and self.pending_restore is not None: + restore, self.pending_restore = self.pending_restore, None retry = self.config.retries retry_state = RetryState(retry) - history: list = [] - trace_id = uuid.uuid4().hex - for attempt in range(retry_state.max_retries + 1): - trace = await self._run_once( - task, - runtime, - tools, - on_trace, - collect_artifacts, - on_score, - trace_id=trace_id, - attempt=attempt + 1, - ) - if attempt == retry_state.max_retries or trace.ok: - break - if runtime is not None: + checkpointing = self._checkpointing() if runtime is None else None + # One id for the run across its attempts; a restored trace keeps its own. + trace_id = restore[0].id if restore is not None else uuid.uuid4().hex + history: list[Error] = [] # errors of the attempts started over from + # `restore` continues a trace another process ran (an orchestrator resuming its + # run): its first attempt restores it, and `on_trace` sees it as the run's trace. + trace: Trace | None = restore[0] if restore is not None else None + target: Checkpoint | None = restore[1] if restore is not None else None + if restore is not None and on_trace is not None: + on_trace(restore[0]) + snapshots: set[str] = set() # every checkpoint any attempt took + deleted: set[str] = set() + failed_restores: set[str] = set() + run: Rollout | None = None + # The `finally` also covers a cancellation between attempts, when no rollout is + # open to delete them on abort. + budget = retry_state.max_retries + attempt = 0 + try: + while True: + restoring = ( + (trace, target) + if trace is not None and target is not None + else None + ) + # This attempt's own errors decide — a restored trace still carries + # the earlier attempts'. + seen = len(restoring[0].errors) if restoring is not None else 0 + run = await self._run_once( + task, + runtime, + tools, + on_trace, + collect_artifacts, + on_score, + trace_id=trace_id, + attempt=attempt + 1, + restore=restoring, + checkpoint=checkpointing, + ) + trace = run.trace + snapshots.update(run.rewound_checkpoints) + snapshots.update(c.id for c in trace.checkpoints if c.box) + deleted.update(run.deleted_checkpoints) + if target is not None and any( + error.stage == "setup" for error in trace.errors[seen:] + ): + # The box never booted from it: the next retry steps back. + failed_restores.add(target.id) + if trace.ok: + break + if ( + attempt == 0 + and restore is not None + and target is not None + and target.id in failed_restores + ): + # Another process's restore that cannot boot (its snapshot is gone) + # is not the rollout's failure: start over, unbudgeted. + logger.warning( + "restoring trace %s from checkpoint %s failed; starting over", + trace.id, + target.id, + ) + history.extend(trace.errors) + target = None + trace = None + attempt += 1 + budget += 1 + continue + if attempt >= budget: + break + if runtime is not None: + logger.warning( + "not retrying the rollout on a borrowed box (its state is no " + "longer the task's start state); the error stands" + ) + break + cause = retry_state.next_error(trace.errors[seen:]) + if cause is None: + break + target = next( + ( + c + for c in reversed(trace.checkpoints) + if c.time.end and not c.deleted and c.id not in failed_restores + ), + None, + ) + if target is None: + history.extend(trace.errors) + delay = backoff(attempt) logger.warning( - "not retrying the rollout on a borrowed box (its state is no " - "longer the task's start state); the error stands" + "retrying agent rollout (retry %d/%d) in %.1fs after error: %s (%s)", + attempt + 1, + budget, + delay, + cause.type, + f"restoring checkpoint {target.id}" + if target is not None + else "from scratch", ) - break - cause = retry_state.next_error(trace.errors) - if cause is None: - break - history.extend(trace.errors) - delay = backoff(attempt) - logger.warning( - "retrying agent rollout (retry %d/%d) in %.1fs after error: %s", - attempt + 1, - retry_state.max_retries, - delay, - cause.type, + await asyncio.sleep(delay) + attempt += 1 + finally: + # The run is final (or cancelled): delete what `after` does not keep. Every + # box is gone by now, so the platform refuses none of them. + keep = ( + kept(run.trace.checkpoints, checkpointing.keep) + if run is not None + and checkpointing is not None + and checkpointing.after == "keep" + else set() ) - await asyncio.sleep(delay) + drop = sorted(snapshots - deleted - keep) + if drop and run is not None and run.runtime is not None: + try: + run.record_deleted( + await run_shielded( + run.runtime.backend.discard_checkpoints(drop) + ) + ) + except Exception: + logger.warning( + "discarding checkpoints failed (rollout %s)", + run.trace.id, + exc_info=True, + ) + assert trace is not None and run is not None if history: # The full history rides the final trace either way; success is the # `ok` stamp, never errors-emptiness. @@ -735,7 +871,9 @@ async def _run_once( *, trace_id: str, attempt: int, - ) -> Trace: + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, + ) -> Rollout: params = self._rollout_params(task, runtime, dict(shared_tools or {})) if ( collect_artifacts @@ -751,12 +889,17 @@ async def _run_once( collect_artifacts=collect_artifacts, on_score=on_score, trace_id=trace_id, + restore=restore, + checkpoint=checkpoint, attempt=attempt, **params, ) try: if await run.open(): - await run.step() + if restore is not None: + await run.restore() + else: + await run.step() if run.ok: run.trace.stop("agent_completed") trace = await run.close() @@ -766,7 +909,7 @@ async def _run_once( raise if trace.agent.runtime is not None: trace.agent.runtime.borrowed = runtime is not None - return trace + return run @asynccontextmanager async def interaction( @@ -995,6 +1138,7 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: async with self._gate or nullcontext(): trace = await super().run( @@ -1004,6 +1148,7 @@ async def run( on_trace=self._watch(on_trace), collect_artifacts=collect_artifacts, on_score=self._scoring_callback(on_score), + restore=restore, ) self._completed.append(trace) return trace @@ -1071,6 +1216,11 @@ def __init__(self, config, make: MakeAgent[AgentT] | None = None) -> None: for name, value in agent_config_fields(config).items() } + def only(self) -> AgentT | None: + """The agent of a config with one agent seat; None when it has several.""" + agents = list(self._agents.values()) + return agents[0] if len(agents) == 1 else None + def __getattr__(self, name: str) -> AgentT: # self.__dict__ directly: attribute lookup re-entering __getattr__ before # __init__ ran (copy/unpickle) must raise, not recurse. diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 19d484563d..c9aa905980 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -1,10 +1,11 @@ """One env agent's config: who plays the seat, and its per-run caps.""" -from typing import Any +from typing import Any, Literal from pydantic import ( BaseModel, Field, + PositiveInt, SerializationInfo, SerializeAsAny, field_serializer, @@ -38,6 +39,39 @@ class TimeoutConfig(BaseConfig): isolated verifier setup and retries. Finalization has its own budget.""" +class RolloutCheckpointConfig(BaseConfig): + """Rollout checkpointing. Snapshot the rollout's box (its filesystem) as model + turns go upstream, so a retry restores the newest snapshot instead of starting + over: the trace keeps its turns up to there, and a new harness process continues + in a box restored to that state, told that the box was restored. Needs a runtime + with checkpoints (prime) and a harness that can restore one (both set + `SUPPORTS_FILESYSTEM_CHECKPOINTS`); elsewhere the agent warns once and a retry starts over. + The snapshot runs alongside the model call, and the reply waits only until its + content is fixed. + + Known limits: a harness with concurrent sub-agents can be snapshotted while one of + them is mid-tool, and processes the agent left running (background jobs, + sub-agents, a REPL's variables) are gone after a restore, which cold boots.""" + + interval: PositiveInt = 1 + """Snapshot before every this-many-th model turn (the 1st, the 1+n-th, ...). A + larger interval takes fewer snapshots, and a restore loses up to this many turns.""" + keep: Literal["all", "last"] | PositiveInt = "last" + """Which snapshots stay while the rollout runs: `"last"` keeps only the newest + restorable one, `"all"` keeps every one, and `k` keeps every k-th (the 1st, the + 1+k-th, ...) plus the newest. The others are deleted as soon as a newer one is + restorable. A retry whose restore fails falls back to an older snapshot that is + still kept, else it starts over.""" + after: Literal["delete", "keep"] = "delete" + """What happens to the kept snapshots when the rollout's run is final (it succeeded, + or its retries ran out): `"delete"` deletes them once the box is gone; `"keep"` + leaves them for their owner (forks, inspection), who must delete them.""" + boxless: bool = False + """Also checkpoint a harness that runs no code (null) on a runtime without + snapshots, from the trace alone. Off by default: such a run records a checkpoint + on its trace at every snapshot turn.""" + + class AgentConfig(BaseConfig): harness: SerializeAsAny[HarnessConfig] | None = None """The agent's program (None = the taskset's default harness).""" @@ -75,6 +109,12 @@ class AgentConfig(BaseConfig): timeout: TimeoutConfig = TimeoutConfig() retries: RetryConfig = RetryConfig() + checkpoint: RolloutCheckpointConfig | None = Field( + default_factory=RolloutCheckpointConfig + ) + """Rollout checkpointing, on by default: snapshot the box as turns go upstream, so + a retry restores the newest snapshot instead of starting over (see + `RolloutCheckpointConfig`). None turns it off.""" network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 1f943574fa..2223174e5c 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -32,7 +32,7 @@ from verifiers.v1.mcp import SharedToolServer, serve_shared from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task -from verifiers.v1.trace import Error, Trace, TraceTask +from verifiers.v1.trace import Checkpoint, Error, Trace, TraceTask from verifiers.v1.utils.generic import concrete_type from verifiers.v1.utils.memory import trim_memory_periodically from verifiers.v1.utils.retries import run_episode_with_retry @@ -244,6 +244,7 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time @@ -266,6 +267,17 @@ async def run_episode( ), ) agents = self._episode_agents(ctx, episode.traces, on_trace, on_discard) + if restore is not None: + # A one-seat episode's run is its one trace: its agent's run continues it. + agent = agents.only() + if agent is None: + logger.warning( + "%s has several agent seats; running trace %s from scratch", + type(self).__name__, + restore[0].id, + ) + else: + agent.pending_restore = restore try: async with asyncio.timeout(self.config.timeout.episode): async with boundary(EnvError, f"{type(self).__name__}.setup()"): @@ -320,13 +332,17 @@ async def run_slot( semaphore: asyncio.Semaphore | None = None, on_complete: Callable[[Episode], Awaitable[None]] | None = None, on_trace: Callable[[Trace], None] | None = None, + restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """Run one planned episode to completion, with whole-episode retries per `--env.retries`; `semaphore` bounds concurrent EPISODES — one permit for the attempt in flight, held across the whole of it (its agents, their boxes, `finalize()`) and released before a retry's backoff and before `on_complete` (the runners' persistence hook, which fires when final). - `on_trace` sees each trace at mint, after it joined `slot.traces`.""" + `on_trace` sees each trace at mint, after it joined `slot.traces`. `restore` + continues a trace another process ran (one-seat episodes); only the first + attempt restores it, a whole-episode retry starts over.""" + pending = [restore] if restore is not None else [] attempts = 0 @@ -354,6 +370,7 @@ def discard(trace: Trace) -> None: on_discard=discard, episode_id=slot.id, attempt=attempts, + restore=pending.pop() if pending else None, ) episode = await run_episode_with_retry(attempt, self.config.retries) diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index a4329bbe61..963d63c931 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -5,7 +5,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import DialectName, HarnessConfig, skill_destination @@ -18,7 +18,7 @@ ) from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData -from verifiers.v1.types import Message, Messages +from verifiers.v1.types import Message, Messages, UserMessage from verifiers.v1.utils.decorators import ( discover_decorated, invoke_all, @@ -36,6 +36,16 @@ ConfigT = TypeVar("ConfigT", bound=HarnessConfig) +RESTORE_PROMPT = ( + "Your workspace was restored from a filesystem checkpoint taken before your last " + "model turn, and you are continuing in a new process. Files are as they were at " + "the checkpoint. Background processes, running shells, sub-agents, and in-memory " + "state such as REPL variables from before the restore are gone; start again " + "whatever you still need. Your last action may not have taken effect, so check " + "before you rely on it. Continue the task." +) +"""The user turn a harness gets after a restore (`HarnessSession.restore`).""" + class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = False @@ -49,6 +59,13 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" + SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False + """Whether a new process of this harness can continue a rollout in a box restored + from a filesystem checkpoint (`HarnessSession.restore`), so a retry restores the + newest checkpoint instead of starting over (`AgentConfig.checkpoint`). A rollout + checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_CHECKPOINTS` too. Distinct + from `SUPPORTS_RESUME`: a restore never finds a live process, its program + continues from the conversation the trace or its own session files hold.""" EXECUTES_CODE: ClassVar[bool] = True """Whether the program hands the model local execution in the runtime — true for every real harness; the tool-less chat loops (`null`) override to False. Read @@ -404,6 +421,21 @@ async def _run(self, messages: Messages | None) -> ProgramResult: self.tool_interception_url, ) + async def checkpoint_state(self) -> dict[str, Any] | None: + """What `restore` needs to continue in a box restored from a checkpoint taken + now, or None when this session cannot be restored (no checkpoint is taken). + The default relaunches on the trace's transcript and needs nothing.""" + return {} + + def restore(self, state: dict[str, Any], tail: Messages) -> Messages: + """Prepare this new session to continue a rollout in a box restored from a + checkpoint, and return the messages of its first segment. `state` is what + `checkpoint_state` returned when the checkpoint was taken; `tail` is the + checkpoint's uncommitted prompt after the trace's branch. The default + relaunches on the branch plus `tail`, then tells the program about the + restore (`RESTORE_PROMPT`).""" + return [*tail, UserMessage(content=RESTORE_PROMPT)] + async def close(self) -> None: """Close session-owned resources. Idempotent.""" self._closed = True diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 8f6d852144..e2790dfbd4 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,6 +52,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 3287be9994..e2b4495e27 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -19,7 +19,7 @@ ACP_VERSION = "0.99.1" CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" -CLAUDE_CONFIG_ROOT = ".vf-claude" +STATE_ROOT = "/tmp/vf-claude-code-runs" ACP_INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -42,6 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed @@ -104,6 +105,10 @@ async def prepare_acp( "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"), "ANTHROPIC_API_KEY": secret, "ANTHROPIC_MODEL": ctx.model, + # Claude Code validates a model it has not seen in this process with a + # one-token "Hi" request (e.g. when a restored session re-selects its + # model); a custom model option is trusted without that request. + "ANTHROPIC_CUSTOM_MODEL_OPTION": ctx.model, "CLAUDE_CODE_EXECUTABLE": CLAUDE_BIN.format(**versions), "CLAUDE_CONFIG_DIR": config_dir, "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", @@ -135,4 +140,4 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def config_dir(trace: Trace) -> str: - return f"{CLAUDE_CONFIG_ROOT}/{trace.id}" + return f"{STATE_ROOT}/{trace.id}" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 59b690ac33..e45406ab40 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -22,6 +22,7 @@ CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}" PACKAGES_DIR = f"{CODEX_DIR}/acp" +STATE_ROOT = "/tmp/vf-codex-runs" ACP_VERSION = "2.5.0" CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp" @@ -62,6 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: @@ -174,7 +176,7 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def trace_home(trace: Trace) -> str: - return f"/tmp/vf-codex-home-{trace.id}" + return f"{STATE_ROOT}/{trace.id}" async def build_env( self, diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 503429ddb6..431c95f698 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,6 +26,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 27d4136748..1c22e675e7 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -31,8 +31,8 @@ BuiltinTool = Literal["bash", "edit", "fetch", "ipython"] RLM_REPO = "github.com/PrimeIntellect-ai/nano-rlm.git" -RLM_CACHE_DIR = "/tmp/vf-rlm" -RLM_STATE_DIR = "/tmp/vf-rlm-state" +RLM_CACHE_DIR = "/var/tmp/vf-rlm" +STATE_ROOT = "/tmp/vf-rlm-runs" RLM_RUNTIME_METADATA_KEY = "ai.prime.rlm/runtime-v1" RLM_SESSION_METADATA_KEY = "ai.prime.rlm/session-v1" @@ -71,7 +71,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="ae1cc01f8293987d18b09286b34d93c0dfbf8dac", min_length=1 + default="a057f58d144384fed6ecfed1dabcf350fe540910", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" @@ -142,12 +142,18 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True + SUPPORTS_FILESYSTEM_CHECKPOINTS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: if self.config.skills: # Editable skill packages and uv's tool environment must belong to this run. - self._skills_install_dir = f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" + # A retry keeps the directory: it holds `RLM_HOME`, whose session a box + # restored from a checkpoint loads. + if self._skills_install_dir is None: + self._skills_install_dir = ( + f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" + ) await self.install_skills(runtime, f"{self._skills_home()}/skills") directory = self._install_dir() binary = f"{directory}/bin/rlm" @@ -306,14 +312,14 @@ def acp_close_result(self, trace: Trace, response_metadata: dict[str, Any]) -> N self._consume_snapshot(trace, response_metadata) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - await runtime.run(["rm", "-rf", f"{RLM_STATE_DIR}/{trace.id}"], {}) + await runtime.run(["rm", "-rf", f"{STATE_ROOT}/{trace.id}"], {}) if self._skills_install_dir is not None: await remove_dir( runtime, self._skills_install_dir, "RLM skill installation" ) def _home(self, trace: Trace) -> str: - return self._skills_home() or f"{RLM_STATE_DIR}/{trace.id}/home" + return self._skills_home() or f"{STATE_ROOT}/{trace.id}/home" def _skills_home(self) -> str | None: """rlm's home for a run that carries skills: they sit in its `skills/`, where diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 8f8a0b3351..a82a177bc4 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -33,6 +33,7 @@ from contextlib import asynccontextmanager from dataclasses import dataclass, field from functools import partial +from typing import NoReturn import httpx from aiohttp import web @@ -606,10 +607,13 @@ async def start(self) -> None: ) def _fail( - self, session: RolloutSession, dialect: Dialect, error: RolloutError + self, + session: RolloutSession, + dialect: Dialect, + error: RolloutError, ) -> web.Response: - """Stash a model-turn-adjacent failure (such as a hook raising) so the rollout - re-raises it as the real cause, and report it to the harness as an HTTP error.""" + """Stash a model-turn-adjacent failure so the rollout re-raises it as the real + cause, and report it to the harness as an HTTP error.""" session.error = error logger.warning( "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error @@ -618,6 +622,17 @@ def _fail( dialect, str(error), status=getattr(error, "status_code", 502) ) + async def _abort(self, session: RolloutSession, error: RolloutError) -> NoReturn: + """End the rollout's segment with a request hook's failure. The harness gets no + reply: the rollout cancels its turn, and releasing the session cancels this + handler.""" + logger.warning( + "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error + ) + session.abort(error) + await asyncio.get_running_loop().create_future() + raise AssertionError("unreachable") + def mediate_capabilities( self, session: RolloutSession, dialect: Dialect, body: dict ) -> tuple[dict, list[str]]: @@ -870,11 +885,10 @@ async def coalesced( if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) except RolloutError as error: - return self._fail(session, dialect, error) + return await self._abort(session, error) except Exception as error: # noqa: BLE001 - surface task hook failures - return self._fail( + return await self._abort( session, - dialect, TaskError( f"model boundary hook failed: {type(error).__name__}: {error}" ), @@ -932,6 +946,14 @@ async def sample() -> web.Response: node: int | None = None error: Exception | None = None started = time.time() + # The box is idle while the harness awaits this turn, so a snapshot captured + # before the reply returns matches the prompt exactly (tool results in, this + # reply not yet). It runs alongside the model call; the reply waits for it. + capture = ( + asyncio.create_task(session.checkpointer(turn)) + if session.checkpointer is not None + else None + ) try: try: # What actually goes upstream: the native body with the rollout's model + @@ -1024,6 +1046,18 @@ async def sample() -> web.Response: error = e raise finally: + if capture is not None: + held = time.monotonic() + try: + checkpoint = await capture + # A lost snapshot never fails the turn. + except Exception: + logger.warning( + "checkpoint failed: id=%s", session.trace.id, exc_info=True + ) + else: + if checkpoint is not None: + checkpoint.held = time.monotonic() - held if node is None: turn.abandon() # The turn's one per-exchange record: settings, timing, outcome, and diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index a5563b982a..96ad61ea28 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -2,15 +2,19 @@ import asyncio import contextlib +import copy import logging import time +import uuid from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass +from typing import Literal from urllib.parse import urlsplit +from verifiers.v1 import graph from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.agent import AgentConfig +from verifiers.v1.configs.agent import AgentConfig, RolloutCheckpointConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.configs.scoring import resolve_scoring from verifiers.v1.errors import ( @@ -39,8 +43,8 @@ from verifiers.v1.runtimes.private_network import tool_vms from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary from verifiers.v1.state import state_cls -from verifiers.v1.task import Task -from verifiers.v1.trace import AgentInfo, Trace, TraceTask +from verifiers.v1.task import Task, TaskData +from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke @@ -97,6 +101,25 @@ async def resolve_agent_runtime( return view +def kept(checkpoints: list[Checkpoint], keep: Literal["all", "last"] | int) -> set[str]: + """The ids of the restorable `checkpoints` (durable, not deleted) that the `keep` + policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every one for + `"all"` or every k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" + live = [ + (index, c) + for index, c in enumerate(checkpoints) + if c.time.end and not c.deleted + ] + if not live: + return set() + if keep == "all": + return {c.id for _, c in live} + held = {live[-1][1].id} + if keep != "last": + held.update(c.id for index, c in live if index % keep == 0) + return held + + class Rollout: """Manages one rollout's lifecycle (open, step, close).""" @@ -119,7 +142,18 @@ def __init__( on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, trace_id: str | None = None, attempt: int = 1, + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, ) -> None: + """`attempt` numbers this run among the trace's retries, from 1 (stamped on + the trace and the errors it captures). `restore` continues a failed attempt's trace from one of + its checkpoints: the trace rewinds to it and this run boots its box from the + snapshot instead of setting the task up again. `checkpoint` snapshots the box as + model turns go upstream (on a runtime and a harness that support it) for a + retry to restore, and deletes the ones its `keep` policy drops as newer ones + become restorable; the owner of the retries deletes the rest once the run is + final (`RolloutCheckpointConfig.after`) — this rollout deletes them only when it + aborts.""" self._on_score = on_score self.task = task self.harness = harness @@ -139,23 +173,44 @@ def __init__( self.runtime = runtime self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts - self.trace: Trace = Trace( - **({"id": trace_id} if trace_id is not None else {}), - attempt=attempt, - task=TraceTask( - type=type(task).__name__, - data=task.data, - key=task.key, - hash=task.hash, - ), - state=state_cls(type(task))(), - # The seat's resolved config, role overrides included — the agent - # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), - ) + self._checkpointing = checkpoint + self._durable: dict[str, asyncio.Task[None]] = {} + self._captures: set[asyncio.Task] = set() + """Snapshots being taken (`_checkpoint`), for `_settle_checkpoints` to wait for.""" + """The snapshots still on their way to restorable, by checkpoint id.""" + self._attempt = attempt + self._restore: Checkpoint | None = None + self._harness_data: TaskData | None = None + self.deleted_checkpoints: set[str] = set() + """The snapshots this run deleted.""" + self._pruning = asyncio.Lock() + self._boxless = False + """Whether this run's checkpoints are box-less (`Checkpoint.box`).""" + self.rewound_checkpoints: list[str] = [] + """Checkpoints the restored trace dropped (taken after the one restored).""" + if restore is not None: + self.trace, self._restore = restore + self.trace.attempt = attempt + self.rewound_checkpoints = self.trace.rewind(self._restore) + self._restore.restored = True + else: + self.trace = Trace( + **({"id": trace_id} if trace_id is not None else {}), + attempt=attempt, + task=TraceTask( + type=type(task).__name__, + data=task.data, + key=task.key, + hash=task.hash, + ), + state=state_cls(type(task))(), + # The seat's resolved config, role overrides included — the agent + # this trace can be reproduced with. + agent=AgentInfo(config=agent_config), + ) + if on_trace is not None: + on_trace(self.trace) self.trace._replay_view = harness.replay_view - if on_trace is not None: - on_trace(self.trace) interceptors = [ (hook_boundary(fn, allow_trace=False), fn) for fn in discover_decorated(task, "intercept") @@ -217,10 +272,11 @@ def failure(self) -> Exception | None: """The original exception most recently captured onto the trace.""" return self._failure - def fail(self, error: Exception) -> None: + def fail(self, error: Exception, stage: str | None = None) -> None: """Record `error` as this rollout's outcome (captured onto the trace, the remaining stages skipped) — the run's owner reporting a failure the run - itself couldn't see, e.g. its user raising between segments.""" + itself couldn't see, e.g. its user raising between segments. `stage` names + the rollout stage it broke in, when known.""" if ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -237,7 +293,7 @@ def fail(self, error: Exception) -> None: logger.exception("unexpected error in rollout %s", self.trace.id) self._failed = True self._failure = error - self.trace.record_error(error) + self.trace.record_error(error, attempt=self._attempt, stage=stage) def timeout(self, stage: str) -> None: """Record `stage`'s expired deadline as this rollout's outcome: a stop, not @@ -254,7 +310,15 @@ async def open(self) -> bool: self.trace.timing.boot.start = time.time() self.trace.notify() if self._borrowed_runtime is None: - self.runtime = make_backend(self.runtime_config, name=self.trace.id).bind() + config = self.runtime_config + name = self.trace.id + if self._restore is not None: + # A restored trace keeps its id; its new box must not collide with the + # box an earlier attempt or process left behind. + name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" + if self._restore.box: + config = config.model_copy(update={"checkpoint": self._restore.id}) + self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -269,13 +333,24 @@ async def open(self) -> bool: runtime = self.runtime assert self.trace.agent is not None # minted with the trace self.trace.agent.runtime = runtime.info.model_copy() - logger.info( - "rollout start: id=%s task=%s harness=%s runtime=%s", - self.trace.id, - self.task.data.idx, - self.harness.config.name, - self.runtime_config.type, - ) + if self._restore is None: + logger.info( + "rollout start: id=%s task=%s harness=%s runtime=%s", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + ) + else: + logger.info( + "rollout restore: id=%s task=%s harness=%s runtime=%s checkpoint=%s turns=%d", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + self._restore.id, + self.trace.num_turns, + ) loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None setup_deadline = ( @@ -320,6 +395,15 @@ async def open(self) -> bool: finally: # Providers may assign an ID before startup fails; retain it for diagnostics. self.trace.agent.runtime = runtime.info.model_copy() + # A box snapshot needs a runtime that takes them; a harness that keeps + # nothing in its box is restorable from the trace alone (box-less). + self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + if ( + self._checkpointing is not None + and self._borrowed_runtime is None + and (not self._boxless or not self.harness.EXECUTES_CODE) + ): + self._session.checkpointer = self._checkpoint if not self.task.scoring_deferred: isolated = self._scoring_policy.mode == "isolated" self._scoring_task = self.task.grading_task( @@ -343,17 +427,21 @@ async def open(self) -> bool: self.trace.timing.boot.end = now self.trace.timing.setup.start = now self.trace.notify() - async with ( - boundary(TaskError, "task setup"), - ): - await invoke( - self.task.setup, - { - "trace": self.trace, - "runtime": runtime, - "deployment": self.deployment, - }, - ) + # A restored box already holds the task's setup; only the harness's + # (idempotent) install runs again. A box-less restore boots a fresh box. + if self._restore is None or not self._restore.box: + async with boundary(TaskError, "task setup"): + await invoke( + self.task.setup, + { + "trace": self.trace, + "runtime": runtime, + "deployment": self.deployment, + }, + ) + if self._restore is not None: + # The trace's state and info are the checkpoint's, not setup's. + self.trace.rewind_state(self._restore) async with ( boundary(HarnessError, "harness setup"), ): @@ -467,9 +555,11 @@ async def open(self) -> bool: harness_data = harness_data.model_copy( update={"system_prompt": network_prompt} ) + # A restored trace already carries its prompt's rewrites. if ( self._session.request_interceptors and harness_data.prompt is not None + and self._restore is None ): prompt = harness_data.prompt system_prompt = harness_data.system_prompt @@ -520,6 +610,7 @@ async def open(self) -> bool: ) else {} ) + self._harness_data = harness_data self._harness_session = await self.harness.session( self.ctx, self.trace, @@ -537,13 +628,14 @@ async def open(self) -> bool: self.timeout("setup") if not self._deployment_started: self.fail( - SandboxError("deployment provisioning exceeded setup timeout") + SandboxError("deployment provisioning exceeded setup timeout"), + stage="setup", ) else: - self.fail(e) + self.fail(e, stage="setup") return False except Exception as e: # noqa: BLE001 - setup boundary records every rollout failure - self.fail(e) + self.fail(e, stage="setup") return False except BaseException: # A cancellation mid-setup kills the driver's await with it, so no @@ -568,15 +660,18 @@ async def step( self, messages: Messages | None = None, *, + intercepted: int = 0, on_input: Callable[[], None] | None = None, ) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - Returns whether the exchange can continue — a refused turn (limit, @stop), - a failure (an expired agent timeout included), or a segment that made no - progress all end it.""" + The first `intercepted` messages pass to the harness as they are — they were + intercepted when the trace first saw them (a restored checkpoint's tail). + Returns whether the exchange can continue — a refused turn (limit, + @stop), a failure (an expired agent timeout included), or a segment that + made no progress all end it.""" if not self._opened or self._closed or not self.ok: return False trace = self.trace @@ -595,25 +690,32 @@ async def step( try: async with deadline: assert self._harness_session is not None - if messages is not None and self._session.request_interceptors: + if ( + messages is not None + and len(messages) > intercepted + and self._session.request_interceptors + ): prepared, rewrites = await self._session.prepare_users( - Request(messages=messages) + Request(messages=messages[intercepted:]) ) - messages = prepared.messages + messages = [*messages[:intercepted], *prepared.messages] self.trace.request_rewrites.extend(rewrites) if self._session.stopped: return False self._session.on_turn_input = on_input assert self.deployment is not None - await self._harness_session.turn(messages) + if await self._turn(messages): + assert self._session.aborted is not None + self.fail(self._session.aborted, stage="agent") + return False except TimeoutError as e: # An expired rollout deadline is the agent breaking its time budget: a - # timeout stop. A TimeoutError from the harness's own I/O with no - # expired deadline stays the raw failure. + # timeout stop. A TimeoutError from the harness's own I/O with no expired + # deadline stays the raw failure. if self.deadline_at is not None and (loop.time() >= self.deadline_at): self.timeout("agent") else: - self.fail(e) + self.fail(e, stage="agent") return False except Exception as e: # noqa: BLE001 - harness boundary records every rollout failure if self._session.stopped: @@ -627,9 +729,9 @@ async def step( real = self._session.error if real is not None and isinstance(e, RolloutError): real.__cause__ = e - self.fail(real) + self.fail(real, stage="agent") else: - self.fail(e) + self.fail(e, stage="agent") return False finally: self._session.on_turn_input = None @@ -649,6 +751,188 @@ async def step( # never moved, forever. return self.ok and trace.num_turns > turns_before + async def _turn(self, messages: Messages | None) -> bool: + """Run one harness segment. A request hook that fails ends it at once (the + session aborts): the harness's turn is cancelled, so the harness never handles + the failure. Returns whether the segment was aborted.""" + assert self._harness_session is not None + turn = asyncio.ensure_future(self._harness_session.turn(messages)) + aborted = asyncio.ensure_future(self._session.abort_event.wait()) + try: + await asyncio.wait({turn, aborted}, return_when=asyncio.FIRST_COMPLETED) + finally: + aborted.cancel() + if not turn.done(): + turn.cancel() + await asyncio.wait({turn}) + if self._session.aborted is not None: + if not turn.cancelled(): + turn.exception() # the abort decides the outcome, whatever the turn did + return True + turn.result() + return False + + async def restore(self) -> bool: + """Run the restored attempt's first segment: a new harness process in the box + restored from the checkpoint continues the conversation the checkpoint left + off at (`HarnessSession.restore`), told that the box was restored.""" + checkpoint = self._restore + assert checkpoint is not None and self._harness_data is not None + assert self._harness_session is not None + # The relaunch re-emits the session's system prompt (the task's, plus a + # network notice) through the program's own system message; the tail's copy + # would double it. + system = self._harness_data.system_prompt + tail = [m for m in checkpoint.tail if m.role != "system" or system is None] + self._session.pinned_tool_results.update(checkpoint.pins) + self._session.mark_intercepted(self.trace.messages, tail) + messages = self._harness_session.restore(checkpoint.harness, tail) + # Only the restore notice is new; user hooks prepare it like any user turn. + return await self.step(messages, intercepted=len(messages) - 1) + + async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: + if self._closed: + # Settling may have drained `_captures` already: a snapshot taken now + # would start its upload after the box is gone. + return None + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + try: + return await self._capture(turn) + finally: + self._captures.discard(task) + + async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: + assert self.runtime is not None + assert self._checkpointing is not None + if self.trace.num_turns % self._checkpointing.interval: + return None + if len(self.trace._pending) > 1: + # Another request is in flight (parallel sub-agents share the trace): the + # box is not idle, so no snapshot would match this turn's prefix. + return + assert self._harness_session is not None + try: + harness = await self._harness_session.checkpoint_state() + except Exception: + logger.warning( + "harness checkpoint state failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + return + if harness is None: + return + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(self._session.pinned_tool_results) + started = time.time() + if self._boxless: + checkpoint_id = uuid.uuid4().hex + else: + try: + checkpoint_id = await self.runtime.backend.checkpoint() + except Exception: + logger.warning( + "checkpoint failed (rollout %s)", self.trace.id, exc_info=True + ) + return None + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + checkpoint = Checkpoint( + id=checkpoint_id, + node=node, + tail=list(turn.tail), + state=state, + info=info, + pins=pins, + harness=harness, + box=not self._boxless, + time=TimeSpan(start=started), + captured=time.time(), + ) + self.trace.checkpoints.append(checkpoint) + if self._boxless: + # Nothing to upload: restorable as soon as it is recorded. + checkpoint.time.end = checkpoint.captured + self.trace.notify() + else: + self._durable[checkpoint_id] = asyncio.create_task( + self._make_durable(checkpoint) + ) + logger.debug( + "checkpoint %s taken: id=%s node=%s", checkpoint_id, self.trace.id, node + ) + return checkpoint + + async def _make_durable(self, checkpoint: Checkpoint) -> None: + assert self.runtime is not None + try: + checkpoint.bytes = await self.runtime.backend.wait_checkpoint(checkpoint.id) + except Exception: + # Not restorable: no retry may pick it. The backend still holds it as + # stray and frees it with the box. + logger.warning( + "checkpoint %s never became durable (rollout %s)", + checkpoint.id, + self.trace.id, + exc_info=True, + ) + with contextlib.suppress(ValueError): + self.trace.checkpoints.remove(checkpoint) + else: + checkpoint.time.end = time.time() + self.trace.notify() + await self._prune() + finally: + self._durable.pop(checkpoint.id, None) + + async def _prune(self) -> None: + """Delete the snapshots the `keep` policy drops now that a newer one is + restorable, and the ones a restore rewound past.""" + assert self.runtime is not None and self._checkpointing is not None + async with self._pruning: + keep = kept(self.trace.checkpoints, self._checkpointing.keep) + drop = [ + *( + c.id + for c in self.trace.checkpoints + if c.box and c.time.end and not c.deleted and c.id not in keep + ), + *( + c + for c in self.rewound_checkpoints + if c not in self.deleted_checkpoints + ), + ] + if drop: + self.record_deleted( + await self.runtime.backend.discard_checkpoints(drop) + ) + + def record_deleted(self, ids: list[str]) -> None: + """Mark the snapshots with these ids deleted.""" + self.deleted_checkpoints.update(ids) + for checkpoint in self.trace.checkpoints: + if checkpoint.id in self.deleted_checkpoints: + checkpoint.deleted = True + + async def _settle_checkpoints(self, *, wait: bool) -> None: + """Before the box goes: wait for the snapshots still uploading, so a retry + can restore them (its teardown would fail them), or, with nothing to retry, + stop waiting and leave them to be freed.""" + # A snapshot still being taken would start its upload after this returned. + await asyncio.gather(*self._captures, return_exceptions=True) + tasks = list(self._durable.values()) + if not wait: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + # A task cancelled before it first ran never reached its own cleanup. + for checkpoint_id, task in list(self._durable.items()): + if task.done(): + del self._durable[checkpoint_id] + async def _cleanup_harness(self, runtime: Runtime) -> None: if self._harness_cleaned: return @@ -666,6 +950,7 @@ async def abort(self) -> None: (a cancellation mid-setup, a lifetime bug raised to the caller) means the driver will never reach `close()`. Safe after a partial `close()`.""" self._closed = True + await self._settle_checkpoints(wait=False) if self._harness_session is not None: with contextlib.suppress(Exception): await self._harness_session.close() @@ -684,6 +969,24 @@ async def abort(self) -> None: ): with contextlib.suppress(Exception): await self.runtime.backend.stop() + if self.runtime is not None: + # Snapshots kept `after` the run belong to their owner, an aborted run's too; + # strays were never restorable, so nobody else knows of them. + owned = self._checkpointing is None or self._checkpointing.after == "delete" + with contextlib.suppress(Exception): + self.record_deleted( + await self.runtime.backend.discard_checkpoints( + [ + *(self.rewound_checkpoints if owned else ()), + *( + c.id + for c in self.trace.checkpoints + if owned and c.box and not c.deleted + ), + *self.runtime.backend.stray_checkpoints, + ] + ) + ) async def _score_harness(self, runtime: Runtime, deadline: float | None) -> None: """Collect best-effort metrics without replacing a task failure.""" @@ -942,7 +1245,7 @@ async def close(self) -> Trace: trace.timing.scoring.end = time.time() trace.notify() except Exception as e: # noqa: BLE001 - finalize boundary records every rollout failure - self.fail(e) + self.fail(e, stage="scoring" if trace.timing.scoring.start else "finalize") finally: if self._harness_session is not None: with contextlib.suppress(Exception): @@ -964,6 +1267,9 @@ async def close(self) -> Trace: trace.split_agent_time() if runtime is not None: await self._cleanup_harness(runtime) + # A failed attempt's retry restores the newest snapshot, which must be + # durable before the box goes. + await self._settle_checkpoints(wait=not trace.ok) # Tear down here — the env's `score()` (later) needs only the traces, # not a live runtime. A borrowed runtime is its creator's to tear down, # not this rollout's. @@ -977,6 +1283,10 @@ async def close(self) -> Trace: ): # Setup can fail before a deployment takes ownership. await runtime.backend.stop() + if runtime is not None and runtime.backend.stray_checkpoints: + await runtime.backend.discard_checkpoints( + runtime.backend.stray_checkpoints + ) except Exception: logger.warning("runtime teardown failed", exc_info=True) logger.info( @@ -985,7 +1295,9 @@ async def close(self) -> Trace: self.task.data.idx, trace.reward, trace.num_turns, - trace.last_error.type if trace.last_error else trace.stop_condition, + trace.last_error.type + if trace.last_error and not trace.ok + else trace.stop_condition, f" dropped={trace.dropped!r}" if trace.is_dropped else "", ) return trace diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index 19a004c1cb..a2e8df945c 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -9,6 +9,7 @@ import math import shlex import tempfile +import time import uuid from collections.abc import AsyncIterator, Mapping from contextlib import asynccontextmanager @@ -51,6 +52,9 @@ _OUTPUT_RETRIES = 10 """Re-reads of a finished job's output that the SDK still failed to fetch, before the exec is reported as failed.""" +_CHECKPOINT_DURABLE_SECONDS = 300 +"""How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it +after the request returns, and only a durable one restores.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" """Domain of the public URLs `https://-.` that a LAN network's `internet` rule publishes ports at.""" @@ -150,6 +154,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True + SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -242,8 +247,8 @@ async def create_and_capture_id(): CreateSandboxRequest( name=self.name, labels=list(dict.fromkeys(labels)), - **source, environment_vars=self.env, + **source, **{k: v for k, v in options.items() if v is not None}, ) ) @@ -266,7 +271,12 @@ async def create_and_capture_id(): ) await self._client.wait_for_creation(self.info.id, max_attempts=900) logger.info( - "prime: sandbox %s up (image=%s)", self.info.id, self.config.image + "prime: sandbox %s up (image=%s%s)", + self.info.id, + self.config.image, + f", checkpoint={self.config.checkpoint}" + if self.config.checkpoint + else "", ) await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" @@ -283,6 +293,56 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e + async def checkpoint(self) -> str: + # The platform fixes the snapshot's content before the request returns, then + # uploads it. Until it is durable the snapshot is stray: the platform refuses + # to delete it while this box lives, so teardown frees it after. + async def create() -> str: + checkpoint = await self._client.checkpoint(self.info.id) + self.stray_checkpoints.append(checkpoint.id) + return checkpoint.id + + try: + return await run_shielded(create()) + except Exception as e: + raise SandboxError(f"prime checkpoint failed: {e}") from e + + async def wait_checkpoint(self, checkpoint: str) -> int | None: + started = time.monotonic() + try: + durable = await self._client.wait_for_checkpoint( + checkpoint, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + ) + except Exception as e: + raise SandboxError(f"prime checkpoint {checkpoint}: {e}") from e + self.stray_checkpoints.remove(checkpoint) + logger.debug( + "prime: checkpoint %s durable after %.1fs on sandbox %s", + checkpoint, + time.monotonic() - started, + self.info.id, + ) + return durable.stored_bytes + + async def discard_checkpoints(self, ids: list[str]) -> list[str]: + # The platform refuses only the newest checkpoint of a running box (the parent + # of its live state); older ones go while it runs, their data kept as long as a + # newer one needs it. + from prime_sandboxes import AsyncSandboxClient + + deleted = [] + async with AsyncSandboxClient() as client: + for checkpoint in ids: + try: + await client.delete_checkpoint(checkpoint) + except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + logger.warning( + "prime: failed to delete checkpoint %s: %s", checkpoint, e + ) + else: + deleted.append(checkpoint) + return deleted + async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None ) -> None: diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index b0c1d32756..7a917b6f81 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -184,6 +184,12 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" + SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False + """Whether `checkpoint` snapshots the box's filesystem for a new box of this config + to boot from (its config's `checkpoint`). A rollout checkpoints before each model + turn, so a retry restores the newest turn instead of replaying the run (see + `AgentConfig.checkpoint`).""" + info: BaseRuntimeInfo def __init__(self, name: str | None = None) -> None: @@ -192,6 +198,9 @@ def __init__(self, name: str | None = None) -> None: # Explicit process values (model credentials, proxy settings, etc.) override these. self.env: dict[str, str] = {} self.cache = RuntimeCache() + self.stray_checkpoints: list[str] = [] + """Checkpoints this box took that never became restorable (a failed or cancelled + durability wait), so no trace records them; freed after the box is gone.""" self._setup_claimed = False self._background: list[RuntimeProcess] = [] self.stopped = False @@ -218,6 +227,24 @@ def type(self) -> str: async def start(self) -> None: pass + async def checkpoint(self) -> str: + """Snapshot the filesystem and return the checkpoint's id once its content is + fixed: writes after this returns are not in it. It becomes restorable only later + (`wait_checkpoint`); the box keeps running.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def wait_checkpoint(self, checkpoint: str) -> int | None: + """Wait until `checkpoint` is restorable and return its stored size in bytes, + when the provider reports it. Raises if it never becomes restorable. The box + must stay alive until then: its teardown fails a pending checkpoint.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def discard_checkpoints(self, ids: list[str]) -> list[str]: + """Free checkpoints nothing will restore from and return the ids it freed. + Best-effort: a provider may refuse one a live box still needs (prime: the + newest one of a running box).""" + return [] + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index a0ec7f120f..2b9bb4043d 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -34,7 +34,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import HarnessError, RolloutError, TaskError from verifiers.v1.harnesses.utils.compaction import bound_tool_message -from verifiers.v1.trace import InterceptRecord, Trace +from verifiers.v1.trace import Checkpoint, InterceptRecord, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -184,6 +184,10 @@ class RolloutSession: when its program dies on it the rollout records this original error instead of a secondary `HarnessError`. A harness that completes cleanly after the failure handled it. Reset before each model turn, so a successful retry clears it.""" + aborted: "RolloutError | None" = None + """A request hook's failure: the task, not the provider, refused a turn. The rollout + ends the running segment with it (`Rollout.step`); the harness never sees it.""" + abort_event: asyncio.Event = field(default_factory=asyncio.Event) idempotent_requests: dict[str, IdempotentRequest] = field(default_factory=dict) """Explicit keys or marked SDK retries mapped to their replay state.""" released: bool = False @@ -209,6 +213,12 @@ class RolloutSession: gated_tools: set[tuple[str, str]] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" prepared_users: Counter[str] = field(default_factory=Counter) + checkpointer: Callable[[graph.PendingTurn], Awaitable[Checkpoint | None]] | None = ( + None + ) + """Snapshots the rollout's box as a turn goes upstream, alongside the model call; + the reply waits for the capture (`Rollout` sets it when its runtime supports + checkpoints and a retry may restore one). Returns the checkpoint it took, if any.""" @property def stopped(self) -> bool: @@ -385,6 +395,21 @@ async def rewrite_request( self.pinned_tool_results[tool_keys[position]] = after return current, records, None + def mark_intercepted(self, branch: Messages, tail: Messages) -> None: + """Treat `tail`, which follows `branch` and was intercepted in an earlier + attempt (a restored checkpoint's), as prepared: the harness re-sends it, and + a non-idempotent hook must not rewrite it twice.""" + request = Request(messages=[*branch, *tail]) + for position, key in self.tool_result_keys(request).items(): + message = request.messages[position] + if position >= len(branch) and isinstance(message, ToolMessage): + self.pinned_tool_results.setdefault(key, message) + self.prepared_users.update( + graph.message_hash(message) + for message in tail + if isinstance(message, UserMessage) + ) + def consume_prepared(self, messages: Messages) -> None: """Forget pre-harness user rewrites only after their model request commits.""" for message in messages: @@ -605,6 +630,12 @@ def adopt(self, task: "asyncio.Task | None") -> None: self.tasks.add(task) task.add_done_callback(self.tasks.discard) + def abort(self, error: "RolloutError") -> None: + """End the running segment with `error`. The first abort wins.""" + if self.aborted is None: + self.aborted = error + self.abort_event.set() + def release(self) -> None: """Seal the session: no further trace mutation, and in-flight handlers cancel.""" self.released = True diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index f526c17d5a..b2de785353 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -250,6 +250,8 @@ class TimeSplit(BaseModel): class AgentSpan(TimeSpan): model: TimeSplit = Field(default_factory=TimeSplit) harness: TimeSplit = Field(default_factory=TimeSplit) + checkpoint: TimeSplit = Field(default_factory=TimeSplit) + """Time the model replies waited for checkpoint captures (`Checkpoint.held`).""" class Timing(BaseModel): @@ -266,6 +268,10 @@ class Error(BaseModel): message: str status_code: int | None = None traceback: str | None = None + attempt: int = 0 + """The run attempt that captured it: 0 for the first, then one per retry.""" + stage: str | None = None + """The rollout stage that failed (`setup`, `agent`, `finalize`, `scoring`), when known.""" class VersionInfo(BaseModel): @@ -369,6 +375,104 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall prev_total = call.usage.total_tokens +class Checkpoint(BaseModel): + """A snapshot of the rollout's box taken as a model turn went upstream: the box holds + the effects of every message in `node`'s conversation plus `tail`, and nothing of the + turn that followed. A retry restores it (`Trace.rewind`).""" + + id: str + """The runtime's checkpoint id (`Runtime.checkpoint`), or this record's own for a + box-less checkpoint.""" + box: bool = True + """Whether a runtime snapshot holds the box. A box-less checkpoint (a harness that + keeps nothing in its box, on a runtime without snapshots) restores into a fresh box + whose task setup runs again; the trace and its typed state carry the rest.""" + node: int | None = None + """The last committed node whose conversation the box's state follows; None when + the snapshot preceded the first turn.""" + tail: Messages = Field(default_factory=list, exclude=True) + """The turn's prompt messages after `node` that had no node yet (tool results, user + turns): with `node`'s conversation, the prompt a restored harness continues from. + Live-only — a retry restores it in the process that took the snapshot.""" + state: Any = Field(None, exclude=True) + """The trace's typed state as the snapshot was taken (tool servers write it over + `/state`), restored with the box so a retry sees one consistent world. Live-only.""" + info: dict[str, Any] | None = Field(None, exclude=True) + """`Trace.info` as the snapshot was taken, restored with `state`. Live-only.""" + pins: dict[tuple[str, str], ToolMessage] = Field(default_factory=dict, exclude=True) + """The rollout's pinned tool results as the snapshot was taken + (`RolloutSession.pinned_tool_results`): a restored harness re-sends its own + copies, which must be pinned again to match the trace. Live-only.""" + time: TimeSpan = Field(default_factory=TimeSpan) + """From the snapshot request until it was durable (restorable); `end` stays 0 while + it is not.""" + captured: float = 0.0 + """When the request returned: the snapshot's content was fixed from then on.""" + held: float = 0.0 + """Seconds the turn's reply waited for the capture; 0 when the model call took + longer, since the snapshot runs alongside it.""" + bytes: int | None = None + """The snapshot's stored size, which the provider bills, once it is durable.""" + harness: dict[str, Any] = Field(default_factory=dict) + """What the harness session needs to continue in a box restored from the snapshot + (`HarnessSession.checkpoint_state`), e.g. an ACP agent's session id.""" + restored: bool = False + """Whether a retry restored the run from this snapshot.""" + deleted: bool = False + """Whether the snapshot was deleted (`RolloutCheckpointConfig.keep`, `.after`): it + can no longer be restored.""" + + +class PinnedResult(BaseModel): + """One pinned tool result (`RolloutSession.pinned_tool_results`), keyed by the + conversation prefix that issued its call and the call's id.""" + + prefix: str + call_id: str + message: ToolMessage + + +class RestorePoint(BaseModel): + """A checkpoint with everything a restore needs, serializable: its live-only fields + travel with it, so another process (an orchestrator resuming its run) can restore + the trace from it (`RunRequest.restore`).""" + + checkpoint: Checkpoint + tail: Messages = Field(default_factory=list) + state: dict[str, Any] | None = None + info: dict[str, Any] | None = None + pins: list[PinnedResult] = Field(default_factory=list) + + @classmethod + def of(cls, checkpoint: Checkpoint) -> RestorePoint: + state = checkpoint.state + return cls( + checkpoint=checkpoint, + tail=list(checkpoint.tail), + state=state.model_dump(mode="json") + if isinstance(state, BaseModel) + else state, + info=checkpoint.info, + pins=[ + PinnedResult(prefix=prefix, call_id=call_id, message=message) + for (prefix, call_id), message in checkpoint.pins.items() + ], + ) + + def live(self, state_type: type[BaseModel]) -> Checkpoint: + """The checkpoint with its live-only fields back, the state as `state_type`.""" + return self.checkpoint.model_copy( + update={ + "tail": list(self.tail), + "state": state_type.model_validate(self.state) + if self.state is not None + else None, + "info": self.info, + "pins": {(p.prefix, p.call_id): p.message for p in self.pins}, + } + ) + + def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: """Node streams (lists or arrays) joined into one array of `dtype`.""" if not parts: @@ -603,6 +707,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) """Response changes made by `@intercept`, in execution order.""" + checkpoints: list[Checkpoint] = Field(default_factory=list) + """Snapshots of the run's box, oldest first — one per model turn on a runtime that + supports them; a retry restores the newest.""" rewards: dict[str, Reward | None] = Field(default_factory=dict) """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a @@ -643,6 +750,14 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): _pending: dict[int, list[Message]] = PrivateAttr(default_factory=dict) """The messages of the request in flight that no node holds yet (the harness's tool results and user turns): a preview for live watchers until the turn commits.""" + _generation: int = PrivateAttr(default=0) + """Bumped by `rewind`: the graph is no longer an extension of what it was, so a + consumer streaming it incrementally starts over.""" + + @property + def generation(self) -> int: + """How many times the trace was rewound (`rewind`).""" + return self._generation def watch(self, on_change: Callable[[Trace], None]) -> None: """Have `on_change` called at each of this trace's phase changes and turns.""" @@ -883,6 +998,19 @@ def last_error(self) -> Error | None: """The last error captured across attempts.""" return self.errors[-1] if self.errors else None + @property + def restorable(self) -> Checkpoint | None: + """The newest checkpoint a restore can use: restorable and not deleted.""" + return next( + (c for c in reversed(self.checkpoints) if c.time.end and not c.deleted), + None, + ) + + @property + def has_restored(self) -> bool: + """Whether a retry restored this run from a checkpoint rather than starting over.""" + return any(checkpoint.restored for checkpoint in self.checkpoints) + @property def transcript(self) -> str: """Text transcript of the last branch's messages; tool outputs are omitted.""" @@ -957,13 +1085,22 @@ def drop(self, reason: str) -> None: self.dropped = reason def split_agent_time(self) -> None: - """Split the agent span into model and harness time.""" + """Split the agent span into model, checkpoint and harness time.""" span = self.timing.agent if not span.end: return - model = sum(call.time.duration for call in self.calls) - span.model.duration = min(model, span.duration) - span.harness.duration = span.duration - span.model.duration + # A restored trace keeps the calls and checkpoints of the attempts before + # this span. + calls = sum( + call.time.duration for call in self.calls if call.time.start >= span.start + ) + # A call's span includes the time its reply waited for the capture. + held = sum(c.held for c in self.checkpoints if c.captured >= span.start) + span.checkpoint.duration = min(held, calls, span.duration) + span.model.duration = min(calls, span.duration) - span.checkpoint.duration + span.harness.duration = ( + span.duration - span.model.duration - span.checkpoint.duration + ) def record_timeout(self, stage: str) -> None: """Record a stage deadline's expiry, and stop the trace as `_timeout`. @@ -972,7 +1109,52 @@ def record_timeout(self, stage: str) -> None: self.is_timeout = True self.stop(f"{stage}_timeout", override=True) - def record_error(self, error: Exception) -> None: + def rewind_state(self, checkpoint: Checkpoint) -> None: + """Set the typed state and `info` back to `checkpoint`'s.""" + if checkpoint.state is not None: + self.state = copy.deepcopy(checkpoint.state) + if checkpoint.info is not None: + self.info = copy.deepcopy(checkpoint.info) + + def rewind(self, checkpoint: Checkpoint) -> list[str]: + """Roll the graph back to `checkpoint` — the nodes committed after it go, its own + conversation stays sampled — and reopen the trace for another attempt: the outcome + stamps reset, while `errors` and every call stay as the attempts' history (the + calls of the dropped nodes unlinked, so usage still counts them). Returns the ids + of the later box snapshots this drops.""" + keep = 0 if checkpoint.node is None else checkpoint.node + 1 + if keep > len(self.nodes): + raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + del self.nodes[keep:] + for node in self.nodes: + node.semantic_parents = [ + link for link in node.semantic_parents if link.node < keep + ] + self._head_index = {} + self._pending = {} + self._generation += 1 + for call in self.calls: + if call.node is not None and call.node >= keep: + call.node = None + self.rewind_state(checkpoint) + # The restored rollout re-stamps every stage span; the run's start stays. + self.timing = Timing(start=self.timing.start) + rewound = [c for c in self.checkpoints if c.node is not None and c.node >= keep] + self.checkpoints = [c for c in self.checkpoints if c not in rewound] + dropped = [c.id for c in rewound if c.box] + self.rewards = {} + self.metrics = {} + self.dropped = None + self.root_reply = None + self.is_completed = False + self.ok = False + self.stop_condition = None + self.is_timeout = False + return dropped + + def record_error( + self, error: Exception, *, attempt: int = 0, stage: str | None = None + ) -> None: """Record an error, and stop the trace as failed: `_error` for a typed rollout error (`errors.stop_condition`), `error` for any other.""" self.errors.append( @@ -980,6 +1162,8 @@ def record_error(self, error: Exception) -> None: type=type(error).__name__, message=str(error), status_code=getattr(error, "status_code", None), + attempt=attempt, + stage=stage, # Provider errors already carry the actionable upstream diagnostic. # Keep full tracebacks for every other failure. traceback=None From 824ca302b1db9c038125ba0ea409156665808031 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 22:40:33 +0000 Subject: [PATCH 02/11] narrow checkpoints to retries and name snapshots Scope rollout checkpointing to retries within one Agent.run: drop the cross-process restore (Env and Agent.run restore params, RestorePoint, PinnedResult, Trace.restorable, Agents.only). Call the runtime side a snapshot: backends return a typed Snapshot (id, type) from snapshot(), wait_snapshot() and discard_snapshots() take it, and restore_config() builds the config a new box boots from. Runtimes and harnesses declare SUPPORTS_FILESYSTEM_SNAPSHOTS, and RolloutCheckpointConfig.type takes a SnapshotType, so memory snapshots can slot in behind the same API. Also from review: - snapshot every 10th turn by default - copy state before the first await of a capture, so response hooks that run meanwhile do not leak into the checkpoint - trim the checkpoints after the restored one by position, not node - build the rollout params once per run, so an RLM retry keeps its skill home and the restored session loads - discard snapshots on the shared leased client Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 2 +- docs/harnesses.md | 2 +- tests/unit/test_checkpoint.py | 57 ++++++------ tests/unit/test_retry_ids.py | 4 +- verifiers/v1/__init__.py | 6 ++ verifiers/v1/agent.py | 80 ++++++----------- verifiers/v1/configs/agent.py | 44 ++++----- verifiers/v1/configs/runtime.py | 4 + verifiers/v1/env.py | 21 +---- verifiers/v1/harness.py | 4 +- verifiers/v1/harnesses/bash/harness.py | 2 +- verifiers/v1/harnesses/claude_code/harness.py | 2 +- verifiers/v1/harnesses/codex/harness.py | 2 +- verifiers/v1/harnesses/null/harness.py | 2 +- verifiers/v1/harnesses/rlm/harness.py | 6 +- verifiers/v1/rollout.py | 82 +++++++++-------- verifiers/v1/runtimes/__init__.py | 4 + verifiers/v1/runtimes/backends/prime.py | 42 +++++---- verifiers/v1/runtimes/base.py | 55 +++++++----- verifiers/v1/trace.py | 90 ++++--------------- 20 files changed, 222 insertions(+), 289 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 7ef3334272..4b775c23c1 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,4 +77,4 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box as model turns go upstream, and a retry restores the newest snapshot and continues the trace from there. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_CHECKPOINTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `interval` sets how often it snapshots, `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box at model turns, and a retry restores the newest snapshot and continues the trace from there. Only retries within one `Agent.run` restore; a new process starts over. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_SNAPSHOTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `type` sets what a snapshot captures (`"filesystem"`, the only type today), `interval` how often it snapshots (every 10th turn by default), `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. diff --git a/docs/harnesses.md b/docs/harnesses.md index e634a3e746..2144e499ff 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `SUPPORTS_FILESYSTEM_CHECKPOINTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem checkpoint, so a retry restores the newest checkpoint instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. +- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem snapshot, so a retry restores the newest snapshot instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 19b1918330..414c877189 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -16,7 +16,6 @@ from verifiers.v1.trace import ( Checkpoint, ModelCall, - RestorePoint, TimeSpan, TraceTask, ) @@ -67,6 +66,16 @@ def test_rewind_keeps_the_dropped_turns_calls(): assert [call.node for call in trace.calls] == [None] +def test_rewind_drops_the_later_checkpoints_at_the_same_node(): + trace = make_trace() + older = Checkpoint(id="older", node=1, snapshot=vf.Snapshot(id="older")) + target = Checkpoint(id="target", node=1, snapshot=vf.Snapshot(id="target")) + later = Checkpoint(id="later", node=1, snapshot=vf.Snapshot(id="later")) + trace.checkpoints.extend([older, target, later]) + assert trace.rewind(target) == [vf.Snapshot(id="later")] + assert [c.id for c in trace.checkpoints] == ["older", "target"] + + def test_restored_span_counts_only_its_own_calls(): trace = make_trace() trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 @@ -128,17 +137,18 @@ def __init__(self) -> None: self.taken = 0 self.deleted: list[str] = [] - async def checkpoint(self) -> str: + async def snapshot(self) -> vf.Snapshot: self.taken += 1 - return f"ck{self.taken}" + return vf.Snapshot(id=f"ck{self.taken}") - async def wait_checkpoint(self, checkpoint: str) -> int | None: + async def wait_snapshot(self, snapshot: vf.Snapshot) -> int | None: await self.durable.wait() if self.fail: raise RuntimeError("upload failed") return 1024 - async def discard_checkpoints(self, ids: list[str]) -> list[str]: + async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: + ids = [snapshot.id for snapshot in snapshots] self.deleted += ids return ids @@ -161,8 +171,8 @@ async def checkpoint_state() -> dict: run._captures = set() run._closed = False run._pruning = asyncio.Lock() - run.deleted_checkpoints = set() - run.rewound_checkpoints = [] + run.deleted_snapshots = set() + run.rewound_snapshots = [] run._boxless = False return run, backend @@ -220,7 +230,12 @@ async def test_a_closing_run_takes_no_checkpoint(): def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: return [ - Checkpoint(id=i, time=TimeSpan(start=1.0, end=2.0), deleted=i in deleted) + Checkpoint( + id=i, + snapshot=vf.Snapshot(id=i), + time=TimeSpan(start=1.0, end=2.0), + deleted=i in deleted, + ) for i in ids ] @@ -239,40 +254,22 @@ def test_keep_policies(): async def test_a_durable_snapshot_prunes_the_older_ones(): run, backend = make_rollout(keep="last") run.trace.checkpoints = durable("old1", "old2") - run.rewound_checkpoints = ["rewound"] + run.rewound_snapshots = [vf.Snapshot(id="rewound")] backend.durable.set() newest = await run._checkpoint(TURN) await run._settle_checkpoints(wait=True) assert sorted(backend.deleted) == ["old1", "old2", "rewound"] assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] - assert run.deleted_checkpoints == {"old1", "old2", "rewound"} + assert run.deleted_snapshots == {"old1", "old2", "rewound"} async def test_a_boxless_checkpoint_is_restorable_at_once(): run, backend = make_rollout() run._boxless = True checkpoint = await run._checkpoint(TURN) - assert checkpoint is not None and not checkpoint.box + assert checkpoint is not None and checkpoint.snapshot is None assert checkpoint.time.end and backend.taken == 0 - assert run.trace.restorable is checkpoint - - -def test_a_restore_point_round_trips(): - checkpoint = Checkpoint( - id="ck", - node=1, - tail=[ToolMessage(tool_call_id="c1", content="out")], - state=vf.State(), - info={"nonce": "n"}, - pins={("prefix", "c1"): ToolMessage(tool_call_id="c1", content="pinned")}, - time=TimeSpan(start=1.0, end=2.0), - ) - wire = RestorePoint.of(checkpoint).model_dump(mode="json") - back = RestorePoint.model_validate(wire).live(vf.State) - assert back.tail == checkpoint.tail - assert back.info == {"nonce": "n"} - assert back.pins == checkpoint.pins - assert back.state == checkpoint.state + assert run.trace.checkpoints[-1] is checkpoint def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 7c3d6f9613..45c540ecf8 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -36,8 +36,8 @@ class FakeRollout: runtime = None def __init__(self, *, on_trace, trace_id, attempt, **_): - self.rewound_checkpoints: list[str] = [] - self.deleted_checkpoints: set[str] = set() + self.rewound_snapshots: list[str] = [] + self.deleted_snapshots: set[str] = set() self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index 33ff9dd52b..f71349fef3 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -95,6 +95,8 @@ Runtime, RuntimeInfo, RuntimeProcess, + Snapshot, + SnapshotType, ) from verifiers.v1.runtimes.compose import ComposeSpec from verifiers.v1.runtimes.deployment import Deployment, ServicesSpec @@ -279,6 +281,8 @@ "RuntimeConfig": "verifiers.v1.configs.runtime", "RuntimeInfo": "verifiers.v1.runtimes", "RuntimeProcess": "verifiers.v1.runtimes", + "Snapshot": "verifiers.v1.runtimes", + "SnapshotType": "verifiers.v1.runtimes", "Sampling": "verifiers.v1.configs.sampling", "SamplingConfig": "verifiers.v1.configs.sampling", "SamplingMask": "verifiers.v1.types", @@ -489,6 +493,8 @@ "RuntimeProcess", "RuntimeConfig", "RuntimeInfo", + "Snapshot", + "SnapshotType", "ProgramResult", "SubprocessConfig", "DockerConfig", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 449ce0718d..5fdab5eacc 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -14,7 +14,7 @@ from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Generic, Literal, Self, cast +from typing import Any, Generic, Literal, Self, cast from weakref import WeakValueDictionary from typing_extensions import TypeVar @@ -39,6 +39,7 @@ from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, backend_cls, provision_backend, runtime_is_local, @@ -575,9 +576,6 @@ def __init__( sampling=config.sampling, ) self._closed = False - self.pending_restore: tuple[Trace, Checkpoint] | None = None - """A restore the next `run` starts from (a trace another process ran); consumed - by that run.""" self.runtime_config: RuntimeConfig = config.runtime self.interception = interception self.limits = RolloutLimits( @@ -656,8 +654,8 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: return None harness = self.harness backend = backend_cls(self.config.runtime) - if harness.SUPPORTS_FILESYSTEM_CHECKPOINTS and ( - backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + if harness.SUPPORTS_FILESYSTEM_SNAPSHOTS and ( + backend.SUPPORTS_FILESYSTEM_SNAPSHOTS or (config.boxless and not harness.EXECUTES_CODE) ): return config @@ -666,7 +664,7 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: _checkpointing_warned.add(key) reason = ( f"harness {harness.config.id!r} does not support it" - if not harness.SUPPORTS_FILESYSTEM_CHECKPOINTS + if not harness.SUPPORTS_FILESYSTEM_SNAPSHOTS else f"the {self.config.runtime.type!r} runtime takes no snapshots" ) logger.warning( @@ -702,7 +700,6 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: """Run this agent on `task` once and return the trace: one segment — the program runs on the task's prompt until it exits (a multi-turn exchange @@ -721,27 +718,23 @@ async def run( the selected verifier workspace, inside its timeout and retry boundary.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") - if restore is None and self.pending_restore is not None: - restore, self.pending_restore = self.pending_restore, None retry = self.config.retries retry_state = RetryState(retry) checkpointing = self._checkpointing() if runtime is None else None - # One id for the run across its attempts; a restored trace keeps its own. - trace_id = restore[0].id if restore is not None else uuid.uuid4().hex + trace_id = uuid.uuid4().hex # one id for the run across its attempts history: list[Error] = [] # errors of the attempts started over from - # `restore` continues a trace another process ran (an orchestrator resuming its - # run): its first attempt restores it, and `on_trace` sees it as the run's trace. - trace: Trace | None = restore[0] if restore is not None else None - target: Checkpoint | None = restore[1] if restore is not None else None - if restore is not None and on_trace is not None: - on_trace(restore[0]) - snapshots: set[str] = set() # every checkpoint any attempt took + trace: Trace | None = None + target: Checkpoint | None = None # the checkpoint the next attempt restores + snapshots: dict[str, Snapshot] = {} # every box snapshot any attempt took deleted: set[str] = set() failed_restores: set[str] = set() run: Rollout | None = None # The `finally` also covers a cancellation between attempts, when no rollout is # open to delete them on abort. budget = retry_state.max_retries + # Built once: every attempt runs the same harness instance, so per-run harness + # state (RLM's skill home) is the one a restored box holds. + params = self._rollout_params(task, runtime, dict(tools or {})) attempt = 0 try: while True: @@ -756,7 +749,7 @@ async def run( run = await self._run_once( task, runtime, - tools, + params, on_trace, collect_artifacts, on_score, @@ -766,9 +759,11 @@ async def run( checkpoint=checkpointing, ) trace = run.trace - snapshots.update(run.rewound_checkpoints) - snapshots.update(c.id for c in trace.checkpoints if c.box) - deleted.update(run.deleted_checkpoints) + snapshots.update((s.id, s) for s in run.rewound_snapshots) + snapshots.update( + (c.snapshot.id, c.snapshot) for c in trace.checkpoints if c.snapshot + ) + deleted.update(run.deleted_snapshots) if target is not None and any( error.stage == "setup" for error in trace.errors[seen:] ): @@ -776,25 +771,6 @@ async def run( failed_restores.add(target.id) if trace.ok: break - if ( - attempt == 0 - and restore is not None - and target is not None - and target.id in failed_restores - ): - # Another process's restore that cannot boot (its snapshot is gone) - # is not the rollout's failure: start over, unbudgeted. - logger.warning( - "restoring trace %s from checkpoint %s failed; starting over", - trace.id, - target.id, - ) - history.extend(trace.errors) - target = None - trace = None - attempt += 1 - budget += 1 - continue if attempt >= budget: break if runtime is not None: @@ -839,13 +815,15 @@ async def run( and checkpointing.after == "keep" else set() ) - drop = sorted(snapshots - deleted - keep) + drop = [ + snapshot + for snapshot_id, snapshot in sorted(snapshots.items()) + if snapshot_id not in deleted and snapshot_id not in keep + ] if drop and run is not None and run.runtime is not None: try: run.record_deleted( - await run_shielded( - run.runtime.backend.discard_checkpoints(drop) - ) + await run_shielded(run.runtime.backend.discard_snapshots(drop)) ) except Exception: logger.warning( @@ -864,7 +842,7 @@ async def _run_once( self, task: Task, runtime: Runtime | None, - shared_tools: Mapping[str, SharedToolServer] | None, + params: dict[str, Any], on_trace: Callable[[Trace], None] | None, collect_artifacts: bool, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None, @@ -874,7 +852,6 @@ async def _run_once( restore: tuple[Trace, Checkpoint] | None = None, checkpoint: RolloutCheckpointConfig | None = None, ) -> Rollout: - params = self._rollout_params(task, runtime, dict(shared_tools or {})) if ( collect_artifacts and not backend_cls(params["runtime_config"]).SUPPORTS_TASK_IMAGES @@ -1138,7 +1115,6 @@ async def run( on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Trace: async with self._gate or nullcontext(): trace = await super().run( @@ -1148,7 +1124,6 @@ async def run( on_trace=self._watch(on_trace), collect_artifacts=collect_artifacts, on_score=self._scoring_callback(on_score), - restore=restore, ) self._completed.append(trace) return trace @@ -1216,11 +1191,6 @@ def __init__(self, config, make: MakeAgent[AgentT] | None = None) -> None: for name, value in agent_config_fields(config).items() } - def only(self) -> AgentT | None: - """The agent of a config with one agent seat; None when it has several.""" - agents = list(self._agents.values()) - return agents[0] if len(agents) == 1 else None - def __getattr__(self, name: str) -> AgentT: # self.__dict__ directly: attribute lookup re-entering __getattr__ before # __init__ ran (copy/unpickle) must raise, not recurse. diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index c9aa905980..833262f827 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -17,7 +17,7 @@ from verifiers.v1.configs.deployment import DeploymentConfig from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig from verifiers.v1.configs.retries import RetryConfig -from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig +from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig, SnapshotType from verifiers.v1.configs.sampling import SamplingConfig from verifiers.v1.configs.scoring import ScoringConfig from verifiers.v1.utils.generic import merge_defaults @@ -40,36 +40,26 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing. Snapshot the rollout's box (its filesystem) as model - turns go upstream, so a retry restores the newest snapshot instead of starting - over: the trace keeps its turns up to there, and a new harness process continues - in a box restored to that state, told that the box was restored. Needs a runtime - with checkpoints (prime) and a harness that can restore one (both set - `SUPPORTS_FILESYSTEM_CHECKPOINTS`); elsewhere the agent warns once and a retry starts over. - The snapshot runs alongside the model call, and the reply waits only until its - content is fixed. - - Known limits: a harness with concurrent sub-agents can be snapshotted while one of - them is mid-tool, and processes the agent left running (background jobs, - sub-agents, a REPL's variables) are gone after a restore, which cold boots.""" - - interval: PositiveInt = 1 - """Snapshot before every this-many-th model turn (the 1st, the 1+n-th, ...). A - larger interval takes fewer snapshots, and a restore loses up to this many turns.""" + """Rollout checkpointing: snapshot the box at model turns, so a retry continues + from the newest snapshot instead of starting over. Needs a runtime and a harness + that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts over. Processes + the agent left running (background jobs, a REPL) are gone after a restore.""" + + type: SnapshotType = "filesystem" + """What a snapshot captures.""" + interval: PositiveInt = 10 + """Snapshot at every n-th model turn (the 1st, the 1+n-th, ...); a restore loses + up to n turns.""" keep: Literal["all", "last"] | PositiveInt = "last" - """Which snapshots stay while the rollout runs: `"last"` keeps only the newest - restorable one, `"all"` keeps every one, and `k` keeps every k-th (the 1st, the - 1+k-th, ...) plus the newest. The others are deleted as soon as a newer one is - restorable. A retry whose restore fails falls back to an older snapshot that is - still kept, else it starts over.""" + """Snapshots kept while the rollout runs: the newest (`"last"`), every one + (`"all"`), or every k-th plus the newest (`k`). A failed restore falls back to an + older kept one.""" after: Literal["delete", "keep"] = "delete" - """What happens to the kept snapshots when the rollout's run is final (it succeeded, - or its retries ran out): `"delete"` deletes them once the box is gone; `"keep"` - leaves them for their owner (forks, inspection), who must delete them.""" + """What happens to the kept snapshots once the run is final: `"delete"` them, or + `"keep"` them for their owner to delete.""" boxless: bool = False """Also checkpoint a harness that runs no code (null) on a runtime without - snapshots, from the trace alone. Off by default: such a run records a checkpoint - on its trace at every snapshot turn.""" + snapshots, from the trace alone.""" class AgentConfig(BaseConfig): diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 2ac1fcdc4e..716dadb1ce 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -14,6 +14,10 @@ parse_network_rule, ) +SnapshotType = Literal["filesystem"] +"""What a runtime snapshot captures: `filesystem` holds the box's files, not its +processes.""" + class BindMount(BaseConfig): """An existing file or directory on the container engine's host.""" diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 2223174e5c..1f943574fa 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -32,7 +32,7 @@ from verifiers.v1.mcp import SharedToolServer, serve_shared from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local from verifiers.v1.task import Task -from verifiers.v1.trace import Checkpoint, Error, Trace, TraceTask +from verifiers.v1.trace import Error, Trace, TraceTask from verifiers.v1.utils.generic import concrete_type from verifiers.v1.utils.memory import trim_memory_periodically from verifiers.v1.utils.retries import run_episode_with_retry @@ -244,7 +244,6 @@ async def run_episode( on_discard: Callable[[Trace], None] | None = None, episode_id: str | None = None, attempt: int = 1, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """One episode of `task`, minted as the wire atom: `setup()` then `run()` over fresh agents, then `finalize()`. Its agents run one at a time @@ -267,17 +266,6 @@ async def run_episode( ), ) agents = self._episode_agents(ctx, episode.traces, on_trace, on_discard) - if restore is not None: - # A one-seat episode's run is its one trace: its agent's run continues it. - agent = agents.only() - if agent is None: - logger.warning( - "%s has several agent seats; running trace %s from scratch", - type(self).__name__, - restore[0].id, - ) - else: - agent.pending_restore = restore try: async with asyncio.timeout(self.config.timeout.episode): async with boundary(EnvError, f"{type(self).__name__}.setup()"): @@ -332,17 +320,13 @@ async def run_slot( semaphore: asyncio.Semaphore | None = None, on_complete: Callable[[Episode], Awaitable[None]] | None = None, on_trace: Callable[[Trace], None] | None = None, - restore: tuple[Trace, Checkpoint] | None = None, ) -> Episode: """Run one planned episode to completion, with whole-episode retries per `--env.retries`; `semaphore` bounds concurrent EPISODES — one permit for the attempt in flight, held across the whole of it (its agents, their boxes, `finalize()`) and released before a retry's backoff and before `on_complete` (the runners' persistence hook, which fires when final). - `on_trace` sees each trace at mint, after it joined `slot.traces`. `restore` - continues a trace another process ran (one-seat episodes); only the first - attempt restores it, a whole-episode retry starts over.""" - pending = [restore] if restore is not None else [] + `on_trace` sees each trace at mint, after it joined `slot.traces`.""" attempts = 0 @@ -370,7 +354,6 @@ def discard(trace: Trace) -> None: on_discard=discard, episode_id=slot.id, attempt=attempts, - restore=pending.pop() if pending else None, ) episode = await run_episode_with_retry(attempt, self.config.retries) diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 963d63c931..0f4d019ae0 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -59,11 +59,11 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" - SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False """Whether a new process of this harness can continue a rollout in a box restored from a filesystem checkpoint (`HarnessSession.restore`), so a retry restores the newest checkpoint instead of starting over (`AgentConfig.checkpoint`). A rollout - checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_CHECKPOINTS` too. Distinct + checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_SNAPSHOTS` too. Distinct from `SUPPORTS_RESUME`: a restore never finds a live process, its program continues from the conversation the trace or its own session files hold.""" EXECUTES_CODE: ClassVar[bool] = True diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index e2790dfbd4..3a7f2e8b70 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,7 +52,7 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index e2b4495e27..d66d9904b0 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -42,7 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index e45406ab40..57f4eced3a 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -63,7 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 431c95f698..252a23118f 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,7 +26,7 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 1c22e675e7..cbf76160d5 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -142,14 +142,14 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_CHECKPOINTS = True + SUPPORTS_FILESYSTEM_SNAPSHOTS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: if self.config.skills: # Editable skill packages and uv's tool environment must belong to this run. - # A retry keeps the directory: it holds `RLM_HOME`, whose session a box - # restored from a checkpoint loads. + # A retry runs the same harness and keeps the directory: it holds + # `RLM_HOME`, whose session a box restored from a checkpoint loads. if self._skills_install_dir is None: self._skills_install_dir = ( f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 96ad61ea28..bf90806e66 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -38,6 +38,8 @@ from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, + backend_cls, make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms @@ -181,17 +183,18 @@ def __init__( self._attempt = attempt self._restore: Checkpoint | None = None self._harness_data: TaskData | None = None - self.deleted_checkpoints: set[str] = set() + self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" self._pruning = asyncio.Lock() self._boxless = False - """Whether this run's checkpoints are box-less (`Checkpoint.box`).""" - self.rewound_checkpoints: list[str] = [] - """Checkpoints the restored trace dropped (taken after the one restored).""" + """Whether this run's checkpoints are box-less (no `Checkpoint.snapshot`).""" + self.rewound_snapshots: list[Snapshot] = [] + """The snapshots of the checkpoints the restored trace dropped (taken after the + one restored).""" if restore is not None: self.trace, self._restore = restore self.trace.attempt = attempt - self.rewound_checkpoints = self.trace.rewind(self._restore) + self.rewound_snapshots = self.trace.rewind(self._restore) self._restore.restored = True else: self.trace = Trace( @@ -316,8 +319,10 @@ async def open(self) -> bool: # A restored trace keeps its id; its new box must not collide with the # box an earlier attempt or process left behind. name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" - if self._restore.box: - config = config.model_copy(update={"checkpoint": self._restore.id}) + if self._restore.snapshot is not None: + config = backend_cls(config).restore_config( + config, self._restore.snapshot + ) self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None @@ -397,7 +402,7 @@ async def open(self) -> bool: self.trace.agent.runtime = runtime.info.model_copy() # A box snapshot needs a runtime that takes them; a harness that keeps # nothing in its box is restorable from the trace alone (box-less). - self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_CHECKPOINTS + self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS if ( self._checkpointing is not None and self._borrowed_runtime is None @@ -429,7 +434,7 @@ async def open(self) -> bool: self.trace.notify() # A restored box already holds the task's setup; only the harness's # (idempotent) install runs again. A box-less restore boots a fresh box. - if self._restore is None or not self._restore.box: + if self._restore is None or self._restore.snapshot is None: async with boundary(TaskError, "task setup"): await invoke( self.task.setup, @@ -813,6 +818,11 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: # box is not idle, so no snapshot would match this turn's prefix. return assert self._harness_session is not None + # Copied before the first await: the reply may commit and run its hooks while + # the harness answers, and the checkpoint must hold the state before them. + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(self._session.pinned_tool_results) try: harness = await self._harness_session.checkpoint_state() except Exception: @@ -824,20 +834,19 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: return if harness is None: return - state = copy.deepcopy(self.trace.state) - info = copy.deepcopy(self.trace.info) - pins = dict(self._session.pinned_tool_results) started = time.time() + snapshot: Snapshot | None = None if self._boxless: checkpoint_id = uuid.uuid4().hex else: try: - checkpoint_id = await self.runtime.backend.checkpoint() + snapshot = await self.runtime.backend.snapshot() except Exception: logger.warning( - "checkpoint failed (rollout %s)", self.trace.id, exc_info=True + "snapshot failed (rollout %s)", self.trace.id, exc_info=True ) return None + checkpoint_id = snapshot.id node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None checkpoint = Checkpoint( id=checkpoint_id, @@ -847,7 +856,7 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: info=info, pins=pins, harness=harness, - box=not self._boxless, + snapshot=snapshot, time=TimeSpan(start=started), captured=time.time(), ) @@ -866,9 +875,11 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: return checkpoint async def _make_durable(self, checkpoint: Checkpoint) -> None: - assert self.runtime is not None + assert self.runtime is not None and checkpoint.snapshot is not None try: - checkpoint.bytes = await self.runtime.backend.wait_checkpoint(checkpoint.id) + checkpoint.bytes = await self.runtime.backend.wait_snapshot( + checkpoint.snapshot + ) except Exception: # Not restorable: no retry may pick it. The backend still holds it as # stray and frees it with the box. @@ -895,26 +906,27 @@ async def _prune(self) -> None: keep = kept(self.trace.checkpoints, self._checkpointing.keep) drop = [ *( - c.id + c.snapshot for c in self.trace.checkpoints - if c.box and c.time.end and not c.deleted and c.id not in keep + if c.snapshot is not None + and c.time.end + and not c.deleted + and c.id not in keep ), *( - c - for c in self.rewound_checkpoints - if c not in self.deleted_checkpoints + s + for s in self.rewound_snapshots + if s.id not in self.deleted_snapshots ), ] if drop: - self.record_deleted( - await self.runtime.backend.discard_checkpoints(drop) - ) + self.record_deleted(await self.runtime.backend.discard_snapshots(drop)) def record_deleted(self, ids: list[str]) -> None: """Mark the snapshots with these ids deleted.""" - self.deleted_checkpoints.update(ids) + self.deleted_snapshots.update(ids) for checkpoint in self.trace.checkpoints: - if checkpoint.id in self.deleted_checkpoints: + if checkpoint.id in self.deleted_snapshots: checkpoint.deleted = True async def _settle_checkpoints(self, *, wait: bool) -> None: @@ -975,15 +987,15 @@ async def abort(self) -> None: owned = self._checkpointing is None or self._checkpointing.after == "delete" with contextlib.suppress(Exception): self.record_deleted( - await self.runtime.backend.discard_checkpoints( + await self.runtime.backend.discard_snapshots( [ - *(self.rewound_checkpoints if owned else ()), + *(self.rewound_snapshots if owned else ()), *( - c.id + c.snapshot for c in self.trace.checkpoints - if owned and c.box and not c.deleted + if owned and c.snapshot is not None and not c.deleted ), - *self.runtime.backend.stray_checkpoints, + *self.runtime.backend.stray_snapshots, ] ) ) @@ -1283,9 +1295,9 @@ async def close(self) -> Trace: ): # Setup can fail before a deployment takes ownership. await runtime.backend.stop() - if runtime is not None and runtime.backend.stray_checkpoints: - await runtime.backend.discard_checkpoints( - runtime.backend.stray_checkpoints + if runtime is not None and runtime.backend.stray_snapshots: + await runtime.backend.discard_snapshots( + runtime.backend.stray_snapshots ) except Exception: logger.warning("runtime teardown failed", exc_info=True) diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 46f82c44f6..7081299e8b 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -28,6 +28,8 @@ ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, + SnapshotType, register, ) from verifiers.v1.runtimes.runtime import Runtime @@ -108,6 +110,8 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "RuntimeConfig", "RuntimeInfo", "RuntimeProcess", + "Snapshot", + "SnapshotType", "SubprocessBackend", "SubprocessConfig", "SubprocessRuntimeInfo", diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index a2e8df945c..081af6f5c9 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -28,6 +28,7 @@ ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, parse_gpu, ) from verifiers.v1.runtimes.limiters import creation_limiter @@ -154,7 +155,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True - SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = True + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -293,56 +294,59 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e - async def checkpoint(self) -> str: + async def snapshot(self) -> Snapshot: # The platform fixes the snapshot's content before the request returns, then # uploads it. Until it is durable the snapshot is stray: the platform refuses # to delete it while this box lives, so teardown frees it after. - async def create() -> str: + async def create() -> Snapshot: checkpoint = await self._client.checkpoint(self.info.id) - self.stray_checkpoints.append(checkpoint.id) - return checkpoint.id + snapshot = Snapshot(id=checkpoint.id) + self.stray_snapshots.append(snapshot) + return snapshot try: return await run_shielded(create()) except Exception as e: raise SandboxError(f"prime checkpoint failed: {e}") from e - async def wait_checkpoint(self, checkpoint: str) -> int | None: + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: started = time.monotonic() try: durable = await self._client.wait_for_checkpoint( - checkpoint, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + snapshot.id, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS ) except Exception as e: - raise SandboxError(f"prime checkpoint {checkpoint}: {e}") from e - self.stray_checkpoints.remove(checkpoint) + raise SandboxError(f"prime checkpoint {snapshot.id}: {e}") from e + self.stray_snapshots.remove(snapshot) logger.debug( "prime: checkpoint %s durable after %.1fs on sandbox %s", - checkpoint, + snapshot.id, time.monotonic() - started, self.info.id, ) return durable.stored_bytes - async def discard_checkpoints(self, ids: list[str]) -> list[str]: + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: # The platform refuses only the newest checkpoint of a running box (the parent # of its live state); older ones go while it runs, their data kept as long as a - # newer one needs it. - from prime_sandboxes import AsyncSandboxClient - + # newer one needs it. Usually called after teardown released `_client`. deleted = [] - async with AsyncSandboxClient() as client: - for checkpoint in ids: + async with leased_client() as client: + for snapshot in snapshots: try: - await client.delete_checkpoint(checkpoint) + await client.delete_checkpoint(snapshot.id) except Exception as e: # noqa: BLE001 - provider cleanup is best-effort logger.warning( - "prime: failed to delete checkpoint %s: %s", checkpoint, e + "prime: failed to delete checkpoint %s: %s", snapshot.id, e ) else: - deleted.append(checkpoint) + deleted.append(snapshot.id) return deleted + @classmethod + def restore_config(cls, config: PrimeConfig, snapshot: Snapshot) -> PrimeConfig: + return config.model_copy(update={"checkpoint": snapshot.id}) + async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None ) -> None: diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 7a917b6f81..f61615fe42 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,11 +13,12 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar +from pydantic import BaseModel from pydantic_config import BaseConfig -from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType from verifiers.v1.errors import SandboxError from verifiers.v1.utils.aio import run_shielded @@ -140,6 +141,15 @@ class BaseRuntimeInfo(BaseConfig): (`Agent.run(runtime=...)`) rather than provisioning its own.""" +class Snapshot(BaseModel): + """A snapshot of a box that new boxes can boot from (`RuntimeBackend.snapshot`, + `restore_config`), as many times as needed.""" + + id: str + """The provider's id for the snapshot.""" + type: SnapshotType = "filesystem" + + @dataclass class RuntimeCache: """Framework state for one provisioned resource, kept off the backend contract: @@ -184,11 +194,10 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" - SUPPORTS_FILESYSTEM_CHECKPOINTS: ClassVar[bool] = False - """Whether `checkpoint` snapshots the box's filesystem for a new box of this config - to boot from (its config's `checkpoint`). A rollout checkpoints before each model - turn, so a retry restores the newest turn instead of replaying the run (see - `AgentConfig.checkpoint`).""" + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False + """Whether `snapshot` captures the box's filesystem for new boxes to boot from + (`restore_config`). Rollout checkpointing (`AgentConfig.checkpoint`) takes one at + model turns, so a retry continues from there instead of starting over.""" info: BaseRuntimeInfo @@ -198,8 +207,8 @@ def __init__(self, name: str | None = None) -> None: # Explicit process values (model credentials, proxy settings, etc.) override these. self.env: dict[str, str] = {} self.cache = RuntimeCache() - self.stray_checkpoints: list[str] = [] - """Checkpoints this box took that never became restorable (a failed or cancelled + self.stray_snapshots: list[Snapshot] = [] + """Snapshots this box took that never became restorable (a failed or cancelled durability wait), so no trace records them; freed after the box is gone.""" self._setup_claimed = False self._background: list[RuntimeProcess] = [] @@ -227,24 +236,30 @@ def type(self) -> str: async def start(self) -> None: pass - async def checkpoint(self) -> str: - """Snapshot the filesystem and return the checkpoint's id once its content is - fixed: writes after this returns are not in it. It becomes restorable only later - (`wait_checkpoint`); the box keeps running.""" - raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + async def snapshot(self) -> Snapshot: + """Snapshot the box and return it once its content is fixed: writes after this + returns are not in it. It becomes restorable only later (`wait_snapshot`); the + box keeps running.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") - async def wait_checkpoint(self, checkpoint: str) -> int | None: - """Wait until `checkpoint` is restorable and return its stored size in bytes, + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: + """Wait until `snapshot` is restorable and return its stored size in bytes, when the provider reports it. Raises if it never becomes restorable. The box - must stay alive until then: its teardown fails a pending checkpoint.""" - raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + must stay alive until then: its teardown fails a pending snapshot.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") - async def discard_checkpoints(self, ids: list[str]) -> list[str]: - """Free checkpoints nothing will restore from and return the ids it freed. + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: + """Free snapshots nothing will restore from and return the ids it freed. Best-effort: a provider may refuse one a live box still needs (prime: the newest one of a running box).""" return [] + @classmethod + def restore_config(cls, config: Any, snapshot: Snapshot) -> Any: + """The runtime config for a new box that boots from `snapshot` instead of the + task's image.""" + raise NotImplementedError(f"{cls.__name__} does not support snapshots") + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index b2de785353..f2ee570738 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -30,7 +30,7 @@ from verifiers.v1.configs.sampling import Sampling from verifiers.v1.errors import ProviderError, stop_condition from verifiers.v1.graph import RECORD_FLOAT_DECIMALS, MessageNode -from verifiers.v1.runtimes import RuntimeInfo +from verifiers.v1.runtimes import RuntimeInfo, Snapshot from verifiers.v1.semantic import ACPInfo, ParentLink, SemanticEdgeSet from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData @@ -381,12 +381,12 @@ class Checkpoint(BaseModel): turn that followed. A retry restores it (`Trace.rewind`).""" id: str - """The runtime's checkpoint id (`Runtime.checkpoint`), or this record's own for a - box-less checkpoint.""" - box: bool = True - """Whether a runtime snapshot holds the box. A box-less checkpoint (a harness that - keeps nothing in its box, on a runtime without snapshots) restores into a fresh box - whose task setup runs again; the trace and its typed state carry the rest.""" + """The snapshot's id, or this record's own for a box-less checkpoint.""" + snapshot: Snapshot | None = None + """The box's snapshot (`RuntimeBackend.checkpoint`) a retry boots a new box from. + None for a box-less checkpoint (a harness that keeps nothing in its box, on a runtime + without snapshots): it restores into a fresh box whose task setup runs again, and + the trace and its typed state carry the rest.""" node: int | None = None """The last committed node whose conversation the box's state follows; None when the snapshot preceded the first turn.""" @@ -423,56 +423,6 @@ class Checkpoint(BaseModel): can no longer be restored.""" -class PinnedResult(BaseModel): - """One pinned tool result (`RolloutSession.pinned_tool_results`), keyed by the - conversation prefix that issued its call and the call's id.""" - - prefix: str - call_id: str - message: ToolMessage - - -class RestorePoint(BaseModel): - """A checkpoint with everything a restore needs, serializable: its live-only fields - travel with it, so another process (an orchestrator resuming its run) can restore - the trace from it (`RunRequest.restore`).""" - - checkpoint: Checkpoint - tail: Messages = Field(default_factory=list) - state: dict[str, Any] | None = None - info: dict[str, Any] | None = None - pins: list[PinnedResult] = Field(default_factory=list) - - @classmethod - def of(cls, checkpoint: Checkpoint) -> RestorePoint: - state = checkpoint.state - return cls( - checkpoint=checkpoint, - tail=list(checkpoint.tail), - state=state.model_dump(mode="json") - if isinstance(state, BaseModel) - else state, - info=checkpoint.info, - pins=[ - PinnedResult(prefix=prefix, call_id=call_id, message=message) - for (prefix, call_id), message in checkpoint.pins.items() - ], - ) - - def live(self, state_type: type[BaseModel]) -> Checkpoint: - """The checkpoint with its live-only fields back, the state as `state_type`.""" - return self.checkpoint.model_copy( - update={ - "tail": list(self.tail), - "state": state_type.model_validate(self.state) - if self.state is not None - else None, - "info": self.info, - "pins": {(p.prefix, p.call_id): p.message for p in self.pins}, - } - ) - - def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: """Node streams (lists or arrays) joined into one array of `dtype`.""" if not parts: @@ -998,14 +948,6 @@ def last_error(self) -> Error | None: """The last error captured across attempts.""" return self.errors[-1] if self.errors else None - @property - def restorable(self) -> Checkpoint | None: - """The newest checkpoint a restore can use: restorable and not deleted.""" - return next( - (c for c in reversed(self.checkpoints) if c.time.end and not c.deleted), - None, - ) - @property def has_restored(self) -> bool: """Whether a retry restored this run from a checkpoint rather than starting over.""" @@ -1116,15 +1058,18 @@ def rewind_state(self, checkpoint: Checkpoint) -> None: if checkpoint.info is not None: self.info = copy.deepcopy(checkpoint.info) - def rewind(self, checkpoint: Checkpoint) -> list[str]: + def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: """Roll the graph back to `checkpoint` — the nodes committed after it go, its own conversation stays sampled — and reopen the trace for another attempt: the outcome stamps reset, while `errors` and every call stay as the attempts' history (the - calls of the dropped nodes unlinked, so usage still counts them). Returns the ids - of the later box snapshots this drops.""" + calls of the dropped nodes unlinked, so usage still counts them). Returns the + later box snapshots this drops.""" keep = 0 if checkpoint.node is None else checkpoint.node + 1 if keep > len(self.nodes): raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + ids = [c.id for c in self.checkpoints] + if checkpoint.id not in ids: + raise ValueError(f"checkpoint {checkpoint.id!r} is not on this trace") del self.nodes[keep:] for node in self.nodes: node.semantic_parents = [ @@ -1139,9 +1084,12 @@ def rewind(self, checkpoint: Checkpoint) -> list[str]: self.rewind_state(checkpoint) # The restored rollout re-stamps every stage span; the run's start stays. self.timing = Timing(start=self.timing.start) - rewound = [c for c in self.checkpoints if c.node is not None and c.node >= keep] - self.checkpoints = [c for c in self.checkpoints if c not in rewound] - dropped = [c.id for c in rewound if c.box] + # Checkpoints are in the order they were taken: the ones after `checkpoint` hold + # the future it rewinds past, including those at its own node. + position = ids.index(checkpoint.id) + 1 + rewound = self.checkpoints[position:] + del self.checkpoints[position:] + dropped = [c.snapshot for c in rewound if c.snapshot is not None] self.rewards = {} self.metrics = {} self.dropped = None From 7a5330a2c6093f598c1c125d1331d3212e266560 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Fri, 6 Nov 2026 23:15:04 +0000 Subject: [PATCH 03/11] record cancelled turns and shorten checkpoint docs A turn cancelled while its reply waits for the snapshot capture now still records its call and abandons its preview. The wait is shielded, so the snapshot finishes in the background, where the rollout awaits it. A capture reads the turn's node and tail before its first await, like the state copies, so everything a checkpoint describes comes from the idle box. Docs: show the checkpoint config as a TOML example, and shorten the harness flag's entry. Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 10 ++++- docs/harnesses.md | 2 +- verifiers/v1/interception/server.py | 66 ++++++++++++++++------------- verifiers/v1/rollout.py | 9 ++-- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 4b775c23c1..fc5f8c0a34 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,4 +77,12 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing (`[env.agent.checkpoint]`) is on by default: the agent snapshots the box at model turns, and a retry restores the newest snapshot and continues the trace from there. Only retries within one `Agent.run` restore; a new process starts over. It needs a runtime that takes snapshots (prime) and a harness that supports it (`SUPPORTS_FILESYSTEM_SNAPSHOTS`). Elsewhere the agent logs a warning once, and each retry starts a fresh attempt from the beginning. `type` sets what a snapshot captures (`"filesystem"`, the only type today), `interval` how often it snapshots (every 10th turn by default), `keep` which snapshots stay while the rollout runs (`"last"`, `"all"`, or every k-th), and `after` whether the kept ones are deleted (`"delete"`) or kept (`"keep"`) when the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Set `--no-env.agent.checkpoint` to turn it off. + +```toml +[env.agent.checkpoint] +type = "filesystem" # what a snapshot captures +interval = 10 # snapshot every 10th model turn +keep = "last" # snapshots kept while the rollout runs: "last", "all", or every k-th +after = "delete" # when the run is final: "delete" the kept snapshots, or "keep" them +``` diff --git a/docs/harnesses.md b/docs/harnesses.md index 2144e499ff..624e909037 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that the harness supports rollout checkpointing: a new process of the harness can continue a rollout in a box restored from a filesystem snapshot, so a retry restores the newest snapshot instead of starting over (`agent.checkpoint`). This is a restore, not a continuation of a live process: the default `HarnessSession.restore` relaunches the program on the trace's conversation, and an ACP harness loads its agent's session from the files in the restored box (`session/load`). Either way the program then gets a user turn that says the box was restored. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the checkpoint holds it. +- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that a new process of the harness can continue a rollout in a box restored from a filesystem snapshot (`agent.checkpoint`). Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index a82a177bc4..f891951ebc 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1046,36 +1046,42 @@ async def sample() -> web.Response: error = e raise finally: - if capture is not None: - held = time.monotonic() - try: - checkpoint = await capture - # A lost snapshot never fails the turn. - except Exception: - logger.warning( - "checkpoint failed: id=%s", session.trace.id, exc_info=True - ) - else: - if checkpoint is not None: - checkpoint.held = time.monotonic() - held - if node is None: - turn.abandon() - # The turn's one per-exchange record: settings, timing, outcome, and - # the error that ended it (if any). - self.record_call( - session, - dialect, - body, - started, - node=node, - finish_reason=call_response.finish_reason - if call_response - else None, - usage=call_response.usage if call_response else None, - error=error, - policy_paths=policy_paths, - acp=acp, - ) + try: + if capture is not None: + held = time.monotonic() + try: + # Shielded: a cancelled exchange leaves the snapshot to + # finish in the background, where the rollout awaits it. + checkpoint = await asyncio.shield(capture) + # A lost snapshot never fails the turn. + except Exception: + logger.warning( + "checkpoint failed: id=%s", + session.trace.id, + exc_info=True, + ) + else: + if checkpoint is not None: + checkpoint.held = time.monotonic() - held + finally: + if node is None: + turn.abandon() + # The turn's one per-exchange record: settings, timing, outcome, + # and the error that ended it (if any). + self.record_call( + session, + dialect, + body, + started, + node=node, + finish_reason=call_response.finish_reason + if call_response + else None, + usage=call_response.usage if call_response else None, + error=error, + policy_paths=policy_paths, + acp=acp, + ) return serve(call_response, events) if streaming: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index bf90806e66..4ec0f307c4 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -818,8 +818,10 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: # box is not idle, so no snapshot would match this turn's prefix. return assert self._harness_session is not None - # Copied before the first await: the reply may commit and run its hooks while - # the harness answers, and the checkpoint must hold the state before them. + # Read before the first await: the reply may commit and run its hooks while + # the harness answers, and the checkpoint must describe the idle box. + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + tail = list(turn.tail) state = copy.deepcopy(self.trace.state) info = copy.deepcopy(self.trace.info) pins = dict(self._session.pinned_tool_results) @@ -847,11 +849,10 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: ) return None checkpoint_id = snapshot.id - node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None checkpoint = Checkpoint( id=checkpoint_id, node=node, - tail=list(turn.tail), + tail=tail, state=state, info=info, pins=pins, From 03a52bd89f9d5bcb859b8cd95685478f34914659 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:06:58 +0000 Subject: [PATCH 04/11] add the checkpoint restore integration test back Restore a rollout from a snapshot end to end on prime: per harness, at the default interval, for rlm with skills, and for each retention policy. The cases run once per SnapshotType, on every harness and runtime that declare SUPPORTS__SNAPSHOTS, so a new snapshot type gets its cases without a test change. CI runs it as "Checkpoint Restore". Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests.yml | 2 + .../fixtures/checkpoint_restore_v1.py | 133 +++++++++++++ tests/integration/fixtures/pyproject.toml | 1 + tests/integration/grid.py | 19 +- tests/integration/test_checkpoint.py | 186 ++++++++++++++++++ uv.lock | 9 +- 6 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 tests/integration/fixtures/checkpoint_restore_v1.py create mode 100644 tests/integration/test_checkpoint.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 067a4883e7..19409ffe18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,6 +97,8 @@ jobs: tests: tests/integration/test_user_echo.py - behavior: Resume Tool tests: tests/integration/test_resume_tool.py + - behavior: Checkpoint Restore + tests: tests/integration/test_checkpoint.py - behavior: User Sim tests: tests/integration/test_user_sim.py - behavior: Agent User diff --git a/tests/integration/fixtures/checkpoint_restore_v1.py b/tests/integration/fixtures/checkpoint_restore_v1.py new file mode 100644 index 0000000000..addfaf2b2e --- /dev/null +++ b/tests/integration/fixtures/checkpoint_restore_v1.py @@ -0,0 +1,133 @@ +"""checkpoint-restore: an agentic task whose first attempt dies mid-rollout, so the retry +restores the box's newest snapshot instead of starting over. + +`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model +for two tool calls that build on each other (the second needs the first's token, so they +take two turns), and the reward reads the second one's file back out of the box. The +tools are colocated: they run in the box, so their files are in its snapshots, and any +harness with MCP tools can drive them. A request `@intercept` raises a `ProviderError` +once the first attempt has made two turns — after the snapshot of turn 2 exists. The +restored attempt continues from that snapshot in a new harness process: it never sees +the injected failure again (it fires once per task), finds the earlier calls' files, and +finishes. Needs a harness and a runtime that both support the snapshot type +(`SUPPORTS__SNAPSHOTS`; today prime's filesystem snapshots). +""" + +import shutil +import uuid +from pathlib import Path + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +MARKER = "marker.txt" +STEP_ONE = "step1.txt" +TOKEN = "token.txt" +DONE = "done.txt" +FAIL_AFTER_TURNS = 2 + + +class RestoreState(vf.State): + workdir: str = "" + """The box's working directory, where `setup` put the marker.""" + + +class StepsToolset(vf.Toolset[vf.ToolsetConfig, RestoreState]): + TOOL_PREFIX = "steps" + + @vf.tool + def copy_marker(self) -> str: + """Copy the marker file to the step-one file. Returns the token `write_done` + needs.""" + workdir = Path(self.state.workdir) + shutil.copyfile(workdir / MARKER, workdir / STEP_ONE) + token = uuid.uuid4().hex[:8] + (workdir / TOKEN).write_text(token) + return f"copied; token={token}" + + @vf.tool + def write_done(self, token: str) -> str: + """Write the step-one file's content to the done file. Needs the token that + `copy_marker` returned.""" + workdir = Path(self.state.workdir) + if not (workdir / TOKEN).is_file() or token != (workdir / TOKEN).read_text(): + return "wrong token: call copy_marker first and pass the token it returns" + shutil.copyfile(workdir / STEP_ONE, workdir / DONE) + return "written" + + +class CheckpointRestoreTaskConfig(vf.TaskConfig): + tools: vf.ToolsetConfig = vf.ToolsetConfig(colocated=True) + + +class CheckpointRestoreConfig(vf.TasksetConfig): + task: CheckpointRestoreTaskConfig = CheckpointRestoreTaskConfig() + + +class CheckpointRestoreTask( + vf.Task[vf.TaskData, RestoreState, CheckpointRestoreTaskConfig] +): + failed = False + """Whether the injected failure fired: once per task, so a retry from scratch + (checkpoints off) also gets through.""" + + @classmethod + def toolsets(cls, config: CheckpointRestoreTaskConfig) -> list[vf.Toolset]: + return [StepsToolset(config.tools)] + + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.write(MARKER, nonce.encode()) + result = await runtime.run(["pwd"], {}) + trace.state.workdir = result.stdout.strip() + + @vf.intercept + def fail_once(self, request: vf.Request, trace: vf.Trace) -> None: + # Request hooks also judge each proposed tool call after a turn commits, over + # the branch plus an empty probe result; fail only as a real turn goes upstream + # (every tool returns text, so a real result is never empty). + last = request.messages[-1] + probe = isinstance(last, vf.ToolMessage) and not last.content + if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not self.failed: + self.failed = True + # A 4xx: harness SDKs end the turn at once instead of retrying it until the + # rollout's deadline. + raise vf.ProviderError("injected mid-rollout failure", status_code=400) + + @vf.reward(weight=1.0) + async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float: + try: + content = (await runtime.read(DONE)).decode(errors="replace") + except (SandboxError, OSError, ValueError): + return 0.0 + return float(content.strip() == trace.info["nonce"]) + + +class CheckpointRestoreTaskset( + vf.Taskset[CheckpointRestoreTask, CheckpointRestoreConfig] +): + def load(self) -> list[CheckpointRestoreTask]: + return [ + CheckpointRestoreTask( + vf.TaskData( + idx=0, + prompt=( + "Use the tools of the `steps` MCP server. Make one tool call " + "per turn and wait for each result before the next.\n" + "1. Call `copy_marker`.\n" + "2. Call `write_done` with the token `copy_marker` returned.\n" + "3. Reply with the single word: done." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["CheckpointRestoreTaskset"] + + +if __name__ == "__main__": + StepsToolset.run() diff --git a/tests/integration/fixtures/pyproject.toml b/tests/integration/fixtures/pyproject.toml index 917d9ba656..675c238d8b 100644 --- a/tests/integration/fixtures/pyproject.toml +++ b/tests/integration/fixtures/pyproject.toml @@ -15,6 +15,7 @@ build-backend = "hatchling.build" [tool.hatch.build] include = [ "agent_user.py", + "checkpoint_restore_v1.py", "counter_tool.py", "echo_acp_resume.py", "echo_tool.py", diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 2d636ccd32..db0fd9d34e 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,7 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). +`SUPPORTS_RESUME` or a native session). A `-snapshots` need (one per +`SnapshotType`) asks both the harness and the runtime for `SUPPORTS__SNAPSHOTS`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -34,7 +35,7 @@ from pydantic import TypeAdapter import verifiers.v1 as vf -from verifiers.v1.configs.runtime import NetworkPolicyConfig +from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType from verifiers.v1.harness import Harness from verifiers.v1.runtimes import PrimeBackend, RuntimeConfig, backend_cls from verifiers.v1.utils.loaders import harness_class, harness_config_type @@ -65,8 +66,15 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", + **{f"{kind}-snapshots": f"{kind} snapshots" for kind in get_args(SnapshotType)}, } + +def snapshot_flag(kind: str) -> str: + """The flag a harness and a runtime both set for snapshots of `kind`.""" + return f"SUPPORTS_{kind.upper()}_SNAPSHOTS" + + # Harness dialects that cannot run against the CI model yet. # TODO: drop each entry once its fix lands. CI_MODEL_GAPS = { @@ -139,6 +147,9 @@ def supports(runtime: str, feature: str) -> bool: return isinstance(config, NetworkPolicyConfig) if feature == "services": return issubclass(runtime_cls(runtime), PrimeBackend) + if feature.endswith("-snapshots"): + flag = snapshot_flag(feature.removesuffix("-snapshots")) + return getattr(runtime_cls(runtime), flag, False) flag = { "container": "SUPPORTS_TASK_IMAGES", "user": "SUPPORTS_USER_SWITCHING", @@ -173,6 +184,10 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" + for need in needs: + kind = need.removesuffix("-snapshots") + if kind != need and not getattr(cls, snapshot_flag(kind), False): + return f"{name} cannot restore {kind} snapshots" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py new file mode 100644 index 0000000000..85a82e4a8e --- /dev/null +++ b/tests/integration/test_checkpoint.py @@ -0,0 +1,186 @@ +"""A retry restores the box's newest snapshot (checkpoint-restore-v1: the first attempt +dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns +before it, and finishes in a new harness process). + +Every case runs once per snapshot type (`SnapshotType`), on each harness and runtime +that both declare `SUPPORTS__SNAPSHOTS`; the other cells are collected as skips. +A new snapshot type adds its cases without a change here. The retention policies run +on one harness: what they keep or delete does not depend on it.""" + +from collections.abc import Sequence +from typing import get_args + +import pytest +from checkpoint_restore_v1 import FAIL_AFTER_TURNS +from grid import grid +from prime_sandboxes import AsyncSandboxClient +from skill_codeword import SKILL + +from verifiers.v1.configs.runtime import SnapshotType +from verifiers.v1.harness import RESTORE_PROMPT +from verifiers.v1.rollout import kept +from verifiers.v1.types import content_text + +RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} + + +def snapshot_grid( + harnesses: Sequence[str] | None = None, needs: tuple[str, ...] = ("mcp",) +) -> list: + """`(snapshot_type, harness, harness_runtime)` cells: the harness grid once per + snapshot type, each cell needing that type's snapshots.""" + return [ + pytest.param(kind, *cell.values, marks=cell.marks, id=f"{kind}-{cell.id}") + for kind in get_args(SnapshotType) + for cell in grid( + *([harnesses] if harnesses else []), needs=(f"{kind}-snapshots", *needs) + ) + ] + + +def checkpoint_env(snapshot_type: str, **checkpoint) -> dict: + return { + "agent": { + "retries": RETRIES, + "checkpoint": {"type": snapshot_type, **checkpoint}, + } + } + + +async def on_platform(checkpoint: str) -> bool: + """Whether prime still holds `checkpoint`, restorable.""" + async with AsyncSandboxClient() as client: + try: + return (await client.get_checkpoint(checkpoint)).state == "DURABLE" + except Exception: # noqa: BLE001 - a deleted checkpoint is not found + return False + + +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(), + indirect=["harness", "harness_runtime"], +) +async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_runtime): + """The box restores to the state before the turn that failed, the turns up to + there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not + rerun, and the harness is told about the restore — the restored attempt builds on + the earlier tool call's file. A snapshot every turn, so the failure at turn 3 + restores a snapshot with turns behind it.""" + env = checkpoint_env(snapshot_type, interval=1) + trace = await restored_trace(run, harness, harness_runtime, env) + assert trace.num_turns >= FAIL_AFTER_TURNS + + +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(["bash"]), + indirect=["harness", "harness_runtime"], +) +async def test_default_config_restores(run, snapshot_type, harness, harness_runtime): + """With the default interval (every 10th turn), the retry restores the turn-1 + snapshot: the box as `setup` left it, before any tool call.""" + env = checkpoint_env(snapshot_type) + trace = await restored_trace(run, harness, harness_runtime, env) + assert len(trace.checkpoints) >= 1 + + +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(["rlm"], needs=("mcp", "skills")), + indirect=["harness", "harness_runtime"], +) +async def test_rlm_with_skills_restores(run, snapshot_type, harness, harness_runtime): + """rlm keeps its session in a per-run skill home when it has skills: the retry + must load the session from the same home the restored box holds.""" + config = harness.model_copy(update={"skills": [SKILL]}) + env = checkpoint_env(snapshot_type, interval=1) + trace = await restored_trace(run, config, harness_runtime, env) + assert trace.num_turns >= FAIL_AFTER_TURNS + + +async def restored_trace(run, harness, harness_runtime, env): + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=env, + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + # The newest snapshot before the failed turn boots the retry's box. + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.checkpoint + assert restored_from is not None + assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] + assert trace.has_restored + assert trace.attempt == 2 + # The error belongs to the first attempt, in the agent stage. + assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] + # The restored process made no side request of its own (it would start a second + # root), e.g. a model probe on session load. + assert sum(node.parent is None for node in trace.nodes) == 1 + # The restored process got the restore notice as a user turn. + assert any( + node.message.role == "user" + and RESTORE_PROMPT in content_text(node.message.content) + for node in trace.nodes + ) + # Every turn kept has its call; the dropped turns' calls stay, unlinked, so usage + # counts them. + assert [call.node for call in trace.calls if call.node is not None] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + assert any(call.node is None for call in trace.calls) + return trace + + +@pytest.mark.parametrize( + "keep,after", + [("last", "delete"), ("last", "keep"), ("all", "keep"), (2, "keep")], + ids=["keep-last-delete", "keep-last", "keep-all", "keep-every-2nd"], +) +@pytest.mark.parametrize( + "snapshot_type,harness,harness_runtime", + snapshot_grid(["bash"]), + indirect=["harness", "harness_runtime"], +) +async def test_checkpoint_retention( + run, snapshot_type, harness, harness_runtime, keep, after +): + """Snapshots the policy drops are deleted while the rollout runs, without breaking + the newest one: the retry still restores it. When the run is final, `after` deletes + the rest or leaves exactly what `keep` holds.""" + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=checkpoint_env(snapshot_type, interval=1, keep=keep, after=after), + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert trace.has_restored + live = [c.id for c in trace.checkpoints if not c.deleted] + try: + if after == "delete": + assert live == [] + else: + assert set(live) == kept(trace.checkpoints, keep) + if keep == "last": + assert len(live) == 1 + if keep == "all": + # Snapshots still uploading when the run succeeded were never restorable. + assert live == [c.id for c in trace.checkpoints if c.time.end] + for checkpoint in trace.checkpoints: + assert await on_platform(checkpoint.id) == (not checkpoint.deleted) + finally: + async with AsyncSandboxClient() as client: + for checkpoint in live: + await client.delete_checkpoint(checkpoint) diff --git a/uv.lock b/uv.lock index 559e7ac5cd..f52a3a2e23 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 5 +revision = 4 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -31,6 +31,12 @@ renderers = false ruff = "2026-07-28T00:00:00Z" ty = "2026-07-28T00:00:00Z" +[manifest] +constraints = [ + { name = "brotli", specifier = ">=1.2.1" }, + { name = "brotlicffi", specifier = ">=1.2.1.0" }, +] + [[package]] name = "aiofile" version = "3.12.3" @@ -5067,7 +5073,6 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } -default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, From 94cf961682298c37eec17099916688f9bc117624 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:07:06 +0000 Subject: [PATCH 05/11] keep uv.lock as on the branch Co-Authored-By: Claude Opus 5.5 --- uv.lock | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/uv.lock b/uv.lock index f52a3a2e23..559e7ac5cd 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 4 +revision = 5 requires-python = ">=3.11, <3.15" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -31,12 +31,6 @@ renderers = false ruff = "2026-07-28T00:00:00Z" ty = "2026-07-28T00:00:00Z" -[manifest] -constraints = [ - { name = "brotli", specifier = ">=1.2.1" }, - { name = "brotlicffi", specifier = ">=1.2.1.0" }, -] - [[package]] name = "aiofile" version = "3.12.3" @@ -5073,6 +5067,7 @@ wheels = [ [[package]] name = "verifiers" source = { editable = "." } +default-groups = ["dev", "examples"] dependencies = [ { name = "aiohttp" }, { name = "anthropic" }, From e04209188ea0e0e051983b628cbd361a252504c9 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:17:34 +0000 Subject: [PATCH 06/11] drop box-less checkpoints A box-less checkpoint let a harness that runs no code restore from the trace alone on a runtime without snapshots. It served restores in another process, which this PR no longer does; for retries in one run it only saves the few turns of a short chat rollout. Every checkpoint now has a snapshot, and the boxless setting is gone. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_checkpoint.py | 31 ++++++------------ verifiers/v1/agent.py | 10 +++--- verifiers/v1/configs/agent.py | 3 -- verifiers/v1/rollout.py | 59 +++++++++++------------------------ verifiers/v1/trace.py | 11 +++---- 5 files changed, 37 insertions(+), 77 deletions(-) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 414c877189..004a086e22 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -50,7 +50,9 @@ def make_trace() -> vf.Trace: def test_rewind_restores_info_with_state(): trace = make_trace() - checkpoint = Checkpoint(id="ck", node=1, info={"setup": "nonce"}) + checkpoint = Checkpoint( + id="ck", snapshot=vf.Snapshot(id="ck"), node=1, info={"setup": "nonce"} + ) trace.checkpoints.append(checkpoint) trace.info = {"setup": "nonce", "scoring_skipped": "stale"} trace.rewind(checkpoint) @@ -59,7 +61,7 @@ def test_rewind_restores_info_with_state(): def test_rewind_keeps_the_dropped_turns_calls(): trace = make_trace() - checkpoint = Checkpoint(id="ck", node=0) + checkpoint = Checkpoint(id="ck", snapshot=vf.Snapshot(id="ck"), node=0) trace.checkpoints.append(checkpoint) trace.rewind(checkpoint) assert len(trace.nodes) == 1 @@ -90,8 +92,8 @@ def test_agent_time_splits_off_the_checkpoint_hold(): trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) trace.checkpoints += [ - Checkpoint(id="old", captured=90.0, held=3.0), - Checkpoint(id="new", captured=106.0, held=1.5), + Checkpoint(id="old", snapshot=vf.Snapshot(id="old"), captured=90.0, held=3.0), + Checkpoint(id="new", snapshot=vf.Snapshot(id="new"), captured=106.0, held=1.5), ] trace.split_agent_time() assert trace.timing.agent.checkpoint.duration == 1.5 @@ -173,7 +175,6 @@ async def checkpoint_state() -> dict: run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] - run._boxless = False return run, backend @@ -247,7 +248,10 @@ def test_keep_policies(): assert kept(checkpoints, 2) == {"a", "c", "e"} assert kept(checkpoints, 3) == {"a", "d", "e"} # Deleted and not yet durable snapshots are never kept. - pending = [*durable("a", "b", deleted=("a",)), Checkpoint(id="c")] + pending = [ + *durable("a", "b", deleted=("a",)), + Checkpoint(id="c", snapshot=vf.Snapshot(id="c")), + ] assert kept(pending, "all") == {"b"} @@ -263,15 +267,6 @@ async def test_a_durable_snapshot_prunes_the_older_ones(): assert run.deleted_snapshots == {"old1", "old2", "rewound"} -async def test_a_boxless_checkpoint_is_restorable_at_once(): - run, backend = make_rollout() - run._boxless = True - checkpoint = await run._checkpoint(TURN) - assert checkpoint is not None and checkpoint.snapshot is None - assert checkpoint.time.end and backend.taken == 0 - assert run.trace.checkpoints[-1] is checkpoint - - def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: return vf.Agent( vf.AgentConfig( @@ -304,9 +299,3 @@ def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): "a failed rollout retries from the start" ), ] - - -def test_boxless_checkpointing_is_opt_in(): - assert gated("null", vf.SubprocessConfig())._checkpointing() is None - assert gated("null", vf.SubprocessConfig(), boxless=True)._checkpointing() - assert gated("bash", vf.SubprocessConfig(), boxless=True)._checkpointing() is None diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 5fdab5eacc..6ca01e0d3e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -654,9 +654,9 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: return None harness = self.harness backend = backend_cls(self.config.runtime) - if harness.SUPPORTS_FILESYSTEM_SNAPSHOTS and ( - backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - or (config.boxless and not harness.EXECUTES_CODE) + if ( + harness.SUPPORTS_FILESYSTEM_SNAPSHOTS + and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS ): return config key = (harness.config.id, self.config.runtime.type) @@ -760,9 +760,7 @@ async def run( ) trace = run.trace snapshots.update((s.id, s) for s in run.rewound_snapshots) - snapshots.update( - (c.snapshot.id, c.snapshot) for c in trace.checkpoints if c.snapshot - ) + snapshots.update((c.snapshot.id, c.snapshot) for c in trace.checkpoints) deleted.update(run.deleted_snapshots) if target is not None and any( error.stage == "setup" for error in trace.errors[seen:] diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 833262f827..090456a3dc 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -57,9 +57,6 @@ class RolloutCheckpointConfig(BaseConfig): after: Literal["delete", "keep"] = "delete" """What happens to the kept snapshots once the run is final: `"delete"` them, or `"keep"` them for their owner to delete.""" - boxless: bool = False - """Also checkpoint a harness that runs no code (null) on a runtime without - snapshots, from the trace alone.""" class AgentConfig(BaseConfig): diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 4ec0f307c4..6f911a67bf 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -186,8 +186,6 @@ def __init__( self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" self._pruning = asyncio.Lock() - self._boxless = False - """Whether this run's checkpoints are box-less (no `Checkpoint.snapshot`).""" self.rewound_snapshots: list[Snapshot] = [] """The snapshots of the checkpoints the restored trace dropped (taken after the one restored).""" @@ -319,10 +317,9 @@ async def open(self) -> bool: # A restored trace keeps its id; its new box must not collide with the # box an earlier attempt or process left behind. name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" - if self._restore.snapshot is not None: - config = backend_cls(config).restore_config( - config, self._restore.snapshot - ) + config = backend_cls(config).restore_config( + config, self._restore.snapshot + ) self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None @@ -400,13 +397,10 @@ async def open(self) -> bool: finally: # Providers may assign an ID before startup fails; retain it for diagnostics. self.trace.agent.runtime = runtime.info.model_copy() - # A box snapshot needs a runtime that takes them; a harness that keeps - # nothing in its box is restorable from the trace alone (box-less). - self._boxless = not runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS if ( self._checkpointing is not None and self._borrowed_runtime is None - and (not self._boxless or not self.harness.EXECUTES_CODE) + and runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS ): self._session.checkpointer = self._checkpoint if not self.task.scoring_deferred: @@ -433,8 +427,8 @@ async def open(self) -> bool: self.trace.timing.setup.start = now self.trace.notify() # A restored box already holds the task's setup; only the harness's - # (idempotent) install runs again. A box-less restore boots a fresh box. - if self._restore is None or self._restore.snapshot is None: + # (idempotent) install runs again. + if self._restore is None: async with boundary(TaskError, "task setup"): await invoke( self.task.setup, @@ -837,20 +831,13 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: if harness is None: return started = time.time() - snapshot: Snapshot | None = None - if self._boxless: - checkpoint_id = uuid.uuid4().hex - else: - try: - snapshot = await self.runtime.backend.snapshot() - except Exception: - logger.warning( - "snapshot failed (rollout %s)", self.trace.id, exc_info=True - ) - return None - checkpoint_id = snapshot.id + try: + snapshot = await self.runtime.backend.snapshot() + except Exception: + logger.warning("snapshot failed (rollout %s)", self.trace.id, exc_info=True) + return None checkpoint = Checkpoint( - id=checkpoint_id, + id=snapshot.id, node=node, tail=tail, state=state, @@ -862,21 +849,16 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: captured=time.time(), ) self.trace.checkpoints.append(checkpoint) - if self._boxless: - # Nothing to upload: restorable as soon as it is recorded. - checkpoint.time.end = checkpoint.captured - self.trace.notify() - else: - self._durable[checkpoint_id] = asyncio.create_task( - self._make_durable(checkpoint) - ) + self._durable[checkpoint.id] = asyncio.create_task( + self._make_durable(checkpoint) + ) logger.debug( - "checkpoint %s taken: id=%s node=%s", checkpoint_id, self.trace.id, node + "checkpoint %s taken: id=%s node=%s", checkpoint.id, self.trace.id, node ) return checkpoint async def _make_durable(self, checkpoint: Checkpoint) -> None: - assert self.runtime is not None and checkpoint.snapshot is not None + assert self.runtime is not None try: checkpoint.bytes = await self.runtime.backend.wait_snapshot( checkpoint.snapshot @@ -909,10 +891,7 @@ async def _prune(self) -> None: *( c.snapshot for c in self.trace.checkpoints - if c.snapshot is not None - and c.time.end - and not c.deleted - and c.id not in keep + if c.time.end and not c.deleted and c.id not in keep ), *( s @@ -994,7 +973,7 @@ async def abort(self) -> None: *( c.snapshot for c in self.trace.checkpoints - if owned and c.snapshot is not None and not c.deleted + if owned and not c.deleted ), *self.runtime.backend.stray_snapshots, ] diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index f2ee570738..30e072b4b7 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -381,12 +381,9 @@ class Checkpoint(BaseModel): turn that followed. A retry restores it (`Trace.rewind`).""" id: str - """The snapshot's id, or this record's own for a box-less checkpoint.""" - snapshot: Snapshot | None = None - """The box's snapshot (`RuntimeBackend.checkpoint`) a retry boots a new box from. - None for a box-less checkpoint (a harness that keeps nothing in its box, on a runtime - without snapshots): it restores into a fresh box whose task setup runs again, and - the trace and its typed state carry the rest.""" + """The snapshot's id.""" + snapshot: Snapshot + """The box's snapshot (`RuntimeBackend.snapshot`) a retry boots a new box from.""" node: int | None = None """The last committed node whose conversation the box's state follows; None when the snapshot preceded the first turn.""" @@ -1089,7 +1086,7 @@ def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: position = ids.index(checkpoint.id) + 1 rewound = self.checkpoints[position:] del self.checkpoints[position:] - dropped = [c.snapshot for c in rewound if c.snapshot is not None] + dropped = [c.snapshot for c in rewound] self.rewards = {} self.metrics = {} self.dropped = None From fcf1800523f7f99c05eb4079e6f6bc02fdb3bf2e Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 00:19:27 +0000 Subject: [PATCH 07/11] check that a checkpointed run leaks no snapshot Record every snapshot the run's boxes take, across attempts, and check each on the platform once the run is done: deleted, except the ones the retention policy keeps, which stay restorable. Before, only the checkpoints left on the final trace were checked, so a rewound snapshot or one still uploading at the end could leak unnoticed. Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_checkpoint.py | 67 +++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index 85a82e4a8e..c9fce04d80 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -5,7 +5,9 @@ Every case runs once per snapshot type (`SnapshotType`), on each harness and runtime that both declare `SUPPORTS__SNAPSHOTS`; the other cells are collected as skips. A new snapshot type adds its cases without a change here. The retention policies run -on one harness: what they keep or delete does not depend on it.""" +on one harness: what they keep or delete does not depend on it. Every case checks that +the run leaks no snapshot: each one its boxes took is deleted on the platform, except +the ones the policy keeps.""" from collections.abc import Sequence from typing import get_args @@ -19,6 +21,7 @@ from verifiers.v1.configs.runtime import SnapshotType from verifiers.v1.harness import RESTORE_PROMPT from verifiers.v1.rollout import kept +from verifiers.v1.runtimes import PrimeBackend from verifiers.v1.types import content_text RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} @@ -47,13 +50,45 @@ def checkpoint_env(snapshot_type: str, **checkpoint) -> dict: } -async def on_platform(checkpoint: str) -> bool: - """Whether prime still holds `checkpoint`, restorable.""" +@pytest.fixture +def taken(monkeypatch) -> list[str]: + """The ids of every snapshot the run's boxes take, in every attempt: also the + ones a restore rewinds past and the ones that never become restorable.""" + ids: list[str] = [] + snapshot = PrimeBackend.snapshot + + async def recording(self): + result = await snapshot(self) + ids.append(result.id) + return result + + monkeypatch.setattr(PrimeBackend, "snapshot", recording) + return ids + + +async def platform_state(snapshot_id: str) -> str | None: + """The snapshot's state on prime, or None once it is gone.""" async with AsyncSandboxClient() as client: try: - return (await client.get_checkpoint(checkpoint)).state == "DURABLE" - except Exception: # noqa: BLE001 - a deleted checkpoint is not found - return False + return (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + return None + + +async def assert_no_leaks(taken: list[str], kept_ids: set[str] = frozenset()) -> None: + """Every snapshot the run took is deleted, except `kept_ids`, which stay + restorable.""" + assert taken + states = {snapshot_id: await platform_state(snapshot_id) for snapshot_id in taken} + leaked = { + snapshot_id: state + for snapshot_id, state in states.items() + if snapshot_id not in kept_ids and state not in (None, "DELETING") + } + assert leaked == {} + assert { + snapshot_id: states[snapshot_id] for snapshot_id in kept_ids + } == dict.fromkeys(kept_ids, "DURABLE") @pytest.mark.parametrize( @@ -61,7 +96,9 @@ async def on_platform(checkpoint: str) -> bool: snapshot_grid(), indirect=["harness", "harness_runtime"], ) -async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_runtime): +async def test_retry_restores_checkpoint( + run, taken, snapshot_type, harness, harness_runtime +): """The box restores to the state before the turn that failed, the turns up to there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not rerun, and the harness is told about the restore — the restored attempt builds on @@ -70,6 +107,7 @@ async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_ru env = checkpoint_env(snapshot_type, interval=1) trace = await restored_trace(run, harness, harness_runtime, env) assert trace.num_turns >= FAIL_AFTER_TURNS + await assert_no_leaks(taken) @pytest.mark.parametrize( @@ -77,12 +115,15 @@ async def test_retry_restores_checkpoint(run, snapshot_type, harness, harness_ru snapshot_grid(["bash"]), indirect=["harness", "harness_runtime"], ) -async def test_default_config_restores(run, snapshot_type, harness, harness_runtime): +async def test_default_config_restores( + run, taken, snapshot_type, harness, harness_runtime +): """With the default interval (every 10th turn), the retry restores the turn-1 snapshot: the box as `setup` left it, before any tool call.""" env = checkpoint_env(snapshot_type) trace = await restored_trace(run, harness, harness_runtime, env) assert len(trace.checkpoints) >= 1 + await assert_no_leaks(taken) @pytest.mark.parametrize( @@ -90,13 +131,16 @@ async def test_default_config_restores(run, snapshot_type, harness, harness_runt snapshot_grid(["rlm"], needs=("mcp", "skills")), indirect=["harness", "harness_runtime"], ) -async def test_rlm_with_skills_restores(run, snapshot_type, harness, harness_runtime): +async def test_rlm_with_skills_restores( + run, taken, snapshot_type, harness, harness_runtime +): """rlm keeps its session in a per-run skill home when it has skills: the retry must load the session from the same home the restored box holds.""" config = harness.model_copy(update={"skills": [SKILL]}) env = checkpoint_env(snapshot_type, interval=1) trace = await restored_trace(run, config, harness_runtime, env) assert trace.num_turns >= FAIL_AFTER_TURNS + await assert_no_leaks(taken) async def restored_trace(run, harness, harness_runtime, env): @@ -150,7 +194,7 @@ async def restored_trace(run, harness, harness_runtime, env): indirect=["harness", "harness_runtime"], ) async def test_checkpoint_retention( - run, snapshot_type, harness, harness_runtime, keep, after + run, taken, snapshot_type, harness, harness_runtime, keep, after ): """Snapshots the policy drops are deleted while the rollout runs, without breaking the newest one: the retry still restores it. When the run is final, `after` deletes @@ -178,8 +222,7 @@ async def test_checkpoint_retention( if keep == "all": # Snapshots still uploading when the run succeeded were never restorable. assert live == [c.id for c in trace.checkpoints if c.time.end] - for checkpoint in trace.checkpoints: - assert await on_platform(checkpoint.id) == (not checkpoint.deleted) + await assert_no_leaks(taken, set(live)) finally: async with AsyncSandboxClient() as client: for checkpoint in live: From d77caf0fbeaa910b9e8c345e96792f5ff1b4322f Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Sat, 7 Nov 2026 02:49:12 +0000 Subject: [PATCH 08/11] join a cancelled harness turn through cancellation When a rollout is cancelled (an agent timeout, shutdown), its harness turn is cancelled and then joined. A second cancellation during that join left the turn running while teardown began, where it could still write the trace or use the box. Join it shielded, then re-raise. Co-Authored-By: Claude Opus 5.5 --- verifiers/v1/rollout.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index afbeead92e..76213f9a57 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -48,6 +48,7 @@ from verifiers.v1.task import Task, TaskData from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage +from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke from verifiers.v1.utils.format import format_time @@ -781,7 +782,9 @@ async def _turn(self, messages: Messages | None) -> bool: aborted.cancel() if not turn.done(): turn.cancel() - await asyncio.wait({turn}) + # Joined even through another cancellation: a turn left running + # could still write the trace or use the box during teardown. + await run_shielded(asyncio.wait({turn})) if self._session.aborted is not None: if not turn.cancelled(): turn.exception() # the abort decides the outcome, whatever the turn did From 3a581f8cf4c484d55317616ce320988ab2c173e7 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Mon, 9 Nov 2026 23:39:05 +0000 Subject: [PATCH 09/11] snapshot from turn n and drop after, all and type From review: keep the checkpoint config to what a user decides. - interval counts from turn n (the n-th, 2n-th, ...), so a rollout shorter than n turns takes no snapshot; the turn a restore resumes at is not snapshotted again, so keep = "last" no longer deletes the snapshot that just restored - every snapshot is deleted once the run is final: drop `after` - drop keep = "all", which keep = 1 already expresses - drop `type` from the config and SnapshotType from the API: the runtime takes the best snapshot it supports and records its type on the checkpoint (Snapshot.type) Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 8 +- tests/integration/grid.py | 23 ++-- tests/integration/test_checkpoint.py | 172 +++++++++------------------ tests/unit/test_checkpoint.py | 18 ++- verifiers/v1/__init__.py | 3 - verifiers/v1/agent.py | 15 +-- verifiers/v1/configs/agent.py | 28 ++--- verifiers/v1/configs/runtime.py | 4 - verifiers/v1/rollout.py | 29 +++-- verifiers/v1/runtimes/__init__.py | 2 - verifiers/v1/runtimes/base.py | 8 +- verifiers/v1/trace.py | 4 +- 12 files changed, 117 insertions(+), 197 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index fc5f8c0a34..6c4f4cf0ca 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,12 +77,10 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] -type = "filesystem" # what a snapshot captures -interval = 10 # snapshot every 10th model turn -keep = "last" # snapshots kept while the rollout runs: "last", "all", or every k-th -after = "delete" # when the run is final: "delete" the kept snapshots, or "keep" them +interval = 10 # snapshot at the 10th, 20th, ... model turn; shorter rollouts take none +keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest ``` diff --git a/tests/integration/grid.py b/tests/integration/grid.py index db0fd9d34e..12e8a07a91 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,8 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). A `-snapshots` need (one per -`SnapshotType`) asks both the harness and the runtime for `SUPPORTS__SNAPSHOTS`. +`SUPPORTS_RESUME` or a native session). A `snapshots` need asks both the harness and +the runtime for `SUPPORTS_FILESYSTEM_SNAPSHOTS`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -35,7 +35,7 @@ from pydantic import TypeAdapter import verifiers.v1 as vf -from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType +from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.harness import Harness from verifiers.v1.runtimes import PrimeBackend, RuntimeConfig, backend_cls from verifiers.v1.utils.loaders import harness_class, harness_config_type @@ -66,15 +66,10 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", - **{f"{kind}-snapshots": f"{kind} snapshots" for kind in get_args(SnapshotType)}, + "snapshots": "snapshots", } -def snapshot_flag(kind: str) -> str: - """The flag a harness and a runtime both set for snapshots of `kind`.""" - return f"SUPPORTS_{kind.upper()}_SNAPSHOTS" - - # Harness dialects that cannot run against the CI model yet. # TODO: drop each entry once its fix lands. CI_MODEL_GAPS = { @@ -147,14 +142,12 @@ def supports(runtime: str, feature: str) -> bool: return isinstance(config, NetworkPolicyConfig) if feature == "services": return issubclass(runtime_cls(runtime), PrimeBackend) - if feature.endswith("-snapshots"): - flag = snapshot_flag(feature.removesuffix("-snapshots")) - return getattr(runtime_cls(runtime), flag, False) flag = { "container": "SUPPORTS_TASK_IMAGES", "user": "SUPPORTS_USER_SWITCHING", "expose": "SUPPORTS_PORT_EXPOSURE", "compose": "SUPPORTS_COMPOSE", + "snapshots": "SUPPORTS_FILESYSTEM_SNAPSHOTS", }[feature] return getattr(runtime_cls(runtime), flag) @@ -184,10 +177,8 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" - for need in needs: - kind = need.removesuffix("-snapshots") - if kind != need and not getattr(cls, snapshot_flag(kind), False): - return f"{name} cannot restore {kind} snapshots" + if "snapshots" in needs and not cls.SUPPORTS_FILESYSTEM_SNAPSHOTS: + return f"{name} cannot restore a snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index c9fce04d80..e2960a56bb 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -2,15 +2,11 @@ dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns before it, and finishes in a new harness process). -Every case runs once per snapshot type (`SnapshotType`), on each harness and runtime -that both declare `SUPPORTS__SNAPSHOTS`; the other cells are collected as skips. -A new snapshot type adds its cases without a change here. The retention policies run +Every harness that declares `SUPPORTS_FILESYSTEM_SNAPSHOTS` runs it on prime, the one +runtime with snapshots; the others are collected as skips. The retention policies run on one harness: what they keep or delete does not depend on it. Every case checks that -the run leaks no snapshot: each one its boxes took is deleted on the platform, except -the ones the policy keeps.""" - -from collections.abc import Sequence -from typing import get_args +the run leaks no snapshot: each one its boxes took is deleted on the platform once the +run is final.""" import pytest from checkpoint_restore_v1 import FAIL_AFTER_TURNS @@ -18,36 +14,16 @@ from prime_sandboxes import AsyncSandboxClient from skill_codeword import SKILL -from verifiers.v1.configs.runtime import SnapshotType from verifiers.v1.harness import RESTORE_PROMPT -from verifiers.v1.rollout import kept from verifiers.v1.runtimes import PrimeBackend from verifiers.v1.types import content_text RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} +NEEDS = ("snapshots", "mcp") -def snapshot_grid( - harnesses: Sequence[str] | None = None, needs: tuple[str, ...] = ("mcp",) -) -> list: - """`(snapshot_type, harness, harness_runtime)` cells: the harness grid once per - snapshot type, each cell needing that type's snapshots.""" - return [ - pytest.param(kind, *cell.values, marks=cell.marks, id=f"{kind}-{cell.id}") - for kind in get_args(SnapshotType) - for cell in grid( - *([harnesses] if harnesses else []), needs=(f"{kind}-snapshots", *needs) - ) - ] - - -def checkpoint_env(snapshot_type: str, **checkpoint) -> dict: - return { - "agent": { - "retries": RETRIES, - "checkpoint": {"type": snapshot_type, **checkpoint}, - } - } +def checkpoint_env(**checkpoint) -> dict: + return {"agent": {"retries": RETRIES, "checkpoint": checkpoint}} @pytest.fixture @@ -75,80 +51,86 @@ async def platform_state(snapshot_id: str) -> str | None: return None -async def assert_no_leaks(taken: list[str], kept_ids: set[str] = frozenset()) -> None: - """Every snapshot the run took is deleted, except `kept_ids`, which stay - restorable.""" - assert taken +async def assert_no_leaks(taken: list[str]) -> None: + """Every snapshot the run took is deleted.""" states = {snapshot_id: await platform_state(snapshot_id) for snapshot_id in taken} leaked = { snapshot_id: state for snapshot_id, state in states.items() - if snapshot_id not in kept_ids and state not in (None, "DELETING") + if state not in (None, "DELETING") } assert leaked == {} - assert { - snapshot_id: states[snapshot_id] for snapshot_id in kept_ids - } == dict.fromkeys(kept_ids, "DURABLE") -@pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(), - indirect=["harness", "harness_runtime"], -) -async def test_retry_restores_checkpoint( - run, taken, snapshot_type, harness, harness_runtime -): +@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) +async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): """The box restores to the state before the turn that failed, the turns up to there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not rerun, and the harness is told about the restore — the restored attempt builds on the earlier tool call's file. A snapshot every turn, so the failure at turn 3 restores a snapshot with turns behind it.""" - env = checkpoint_env(snapshot_type, interval=1) - trace = await restored_trace(run, harness, harness_runtime, env) + trace = await restored_trace(run, harness, harness_runtime, interval=1) assert trace.num_turns >= FAIL_AFTER_TURNS await assert_no_leaks(taken) @pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(["bash"]), - indirect=["harness", "harness_runtime"], + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True ) -async def test_default_config_restores( - run, taken, snapshot_type, harness, harness_runtime -): - """With the default interval (every 10th turn), the retry restores the turn-1 - snapshot: the box as `setup` left it, before any tool call.""" - env = checkpoint_env(snapshot_type) - trace = await restored_trace(run, harness, harness_runtime, env) - assert len(trace.checkpoints) >= 1 - await assert_no_leaks(taken) +async def test_a_short_rollout_takes_no_snapshot(run, taken, harness, harness_runtime): + """With the default interval (every 10th turn), a rollout that fails at turn 3 + has taken no snapshot: its retry starts over and still succeeds.""" + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=checkpoint_env(), + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + assert trace.attempt == 2 + assert not trace.has_restored + assert trace.checkpoints == [] + assert taken == [] @pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(["rlm"], needs=("mcp", "skills")), - indirect=["harness", "harness_runtime"], + "harness,harness_runtime", + grid(["rlm"], needs=(*NEEDS, "skills")), + indirect=True, ) -async def test_rlm_with_skills_restores( - run, taken, snapshot_type, harness, harness_runtime -): +async def test_rlm_with_skills_restores(run, taken, harness, harness_runtime): """rlm keeps its session in a per-run skill home when it has skills: the retry must load the session from the same home the restored box holds.""" config = harness.model_copy(update={"skills": [SKILL]}) - env = checkpoint_env(snapshot_type, interval=1) - trace = await restored_trace(run, config, harness_runtime, env) + trace = await restored_trace(run, config, harness_runtime, interval=1) assert trace.num_turns >= FAIL_AFTER_TURNS await assert_no_leaks(taken) -async def restored_trace(run, harness, harness_runtime, env): +@pytest.mark.parametrize("keep", ["last", 2], ids=["keep-last", "keep-every-2nd"]) +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_checkpoint_retention(run, taken, harness, harness_runtime, keep): + """Snapshots the policy drops are deleted while the rollout runs, without breaking + the newest one: the retry still restores it. Every snapshot is deleted once the + run is final.""" + trace = await restored_trace(run, harness, harness_runtime, interval=1, keep=keep) + assert all(checkpoint.deleted for checkpoint in trace.checkpoints) + await assert_no_leaks(taken) + + +async def restored_trace(run, harness, harness_runtime, **checkpoint): (episode,) = await run( "checkpoint-restore-v1", harness=harness, runtime=harness_runtime, - env=env, + env=checkpoint_env(**checkpoint), max_turns=12, max_tokens=4096, ) @@ -163,6 +145,8 @@ async def restored_trace(run, harness, harness_runtime, env): assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] assert trace.has_restored assert trace.attempt == 2 + # Each checkpoint records what its snapshot captured. + assert {c.snapshot.type for c in trace.checkpoints} == {"filesystem"} # The error belongs to the first attempt, in the agent stage. assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] # The restored process made no side request of its own (it would start a second @@ -181,49 +165,3 @@ async def restored_trace(run, harness, harness_runtime, env): ] assert any(call.node is None for call in trace.calls) return trace - - -@pytest.mark.parametrize( - "keep,after", - [("last", "delete"), ("last", "keep"), ("all", "keep"), (2, "keep")], - ids=["keep-last-delete", "keep-last", "keep-all", "keep-every-2nd"], -) -@pytest.mark.parametrize( - "snapshot_type,harness,harness_runtime", - snapshot_grid(["bash"]), - indirect=["harness", "harness_runtime"], -) -async def test_checkpoint_retention( - run, taken, snapshot_type, harness, harness_runtime, keep, after -): - """Snapshots the policy drops are deleted while the rollout runs, without breaking - the newest one: the retry still restores it. When the run is final, `after` deletes - the rest or leaves exactly what `keep` holds.""" - (episode,) = await run( - "checkpoint-restore-v1", - harness=harness, - runtime=harness_runtime, - env=checkpoint_env(snapshot_type, interval=1, keep=keep, after=after), - max_turns=12, - max_tokens=4096, - ) - (trace,) = episode.traces - assert trace.ok, [error.message for error in trace.errors] - assert trace.reward == 1.0 - assert trace.has_restored - live = [c.id for c in trace.checkpoints if not c.deleted] - try: - if after == "delete": - assert live == [] - else: - assert set(live) == kept(trace.checkpoints, keep) - if keep == "last": - assert len(live) == 1 - if keep == "all": - # Snapshots still uploading when the run succeeded were never restorable. - assert live == [c.id for c in trace.checkpoints if c.time.end] - await assert_no_leaks(taken, set(live)) - finally: - async with AsyncSandboxClient() as client: - for checkpoint in live: - await client.delete_checkpoint(checkpoint) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index 004a086e22..ebf2398fd4 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -156,7 +156,7 @@ async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: def make_rollout( - interval: int = 1, keep: Literal["all", "last"] | int = "last" + interval: int = 1, keep: Literal["last"] | int = "last" ) -> tuple[Rollout, SlowBackend]: backend = SlowBackend() run = Rollout.__new__(Rollout) @@ -175,6 +175,7 @@ async def checkpoint_state() -> dict: run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] + run._restored_turn = None return run, backend @@ -183,15 +184,22 @@ async def checkpoint_state() -> dict: async def test_checkpoint_interval_skips_turns(): run, backend = make_rollout(interval=2) - assert await run._checkpoint(TURN) is None # turn 2: 1 % 2 + assert await run._checkpoint(TURN) is not None # turn 2: the 2nd run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) run.trace.nodes.append( MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) ) - assert await run._checkpoint(TURN) is not None # turn 3: 2 % 2 + assert await run._checkpoint(TURN) is None # turn 3 assert backend.taken == 1 +async def test_the_restored_turn_is_not_snapshotted_again(): + run, backend = make_rollout(interval=2) + run._restored_turn = run.trace.num_turns + 1 + assert await run._checkpoint(TURN) is None + assert backend.taken == 0 + + async def test_checkpoint_becomes_durable_in_the_background(): run, backend = make_rollout(interval=1) checkpoint = await run._checkpoint(TURN) @@ -244,7 +252,7 @@ def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: def test_keep_policies(): checkpoints = durable("a", "b", "c", "d", "e") assert kept(checkpoints, "last") == {"e"} - assert kept(checkpoints, "all") == {"a", "b", "c", "d", "e"} + assert kept(checkpoints, 1) == {"a", "b", "c", "d", "e"} assert kept(checkpoints, 2) == {"a", "c", "e"} assert kept(checkpoints, 3) == {"a", "d", "e"} # Deleted and not yet durable snapshots are never kept. @@ -252,7 +260,7 @@ def test_keep_policies(): *durable("a", "b", deleted=("a",)), Checkpoint(id="c", snapshot=vf.Snapshot(id="c")), ] - assert kept(pending, "all") == {"b"} + assert kept(pending, 1) == {"b"} async def test_a_durable_snapshot_prunes_the_older_ones(): diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index f71349fef3..b4d9844c0a 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -96,7 +96,6 @@ RuntimeInfo, RuntimeProcess, Snapshot, - SnapshotType, ) from verifiers.v1.runtimes.compose import ComposeSpec from verifiers.v1.runtimes.deployment import Deployment, ServicesSpec @@ -282,7 +281,6 @@ "RuntimeInfo": "verifiers.v1.runtimes", "RuntimeProcess": "verifiers.v1.runtimes", "Snapshot": "verifiers.v1.runtimes", - "SnapshotType": "verifiers.v1.runtimes", "Sampling": "verifiers.v1.configs.sampling", "SamplingConfig": "verifiers.v1.configs.sampling", "SamplingMask": "verifiers.v1.types", @@ -494,7 +492,6 @@ "RuntimeConfig", "RuntimeInfo", "Snapshot", - "SnapshotType", "ProgramResult", "SubprocessConfig", "DockerConfig", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d46400368c..d89e1725d4 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -35,7 +35,7 @@ from verifiers.v1.harness import Harness from verifiers.v1.interception import Interception, InterceptionServer from verifiers.v1.mcp import SharedToolServer -from verifiers.v1.rollout import Rollout, RolloutTimeouts, kept +from verifiers.v1.rollout import Rollout, RolloutTimeouts from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, @@ -819,19 +819,12 @@ async def run( await asyncio.sleep(delay) attempt += 1 finally: - # The run is final (or cancelled): delete what `after` does not keep. Every - # box is gone by now, so the platform refuses none of them. - keep = ( - kept(run.trace.checkpoints, checkpointing.keep) - if run is not None - and checkpointing is not None - and checkpointing.after == "keep" - else set() - ) + # The run is final (or cancelled): delete every snapshot left. Every box + # is gone by now, so the platform refuses none of them. drop = [ snapshot for snapshot_id, snapshot in sorted(snapshots.items()) - if snapshot_id not in deleted and snapshot_id not in keep + if snapshot_id not in deleted ] if drop and run is not None and run.runtime is not None: try: diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 090456a3dc..384a4c96c8 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -17,7 +17,7 @@ from verifiers.v1.configs.deployment import DeploymentConfig from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig from verifiers.v1.configs.retries import RetryConfig -from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig, SnapshotType +from verifiers.v1.configs.runtime import PrimeConfig, RuntimeConfig from verifiers.v1.configs.sampling import SamplingConfig from verifiers.v1.configs.scoring import ScoringConfig from verifiers.v1.utils.generic import merge_defaults @@ -40,23 +40,19 @@ class TimeoutConfig(BaseConfig): class RolloutCheckpointConfig(BaseConfig): - """Rollout checkpointing: snapshot the box at model turns, so a retry continues - from the newest snapshot instead of starting over. Needs a runtime and a harness - that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts over. Processes - the agent left running (background jobs, a REPL) are gone after a restore.""" + """Rollout checkpointing: snapshot the box every n-th model turn, so a retry + continues from the newest snapshot instead of starting over. The runtime takes the + best snapshot it supports, and each checkpoint records its type. Needs a runtime + and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts + over. Every snapshot is deleted once the run is final.""" - type: SnapshotType = "filesystem" - """What a snapshot captures.""" interval: PositiveInt = 10 - """Snapshot at every n-th model turn (the 1st, the 1+n-th, ...); a restore loses - up to n turns.""" - keep: Literal["all", "last"] | PositiveInt = "last" - """Snapshots kept while the rollout runs: the newest (`"last"`), every one - (`"all"`), or every k-th plus the newest (`k`). A failed restore falls back to an - older kept one.""" - after: Literal["delete", "keep"] = "delete" - """What happens to the kept snapshots once the run is final: `"delete"` them, or - `"keep"` them for their owner to delete.""" + """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter + than n turns takes none, and a restore loses fewer than n turns.""" + keep: Literal["last"] | PositiveInt = "last" + """Snapshots kept while the rollout runs: the newest (`"last"`), or every k-th plus + the newest (`k`; 1 keeps every one). A failed restore falls back to an older kept + one.""" class AgentConfig(BaseConfig): diff --git a/verifiers/v1/configs/runtime.py b/verifiers/v1/configs/runtime.py index 716dadb1ce..2ac1fcdc4e 100644 --- a/verifiers/v1/configs/runtime.py +++ b/verifiers/v1/configs/runtime.py @@ -14,10 +14,6 @@ parse_network_rule, ) -SnapshotType = Literal["filesystem"] -"""What a runtime snapshot captures: `filesystem` holds the box's files, not its -processes.""" - class BindMount(BaseConfig): """An existing file or directory on the container engine's host.""" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 76213f9a57..00ef74e7be 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -113,10 +113,10 @@ async def resolve_agent_runtime( return view -def kept(checkpoints: list[Checkpoint], keep: Literal["all", "last"] | int) -> set[str]: +def kept(checkpoints: list[Checkpoint], keep: Literal["last"] | int) -> set[str]: """The ids of the restorable `checkpoints` (durable, not deleted) that the `keep` - policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every one for - `"all"` or every k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" + policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every + k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" live = [ (index, c) for index, c in enumerate(checkpoints) @@ -124,8 +124,6 @@ def kept(checkpoints: list[Checkpoint], keep: Literal["all", "last"] | int) -> s ] if not live: return set() - if keep == "all": - return {c.id for _, c in live} held = {live[-1][1].id} if keep != "last": held.update(c.id for index, c in live if index % keep == 0) @@ -164,8 +162,7 @@ def __init__( model turns go upstream (on a runtime and a harness that support it) for a retry to restore, and deletes the ones its `keep` policy drops as newer ones become restorable; the owner of the retries deletes the rest once the run is - final (`RolloutCheckpointConfig.after`) — this rollout deletes them only when it - aborts.""" + final — this rollout deletes them only when it aborts.""" self._on_score = on_score self.task = task self.harness = harness @@ -192,6 +189,9 @@ def __init__( """The snapshots still on their way to restorable, by checkpoint id.""" self._attempt = attempt self._restore: Checkpoint | None = None + self._restored_turn: int | None = None + """The turn the restored snapshot was taken at, which this run does not + snapshot again.""" self._harness_data: TaskData | None = None self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" @@ -203,6 +203,7 @@ def __init__( self.trace, self._restore = restore self.trace.attempt = attempt self.rewound_snapshots = self.trace.rewind(self._restore) + self._restored_turn = self.trace.num_turns + 1 self._restore.restored = True else: self.trace = Trace( @@ -826,7 +827,11 @@ async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: assert self.runtime is not None assert self._checkpointing is not None - if self.trace.num_turns % self._checkpointing.interval: + turn_number = self.trace.num_turns + 1 + if ( + turn_number % self._checkpointing.interval + or turn_number == self._restored_turn + ): return None if len(self.trace._pending) > 1: # Another request is in flight (parallel sub-agents share the trace): the @@ -986,18 +991,16 @@ async def abort(self) -> None: with contextlib.suppress(Exception): await self.runtime.backend.stop() if self.runtime is not None: - # Snapshots kept `after` the run belong to their owner, an aborted run's too; - # strays were never restorable, so nobody else knows of them. - owned = self._checkpointing is None or self._checkpointing.after == "delete" + # Strays were never restorable, so nobody else knows of them. with contextlib.suppress(Exception): self.record_deleted( await self.runtime.backend.discard_snapshots( [ - *(self.rewound_snapshots if owned else ()), + *self.rewound_snapshots, *( c.snapshot for c in self.trace.checkpoints - if owned and not c.deleted + if not c.deleted ), *self.runtime.backend.stray_snapshots, ] diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 7081299e8b..1acfc71a09 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -29,7 +29,6 @@ RuntimeBackend, RuntimeProcess, Snapshot, - SnapshotType, register, ) from verifiers.v1.runtimes.runtime import Runtime @@ -111,7 +110,6 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "RuntimeInfo", "RuntimeProcess", "Snapshot", - "SnapshotType", "SubprocessBackend", "SubprocessConfig", "SubprocessRuntimeInfo", diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 722df77fd4..71c354fba6 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,12 +13,12 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Literal from pydantic import BaseModel from pydantic_config import BaseConfig -from verifiers.v1.configs.runtime import NetworkPolicyConfig, SnapshotType +from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import SandboxError from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.format import format_time @@ -149,7 +149,9 @@ class Snapshot(BaseModel): id: str """The provider's id for the snapshot.""" - type: SnapshotType = "filesystem" + type: Literal["filesystem"] = "filesystem" + """What the snapshot captures, as the runtime took it: `filesystem` holds the box's + files, not its processes.""" @dataclass diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index 8d374b1391..0ae847959c 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -418,8 +418,8 @@ class Checkpoint(BaseModel): restored: bool = False """Whether a retry restored the run from this snapshot.""" deleted: bool = False - """Whether the snapshot was deleted (`RolloutCheckpointConfig.keep`, `.after`): it - can no longer be restored.""" + """Whether the snapshot was deleted (by `RolloutCheckpointConfig.keep`, or once the + run is final): it can no longer be restored.""" def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: From 8b20e184c37f9cb179cddf95fbcbeb5e24b914c8 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 00:13:19 +0000 Subject: [PATCH 10/11] derive restorable harnesses and step back after a dead restore A harness can continue a rollout in a box restored from a snapshot when a new process of it can take the rollout over. For a harness that resumes on the transcript (SUPPORTS_RESUME), a restore is that same relaunch, so it needs nothing more. An ACP harness keeps its session in a live process and needs its agent to load the session from its files: ACPHarness.LOADS_SESSIONS (claude-code, codex, rlm). This replaces the harness-side SUPPORTS_FILESYSTEM_SNAPSHOTS (Harness.can_restore). browser-use now restores like any harness that resumes on its transcript. A restore whose attempt fails before its first new turn is marked failed like one whose box never booted, so the next retry steps back to an older snapshot or starts over instead of restoring the same snapshot until the retries run out. Co-Authored-By: Claude Opus 5.5 --- docs/agent.md | 2 +- docs/harnesses.md | 2 +- tests/integration/grid.py | 6 +-- tests/integration/test_checkpoint.py | 4 +- tests/unit/test_checkpoint.py | 4 +- tests/unit/test_retry_ids.py | 53 +++++++++++++++++++ verifiers/v1/acp/__init__.py | 11 +++- verifiers/v1/agent.py | 22 +++++--- verifiers/v1/configs/agent.py | 3 +- verifiers/v1/harness.py | 15 +++--- verifiers/v1/harnesses/bash/harness.py | 1 - verifiers/v1/harnesses/claude_code/harness.py | 2 +- verifiers/v1/harnesses/codex/harness.py | 2 +- verifiers/v1/harnesses/null/harness.py | 1 - verifiers/v1/harnesses/rlm/harness.py | 2 +- verifiers/v1/rollout.py | 14 ++--- 16 files changed, 106 insertions(+), 38 deletions(-) diff --git a/docs/agent.md b/docs/agent.md index 6c4f4cf0ca..8aabc1e8aa 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -77,7 +77,7 @@ skip_on_timeout = true `[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. -Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS` (prime; null, bash, rlm, codex, claude-code). Elsewhere a retry starts over. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime that takes snapshots (prime) and a harness whose new process can continue the rollout: one that resumes on the transcript (`SUPPORTS_RESUME`: null, bash, browser-use) or an ACP harness whose agent loads its session from files (`LOADS_SESSIONS`: claude-code, codex, rlm). Elsewhere a retry starts over, as it does when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. ```toml [env.agent.checkpoint] diff --git a/docs/harnesses.md b/docs/harnesses.md index 624e909037..67ad0f0503 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,7 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). -- `SUPPORTS_FILESYSTEM_SNAPSHOTS`: indicates that a new process of the harness can continue a rollout in a box restored from a filesystem snapshot (`agent.checkpoint`). Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. +- `LOADS_SESSIONS` (ACP harnesses): indicates that the pinned agent implements ACP `session/load`, so a new process can continue a rollout in a box restored from a snapshot (`agent.checkpoint`). A harness with `SUPPORTS_RESUME` needs nothing more: a restore relaunches it on the transcript. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 12e8a07a91..1b163aa4bf 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,8 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). A `snapshots` need asks both the harness and -the runtime for `SUPPORTS_FILESYSTEM_SNAPSHOTS`. +`SUPPORTS_RESUME` or a native session). A `snapshots` need asks the runtime for +`SUPPORTS_FILESYSTEM_SNAPSHOTS` and the harness for `Harness.can_restore`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -177,7 +177,7 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" - if "snapshots" in needs and not cls.SUPPORTS_FILESYSTEM_SNAPSHOTS: + if "snapshots" in needs and not cls.can_restore(): return f"{name} cannot restore a snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py index e2960a56bb..e8e872159c 100644 --- a/tests/integration/test_checkpoint.py +++ b/tests/integration/test_checkpoint.py @@ -2,8 +2,8 @@ dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns before it, and finishes in a new harness process). -Every harness that declares `SUPPORTS_FILESYSTEM_SNAPSHOTS` runs it on prime, the one -runtime with snapshots; the others are collected as skips. The retention policies run +Every harness that can continue in a new process (`Harness.can_restore`) runs it on +prime, the one runtime with snapshots; the others are collected as skips. The retention policies run on one harness: what they keep or delete does not depend on it. Every case checks that the run leaks no snapshot: each one its boxes took is deleted on the platform once the run is final.""" diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py index ebf2398fd4..06ede65093 100644 --- a/tests/unit/test_checkpoint.py +++ b/tests/unit/test_checkpoint.py @@ -175,7 +175,7 @@ async def checkpoint_state() -> dict: run._pruning = asyncio.Lock() run.deleted_snapshots = set() run.rewound_snapshots = [] - run._restored_turn = None + run.restored_turns = None return run, backend @@ -195,7 +195,7 @@ async def test_checkpoint_interval_skips_turns(): async def test_the_restored_turn_is_not_snapshotted_again(): run, backend = make_rollout(interval=2) - run._restored_turn = run.trace.num_turns + 1 + run.restored_turns = run.trace.num_turns assert await run._checkpoint(TURN) is None assert backend.taken == 0 diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 45c540ecf8..3e46decd6f 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -6,6 +6,7 @@ import verifiers.v1 as vf import verifiers.v1.agent as agent_module import verifiers.v1.utils.retries as retries_module +from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.env import Env, RunSlot RETRIES = vf.RetryConfig(max_retries=2) @@ -65,6 +66,58 @@ async def close(self) -> vf.Trace: assert [e.type for e in trace.errors] == ["SandboxError", "SandboxError"] +@pytest.mark.asyncio +async def test_a_restore_that_leads_nowhere_is_not_picked_again(monkeypatch): + """Attempt 1 takes a snapshot and fails. Attempt 2 restores it and fails before its + first new turn, as a harness that cannot load its session would every time. The + next retry starts over instead of restoring the same snapshot again.""" + restores: list[str | None] = [] + + class FakeRollout: + runtime = None + + def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): + restores.append(None if restore is None else restore[1].id) + self.rewound_snapshots: list[vf.Snapshot] = [] + self.deleted_snapshots: set[str] = set() + self.restored_turns = None + if restore is None: + self.trace = make_trace(id=trace_id, attempt=attempt) + else: + self.trace = restore[0] + self.trace.attempt = attempt + self.restored_turns = self.trace.num_turns + on_trace(self.trace) + + async def open(self) -> bool: + if self.trace.attempt == 1: + self.trace.checkpoints.append( + vf.Checkpoint( + id="c1", + snapshot=vf.Snapshot(id="c1"), + time=vf.TimeSpan(start=1.0, end=2.0), + ) + ) + if self.trace.attempt < 3: + self.trace.errors.append(FAILURE) + else: + self.trace.ok = True + return False + + async def close(self) -> vf.Trace: + return self.trace + + monkeypatch.setattr(agent_module, "Rollout", FakeRollout) + agent = vf.Agent(vf.AgentConfig(model="test-model", retries=RETRIES)) + monkeypatch.setattr(agent, "_rollout_params", lambda *args: {}) + monkeypatch.setattr(agent, "_checkpointing", lambda: RolloutCheckpointConfig()) + + trace = await agent.run(vf.Task(vf.TaskData(idx=0, prompt=None))) + + assert restores == [None, "c1", None] + assert trace.ok and trace.attempt == 3 + + @pytest.mark.asyncio async def test_episode_retries_keep_the_episode_id(): calls: list[tuple[str, int]] = [] diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index d5fec9315b..22169fbbf9 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -7,7 +7,7 @@ from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeAlias, TypeVar, cast +from typing import Any, ClassVar, TypeAlias, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field @@ -70,6 +70,15 @@ class ACPConfig: class ACPHarness(Harness[ConfigT]): """Harness backed by one live ACP process and native session per rollout.""" + LOADS_SESSIONS: ClassVar[bool] = False + """Whether the pinned agent implements ACP `session/load`, so a new process can + continue a session from its files in the box. An agent that does not advertise + `loadSession` at runtime takes no snapshots either way.""" + + @classmethod + def can_restore(cls) -> bool: + return cls.LOADS_SESSIONS + async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script( ACP_SOURCE, {**self.config.resolved_env, "UV_FROZEN": "false"} diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index d89e1725d4..b3f3b10a2e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -663,17 +663,14 @@ def _checkpointing(self) -> RolloutCheckpointConfig | None: return None harness = self.harness backend = backend_cls(self.config.runtime) - if ( - harness.SUPPORTS_FILESYSTEM_SNAPSHOTS - and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS - ): + if harness.can_restore() and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS: return config key = (harness.config.id, self.config.runtime.type) if key not in _checkpointing_warned: _checkpointing_warned.add(key) reason = ( f"harness {harness.config.id!r} does not support it" - if not harness.SUPPORTS_FILESYSTEM_SNAPSHOTS + if not harness.can_restore() else f"the {self.config.runtime.type!r} runtime takes no snapshots" ) logger.warning( @@ -774,10 +771,19 @@ async def run( (c.snapshot.id, c.snapshot) for c in trace.checkpoints ) deleted.update(run.deleted_snapshots) - if target is not None and any( - error.stage == "setup" for error in trace.errors[seen:] + if ( + target is not None + and any(error.stage == "setup" for error in trace.errors[seen:]) + or ( + not trace.ok + and run.restored_turns is not None + and trace.num_turns <= run.restored_turns + ) ): - # The box never booted from it: the next retry steps back. + # The restore led nowhere: the box never booted from it, or the + # attempt failed before its first new turn (a harness that + # cannot load its session fails the same way every time). The + # next retry steps back to an older snapshot, or starts over. failed_restores.add(target.id) if trace.ok: break diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 384a4c96c8..7a920b279c 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -43,7 +43,8 @@ class RolloutCheckpointConfig(BaseConfig): """Rollout checkpointing: snapshot the box every n-th model turn, so a retry continues from the newest snapshot instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. Needs a runtime - and a harness that set `SUPPORTS_FILESYSTEM_SNAPSHOTS`; elsewhere a retry starts + that takes snapshots (`SUPPORTS_FILESYSTEM_SNAPSHOTS`) and a harness whose new + process can continue the rollout (`Harness.can_restore`); elsewhere a retry starts over. Every snapshot is deleted once the run is final.""" interval: PositiveInt = 10 diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index b91ff74957..84b7c10f46 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -63,13 +63,6 @@ class Harness(ABC, Generic[ConfigT]): SUPPORTS_RESUME: ClassVar[bool] = False """Whether the default `resume()` can relaunch this harness from the accumulated Messages transcript.""" - SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False - """Whether a new process of this harness can continue a rollout in a box restored - from a filesystem checkpoint (`HarnessSession.restore`), so a retry restores the - newest checkpoint instead of starting over (`AgentConfig.checkpoint`). A rollout - checkpoints only when its runtime backend sets `SUPPORTS_FILESYSTEM_SNAPSHOTS` too. Distinct - from `SUPPORTS_RESUME`: a restore never finds a live process, its program - continues from the conversation the trace or its own session files hold.""" EXECUTES_CODE: ClassVar[bool] = True """Whether the program hands the model local execution in the runtime — true for every real harness; the tool-less chat loops (`null`) override to False. Read @@ -85,6 +78,14 @@ class Harness(ABC, Generic[ConfigT]): leaks host state (auth, config, processes) both ways. Only the minimal in-house loops (`bash`, `null`) override to False.""" + @classmethod + def can_restore(cls) -> bool: + """Whether a new process of this harness can continue a rollout in a box + restored from a snapshot (`AgentConfig.checkpoint`): the default relaunches on + the trace's transcript, so a harness that resumes that way (`SUPPORTS_RESUME`) + can.""" + return cls.SUPPORTS_RESUME + def __init__(self, config: ConfigT) -> None: self.config = config diff --git a/verifiers/v1/harnesses/bash/harness.py b/verifiers/v1/harnesses/bash/harness.py index 3a7f2e8b70..8f6d852144 100644 --- a/verifiers/v1/harnesses/bash/harness.py +++ b/verifiers/v1/harnesses/bash/harness.py @@ -52,7 +52,6 @@ class BashHarness(Harness[BashHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index d66d9904b0..f4b7462916 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -42,7 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True + LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index fb50cc6481..3b28c64fcd 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -63,7 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True + LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: diff --git a/verifiers/v1/harnesses/null/harness.py b/verifiers/v1/harnesses/null/harness.py index 252a23118f..503429ddb6 100644 --- a/verifiers/v1/harnesses/null/harness.py +++ b/verifiers/v1/harnesses/null/harness.py @@ -26,7 +26,6 @@ class NullHarness(Harness[NullHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_RESUME = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True SUPPORTS_TOOL_INTERCEPTION = True EXECUTES_CODE = False NEEDS_CONTAINER = False diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 8aac1ca420..61f49228ea 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -142,7 +142,7 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True - SUPPORTS_FILESYSTEM_SNAPSHOTS = True + LOADS_SESSIONS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 00ef74e7be..9d507ef30e 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -189,9 +189,10 @@ def __init__( """The snapshots still on their way to restorable, by checkpoint id.""" self._attempt = attempt self._restore: Checkpoint | None = None - self._restored_turn: int | None = None - """The turn the restored snapshot was taken at, which this run does not - snapshot again.""" + self.restored_turns: int | None = None + """The model turns the restored trace held when this run began (None for a run + that did not restore). The next turn is the one the snapshot was taken at, which + this run does not snapshot again.""" self._harness_data: TaskData | None = None self.deleted_snapshots: set[str] = set() """The snapshots this run deleted.""" @@ -203,7 +204,7 @@ def __init__( self.trace, self._restore = restore self.trace.attempt = attempt self.rewound_snapshots = self.trace.rewind(self._restore) - self._restored_turn = self.trace.num_turns + 1 + self.restored_turns = self.trace.num_turns self._restore.restored = True else: self.trace = Trace( @@ -828,9 +829,8 @@ async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: assert self.runtime is not None assert self._checkpointing is not None turn_number = self.trace.num_turns + 1 - if ( - turn_number % self._checkpointing.interval - or turn_number == self._restored_turn + if turn_number % self._checkpointing.interval or ( + self.restored_turns is not None and turn_number == self.restored_turns + 1 ): return None if len(self.trace._pending) > 1: From 014d6389890559ee012c64767eae05ced8a9c186 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Tue, 10 Nov 2026 00:19:57 +0000 Subject: [PATCH 11/11] fix the dead-restore check's grouping The condition read `target is not None and A or B`, so B ran for runs that restored nothing and read `restored_turns` on them. Group it as `target is not None and (A or B)`. The new test's fake rollout also called the run's `on_trace` hook, which it does not pass. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_retry_ids.py | 1 - verifiers/v1/agent.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 3e46decd6f..3a046037bc 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -87,7 +87,6 @@ def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): self.trace = restore[0] self.trace.attempt = attempt self.restored_turns = self.trace.num_turns - on_trace(self.trace) async def open(self) -> bool: if self.trace.attempt == 1: diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index b3f3b10a2e..df4be8c0c9 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -771,9 +771,8 @@ async def run( (c.snapshot.id, c.snapshot) for c in trace.checkpoints ) deleted.update(run.deleted_snapshots) - if ( - target is not None - and any(error.stage == "setup" for error in trace.errors[seen:]) + if target is not None and ( + any(error.stage == "setup" for error in trace.errors[seen:]) or ( not trace.ok and run.restored_turns is not None