diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 067a4883e7..19409ffe18 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -97,6 +97,8 @@ jobs: tests: tests/integration/test_user_echo.py - behavior: Resume Tool tests: tests/integration/test_resume_tool.py + - behavior: Checkpoint Restore + tests: tests/integration/test_checkpoint.py - behavior: User Sim tests: tests/integration/test_user_sim.py - behavior: Agent User diff --git a/docs/agent.md b/docs/agent.md index db057cc754..8aabc1e8aa 100644 --- a/docs/agent.md +++ b/docs/agent.md @@ -72,3 +72,15 @@ max_turn_tokens = 4096 skip_on_truncated = true skip_on_timeout = true ``` + +## Retries and checkpoints + +`[env.agent.retries]` sets a retry budget per error type: `rules` are checked in order, and the first rule where all fields match sets the budget for that error; unmatched errors use the top-level `max_retries` budget. + +Rollout checkpointing is on by default: a retry restores the newest snapshot of the box and continues the trace from there, instead of starting over. The runtime takes the best snapshot it supports, and each checkpoint records its type. It needs a runtime that takes snapshots (prime) and a harness whose new process can continue the rollout: one that resumes on the transcript (`SUPPORTS_RESUME`: null, bash, browser-use) or an ACP harness whose agent loads its session from files (`LOADS_SESSIONS`: claude-code, codex, rlm). Elsewhere a retry starts over, as it does when a restore fails before the restored attempt's first new turn. Every snapshot is deleted once the run is final. Set `--no-env.agent.checkpoint` to turn it off. + +```toml +[env.agent.checkpoint] +interval = 10 # snapshot at the 10th, 20th, ... model turn; shorter rollouts take none +keep = "last" # snapshots kept while the rollout runs: "last", or every k-th plus the newest +``` diff --git a/docs/harnesses.md b/docs/harnesses.md index 54b1f418a1..67ad0f0503 100644 --- a/docs/harnesses.md +++ b/docs/harnesses.md @@ -75,6 +75,7 @@ Class-level capability flags declare which optional features your harness implem - `APPENDS_SYSTEM_PROMPT`: passes `TaskData.system_prompt` as a separate system message; when `False`, `resolve_prompt` prepends it to the first user message instead. - `SUPPORTS_MCP`: indicates that `launch` wires `mcp_urls` into the program. - `SUPPORTS_RESUME`: indicates that a multi-turn exchange can restart from the conversation so far (programs with native session state should override `resume` or `session` instead). Note that live mid-turn steering requires a `HarnessSession` that implements `steer`; conversation resume support alone is not enough (see [message delivery](agent.md#messages-during-an-active-turn)). +- `LOADS_SESSIONS` (ACP harnesses): indicates that the pinned agent implements ACP `session/load`, so a new process can continue a rollout in a box restored from a snapshot (`agent.checkpoint`). A harness with `SUPPORTS_RESUME` needs nothing more: a restore relaunches it on the transcript. Keep per-rollout state in the box (for example under `/tmp/vf--runs/`), so the snapshot holds it. - `SUPPORTS_SKILLS`: indicates that the harness calls `install_skills` to copy configured skills into the directory where the program discovers them. - `SUPPORTS_TOOL_INTERCEPTION`: indicates that the harness checks tool calls against the rollout's tool gate (`tool_interception_url`) before executing them. diff --git a/tests/integration/fixtures/checkpoint_restore_v1.py b/tests/integration/fixtures/checkpoint_restore_v1.py new file mode 100644 index 0000000000..addfaf2b2e --- /dev/null +++ b/tests/integration/fixtures/checkpoint_restore_v1.py @@ -0,0 +1,133 @@ +"""checkpoint-restore: an agentic task whose first attempt dies mid-rollout, so the retry +restores the box's newest snapshot instead of starting over. + +`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model +for two tool calls that build on each other (the second needs the first's token, so they +take two turns), and the reward reads the second one's file back out of the box. The +tools are colocated: they run in the box, so their files are in its snapshots, and any +harness with MCP tools can drive them. A request `@intercept` raises a `ProviderError` +once the first attempt has made two turns — after the snapshot of turn 2 exists. The +restored attempt continues from that snapshot in a new harness process: it never sees +the injected failure again (it fires once per task), finds the earlier calls' files, and +finishes. Needs a harness and a runtime that both support the snapshot type +(`SUPPORTS__SNAPSHOTS`; today prime's filesystem snapshots). +""" + +import shutil +import uuid +from pathlib import Path + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +MARKER = "marker.txt" +STEP_ONE = "step1.txt" +TOKEN = "token.txt" +DONE = "done.txt" +FAIL_AFTER_TURNS = 2 + + +class RestoreState(vf.State): + workdir: str = "" + """The box's working directory, where `setup` put the marker.""" + + +class StepsToolset(vf.Toolset[vf.ToolsetConfig, RestoreState]): + TOOL_PREFIX = "steps" + + @vf.tool + def copy_marker(self) -> str: + """Copy the marker file to the step-one file. Returns the token `write_done` + needs.""" + workdir = Path(self.state.workdir) + shutil.copyfile(workdir / MARKER, workdir / STEP_ONE) + token = uuid.uuid4().hex[:8] + (workdir / TOKEN).write_text(token) + return f"copied; token={token}" + + @vf.tool + def write_done(self, token: str) -> str: + """Write the step-one file's content to the done file. Needs the token that + `copy_marker` returned.""" + workdir = Path(self.state.workdir) + if not (workdir / TOKEN).is_file() or token != (workdir / TOKEN).read_text(): + return "wrong token: call copy_marker first and pass the token it returns" + shutil.copyfile(workdir / STEP_ONE, workdir / DONE) + return "written" + + +class CheckpointRestoreTaskConfig(vf.TaskConfig): + tools: vf.ToolsetConfig = vf.ToolsetConfig(colocated=True) + + +class CheckpointRestoreConfig(vf.TasksetConfig): + task: CheckpointRestoreTaskConfig = CheckpointRestoreTaskConfig() + + +class CheckpointRestoreTask( + vf.Task[vf.TaskData, RestoreState, CheckpointRestoreTaskConfig] +): + failed = False + """Whether the injected failure fired: once per task, so a retry from scratch + (checkpoints off) also gets through.""" + + @classmethod + def toolsets(cls, config: CheckpointRestoreTaskConfig) -> list[vf.Toolset]: + return [StepsToolset(config.tools)] + + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.write(MARKER, nonce.encode()) + result = await runtime.run(["pwd"], {}) + trace.state.workdir = result.stdout.strip() + + @vf.intercept + def fail_once(self, request: vf.Request, trace: vf.Trace) -> None: + # Request hooks also judge each proposed tool call after a turn commits, over + # the branch plus an empty probe result; fail only as a real turn goes upstream + # (every tool returns text, so a real result is never empty). + last = request.messages[-1] + probe = isinstance(last, vf.ToolMessage) and not last.content + if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not self.failed: + self.failed = True + # A 4xx: harness SDKs end the turn at once instead of retrying it until the + # rollout's deadline. + raise vf.ProviderError("injected mid-rollout failure", status_code=400) + + @vf.reward(weight=1.0) + async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float: + try: + content = (await runtime.read(DONE)).decode(errors="replace") + except (SandboxError, OSError, ValueError): + return 0.0 + return float(content.strip() == trace.info["nonce"]) + + +class CheckpointRestoreTaskset( + vf.Taskset[CheckpointRestoreTask, CheckpointRestoreConfig] +): + def load(self) -> list[CheckpointRestoreTask]: + return [ + CheckpointRestoreTask( + vf.TaskData( + idx=0, + prompt=( + "Use the tools of the `steps` MCP server. Make one tool call " + "per turn and wait for each result before the next.\n" + "1. Call `copy_marker`.\n" + "2. Call `write_done` with the token `copy_marker` returned.\n" + "3. Reply with the single word: done." + ), + ), + self.config.task, + ) + ] + + +__all__ = ["CheckpointRestoreTaskset"] + + +if __name__ == "__main__": + StepsToolset.run() diff --git a/tests/integration/fixtures/pyproject.toml b/tests/integration/fixtures/pyproject.toml index 917d9ba656..675c238d8b 100644 --- a/tests/integration/fixtures/pyproject.toml +++ b/tests/integration/fixtures/pyproject.toml @@ -15,6 +15,7 @@ build-backend = "hatchling.build" [tool.hatch.build] include = [ "agent_user.py", + "checkpoint_restore_v1.py", "counter_tool.py", "echo_acp_resume.py", "echo_tool.py", diff --git a/tests/integration/grid.py b/tests/integration/grid.py index 2d636ccd32..1b163aa4bf 100644 --- a/tests/integration/grid.py +++ b/tests/integration/grid.py @@ -11,7 +11,8 @@ A harness without a capability the behavior needs is collected as a skip with its reason (`Harness.EXECUTES_CODE`, `SUPPORTS_MCP`, `SUPPORTS_SKILLS`, resume support via -`SUPPORTS_RESUME` or a native session). +`SUPPORTS_RESUME` or a native session). A `snapshots` need asks the runtime for +`SUPPORTS_FILESYSTEM_SNAPSHOTS` and the harness for `Harness.can_restore`. A runtime without a feature the behavior needs (`RuntimeBackend.SUPPORTS_*`, or a `NetworkPolicyConfig` for a network policy) skips the cell the same way. That a runtime @@ -65,8 +66,10 @@ "expose": "port exposure", "compose": "Compose services", "services": "native service VMs", + "snapshots": "snapshots", } + # Harness dialects that cannot run against the CI model yet. # TODO: drop each entry once its fix lands. CI_MODEL_GAPS = { @@ -144,6 +147,7 @@ def supports(runtime: str, feature: str) -> bool: "user": "SUPPORTS_USER_SWITCHING", "expose": "SUPPORTS_PORT_EXPOSURE", "compose": "SUPPORTS_COMPOSE", + "snapshots": "SUPPORTS_FILESYSTEM_SNAPSHOTS", }[feature] return getattr(runtime_cls(runtime), flag) @@ -173,6 +177,8 @@ def unsupported( return f"{name} has no native skill support" if "resume" in needs and not resumes(cls): return f"{name} cannot resume an exchange" + if "snapshots" in needs and not cls.can_restore(): + return f"{name} cannot restore a snapshot" runtime_needs = (*(("container",) if cls.NEEDS_CONTAINER else ()), *needs) for feature in runtime_needs: if feature in RUNTIME_FEATURES and not supports(runtime, feature): diff --git a/tests/integration/test_checkpoint.py b/tests/integration/test_checkpoint.py new file mode 100644 index 0000000000..e8e872159c --- /dev/null +++ b/tests/integration/test_checkpoint.py @@ -0,0 +1,167 @@ +"""A retry restores the box's newest snapshot (checkpoint-restore-v1: the first attempt +dies as turn 3 goes upstream; the retry restores the newest snapshot, keeps the turns +before it, and finishes in a new harness process). + +Every harness that can continue in a new process (`Harness.can_restore`) runs it on +prime, the one runtime with snapshots; the others are collected as skips. The retention policies run +on one harness: what they keep or delete does not depend on it. Every case checks that +the run leaks no snapshot: each one its boxes took is deleted on the platform once the +run is final.""" + +import pytest +from checkpoint_restore_v1 import FAIL_AFTER_TURNS +from grid import grid +from prime_sandboxes import AsyncSandboxClient +from skill_codeword import SKILL + +from verifiers.v1.harness import RESTORE_PROMPT +from verifiers.v1.runtimes import PrimeBackend +from verifiers.v1.types import content_text + +RETRIES = {"max_retries": 0, "rules": [{"type": "ProviderError", "max_retries": 1}]} +NEEDS = ("snapshots", "mcp") + + +def checkpoint_env(**checkpoint) -> dict: + return {"agent": {"retries": RETRIES, "checkpoint": checkpoint}} + + +@pytest.fixture +def taken(monkeypatch) -> list[str]: + """The ids of every snapshot the run's boxes take, in every attempt: also the + ones a restore rewinds past and the ones that never become restorable.""" + ids: list[str] = [] + snapshot = PrimeBackend.snapshot + + async def recording(self): + result = await snapshot(self) + ids.append(result.id) + return result + + monkeypatch.setattr(PrimeBackend, "snapshot", recording) + return ids + + +async def platform_state(snapshot_id: str) -> str | None: + """The snapshot's state on prime, or None once it is gone.""" + async with AsyncSandboxClient() as client: + try: + return (await client.get_checkpoint(snapshot_id)).state + except Exception: # noqa: BLE001 - a deleted snapshot is not found + return None + + +async def assert_no_leaks(taken: list[str]) -> None: + """Every snapshot the run took is deleted.""" + states = {snapshot_id: await platform_state(snapshot_id) for snapshot_id in taken} + leaked = { + snapshot_id: state + for snapshot_id, state in states.items() + if state not in (None, "DELETING") + } + assert leaked == {} + + +@pytest.mark.parametrize("harness,harness_runtime", grid(needs=NEEDS), indirect=True) +async def test_retry_restores_checkpoint(run, taken, harness, harness_runtime): + """The box restores to the state before the turn that failed, the turns up to + there stay sampled, the task's `setup` (a nonce the prompt never reveals) is not + rerun, and the harness is told about the restore — the restored attempt builds on + the earlier tool call's file. A snapshot every turn, so the failure at turn 3 + restores a snapshot with turns behind it.""" + trace = await restored_trace(run, harness, harness_runtime, interval=1) + assert trace.num_turns >= FAIL_AFTER_TURNS + await assert_no_leaks(taken) + + +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_a_short_rollout_takes_no_snapshot(run, taken, harness, harness_runtime): + """With the default interval (every 10th turn), a rollout that fails at turn 3 + has taken no snapshot: its retry starts over and still succeeds.""" + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=checkpoint_env(), + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + assert trace.attempt == 2 + assert not trace.has_restored + assert trace.checkpoints == [] + assert taken == [] + + +@pytest.mark.parametrize( + "harness,harness_runtime", + grid(["rlm"], needs=(*NEEDS, "skills")), + indirect=True, +) +async def test_rlm_with_skills_restores(run, taken, harness, harness_runtime): + """rlm keeps its session in a per-run skill home when it has skills: the retry + must load the session from the same home the restored box holds.""" + config = harness.model_copy(update={"skills": [SKILL]}) + trace = await restored_trace(run, config, harness_runtime, interval=1) + assert trace.num_turns >= FAIL_AFTER_TURNS + await assert_no_leaks(taken) + + +@pytest.mark.parametrize("keep", ["last", 2], ids=["keep-last", "keep-every-2nd"]) +@pytest.mark.parametrize( + "harness,harness_runtime", grid(["bash"], needs=NEEDS), indirect=True +) +async def test_checkpoint_retention(run, taken, harness, harness_runtime, keep): + """Snapshots the policy drops are deleted while the rollout runs, without breaking + the newest one: the retry still restores it. Every snapshot is deleted once the + run is final.""" + trace = await restored_trace(run, harness, harness_runtime, interval=1, keep=keep) + assert all(checkpoint.deleted for checkpoint in trace.checkpoints) + await assert_no_leaks(taken) + + +async def restored_trace(run, harness, harness_runtime, **checkpoint): + (episode,) = await run( + "checkpoint-restore-v1", + harness=harness, + runtime=harness_runtime, + env=checkpoint_env(**checkpoint), + max_turns=12, + max_tokens=4096, + ) + (trace,) = episode.traces + assert trace.ok, [error.message for error in trace.errors] + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + # The newest snapshot before the failed turn boots the retry's box. + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.checkpoint + assert restored_from is not None + assert [c.id for c in trace.checkpoints if c.restored] == [restored_from] + assert trace.has_restored + assert trace.attempt == 2 + # Each checkpoint records what its snapshot captured. + assert {c.snapshot.type for c in trace.checkpoints} == {"filesystem"} + # The error belongs to the first attempt, in the agent stage. + assert [(error.attempt, error.stage) for error in trace.errors] == [(1, "agent")] + # The restored process made no side request of its own (it would start a second + # root), e.g. a model probe on session load. + assert sum(node.parent is None for node in trace.nodes) == 1 + # The restored process got the restore notice as a user turn. + assert any( + node.message.role == "user" + and RESTORE_PROMPT in content_text(node.message.content) + for node in trace.nodes + ) + # Every turn kept has its call; the dropped turns' calls stay, unlinked, so usage + # counts them. + assert [call.node for call in trace.calls if call.node is not None] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + assert any(call.node is None for call in trace.calls) + return trace diff --git a/tests/unit/test_agent.py b/tests/unit/test_agent.py index d9cebddb1f..3e4708edcd 100644 --- a/tests/unit/test_agent.py +++ b/tests/unit/test_agent.py @@ -18,6 +18,7 @@ class FailingSegmentRollout: closed = Rollout.closed fail = Rollout.fail step = Rollout.step + _turn = Rollout._turn timeout = Rollout.timeout @@ -39,6 +40,7 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._opened = True run._closed = False run._failed = False + run._attempt = 0 run._failure = None run._borrowed_runtime = None run.runtime = None @@ -49,6 +51,8 @@ def make_rollout(trace: vf.Trace, session, time_remaining=None): run._session = SimpleNamespace( request_interceptors=[], error=None, + aborted=None, + abort_event=asyncio.Event(), stopped=False, ) run.deadline_at = None @@ -99,3 +103,33 @@ async def turn(self, messages): assert trace.is_timeout and not trace.errors assert trace.stop_condition == "agent_timeout" + + +@pytest.mark.parametrize("outcome", ["keeps waiting", "dies"]) +async def test_request_hook_failure_fails_the_rollout(outcome): + """A request hook's failure is the task's verdict: it ends the segment at once and + fails the rollout, whatever the harness does meanwhile.""" + trace = make_trace() + error = vf.ProviderError("hook refused the turn", status_code=503) + cancelled = [] + + class WaitingSession: + async def turn(self, messages): + run._session.aborted = error + run._session.abort_event.set() + try: + if outcome == "keeps waiting": + await asyncio.sleep(60) + else: + raise RuntimeError("harness died") + except asyncio.CancelledError: + cancelled.append(True) + raise + + run = make_rollout(trace, WaitingSession(), time_remaining=5) + + await asyncio.wait_for(Interaction(run).turn("next"), timeout=1) + + assert [(e.type, e.stage) for e in trace.errors] == [("ProviderError", "agent")] + assert not trace.is_timeout + assert cancelled == ([True] if outcome == "keeps waiting" else []) diff --git a/tests/unit/test_checkpoint.py b/tests/unit/test_checkpoint.py new file mode 100644 index 0000000000..06ede65093 --- /dev/null +++ b/tests/unit/test_checkpoint.py @@ -0,0 +1,309 @@ +"""Rollout checkpointing: when a rollout takes checkpoints, what `rewind` restores, and +what a restored attempt must not redo.""" + +import asyncio +from types import SimpleNamespace +from typing import Literal + +import verifiers.v1 as vf +import verifiers.v1.agent as agent_module +from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.agent import RolloutCheckpointConfig +from verifiers.v1.configs.client import EvalClientConfig +from verifiers.v1.graph import MessageNode +from verifiers.v1.rollout import Rollout, kept +from verifiers.v1.session import RolloutSession +from verifiers.v1.trace import ( + Checkpoint, + ModelCall, + TimeSpan, + TraceTask, +) +from verifiers.v1.types import ( + AssistantMessage, + Request, + ToolCall, + ToolMessage, + UserMessage, +) + + +def make_trace() -> vf.Trace: + trace = vf.Trace( + agent=vf.AgentInfo(config=vf.AgentConfig()), + task=TraceTask(type="TaskData", data=vf.TaskData(idx=0, prompt="q")), + ) + trace.nodes.append(MessageNode(parent=None, message=UserMessage(content="q"))) + trace.nodes.append( + MessageNode( + parent=0, + message=AssistantMessage( + content="", + tool_calls=[ToolCall(id="c1", name="bash", arguments="{}")], + ), + sampled=True, + ) + ) + trace.calls.append(ModelCall(node=1, time=TimeSpan(start=10.0, end=40.0))) + return trace + + +def test_rewind_restores_info_with_state(): + trace = make_trace() + checkpoint = Checkpoint( + id="ck", snapshot=vf.Snapshot(id="ck"), node=1, info={"setup": "nonce"} + ) + trace.checkpoints.append(checkpoint) + trace.info = {"setup": "nonce", "scoring_skipped": "stale"} + trace.rewind(checkpoint) + assert trace.info == {"setup": "nonce"} + + +def test_rewind_keeps_the_dropped_turns_calls(): + trace = make_trace() + checkpoint = Checkpoint(id="ck", snapshot=vf.Snapshot(id="ck"), node=0) + trace.checkpoints.append(checkpoint) + trace.rewind(checkpoint) + assert len(trace.nodes) == 1 + assert [call.node for call in trace.calls] == [None] + + +def test_rewind_drops_the_later_checkpoints_at_the_same_node(): + trace = make_trace() + older = Checkpoint(id="older", node=1, snapshot=vf.Snapshot(id="older")) + target = Checkpoint(id="target", node=1, snapshot=vf.Snapshot(id="target")) + later = Checkpoint(id="later", node=1, snapshot=vf.Snapshot(id="later")) + trace.checkpoints.extend([older, target, later]) + assert trace.rewind(target) == [vf.Snapshot(id="later")] + assert [c.id for c in trace.checkpoints] == ["older", "target"] + + +def test_restored_span_counts_only_its_own_calls(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.split_agent_time() + assert trace.timing.agent.model.duration == 5.0 + assert trace.timing.agent.harness.duration == 15.0 + + +def test_agent_time_splits_off_the_checkpoint_hold(): + trace = make_trace() + trace.timing.agent.start, trace.timing.agent.end = 100.0, 120.0 + trace.calls.append(ModelCall(time=TimeSpan(start=105.0, end=110.0))) + trace.checkpoints += [ + Checkpoint(id="old", snapshot=vf.Snapshot(id="old"), captured=90.0, held=3.0), + Checkpoint(id="new", snapshot=vf.Snapshot(id="new"), captured=106.0, held=1.5), + ] + trace.split_agent_time() + assert trace.timing.agent.checkpoint.duration == 1.5 + assert trace.timing.agent.model.duration == 3.5 + assert trace.timing.agent.harness.duration == 15.0 + + +async def test_restored_tail_is_not_intercepted_twice(): + trace = make_trace() + seen: list[str] = [] + + def tag(request: Request) -> Request: + for message in request.messages: + if isinstance(message, ToolMessage | UserMessage): + seen.append(str(message.content)) + if isinstance(message, ToolMessage): + message.content = f"{message.content} [tagged]" + return request + + session = RolloutSession( + ctx=ModelContext(model="m", client=EvalClientConfig()), + trace=trace, + request_interceptors=[tag], + ) + tail = [ + ToolMessage(tool_call_id="c1", content="out [tagged]"), + UserMessage(content="go on"), + ] + session.mark_intercepted(trace.messages, tail) + request = Request(messages=[*trace.messages, *tail]) + rewritten, records, _ = await session.rewrite_request(request) + assert rewritten.messages == request.messages + assert records == [] + assert seen == [] + + +class SlowBackend: + """Captures at once; becomes durable when the test says so.""" + + def __init__(self) -> None: + self.durable = asyncio.Event() + self.fail = False + self.taken = 0 + self.deleted: list[str] = [] + + async def snapshot(self) -> vf.Snapshot: + self.taken += 1 + return vf.Snapshot(id=f"ck{self.taken}") + + async def wait_snapshot(self, snapshot: vf.Snapshot) -> int | None: + await self.durable.wait() + if self.fail: + raise RuntimeError("upload failed") + return 1024 + + async def discard_snapshots(self, snapshots: list[vf.Snapshot]) -> list[str]: + ids = [snapshot.id for snapshot in snapshots] + self.deleted += ids + return ids + + +def make_rollout( + interval: int = 1, keep: Literal["last"] | int = "last" +) -> tuple[Rollout, SlowBackend]: + backend = SlowBackend() + run = Rollout.__new__(Rollout) + run.trace = make_trace() # one turn so far + run.runtime = SimpleNamespace(backend=backend) # type: ignore[assignment] + + async def checkpoint_state() -> dict: + return {} + + run._harness_session = SimpleNamespace(checkpoint_state=checkpoint_state) # type: ignore[assignment] + run._session = SimpleNamespace(pinned_tool_results={}) # type: ignore[assignment] + run._checkpointing = RolloutCheckpointConfig(interval=interval, keep=keep) + run._durable = {} + run._captures = set() + run._closed = False + run._pruning = asyncio.Lock() + run.deleted_snapshots = set() + run.rewound_snapshots = [] + run.restored_turns = None + return run, backend + + +TURN = SimpleNamespace(prefix_node_ids=[0, 1], tail=[]) + + +async def test_checkpoint_interval_skips_turns(): + run, backend = make_rollout(interval=2) + assert await run._checkpoint(TURN) is not None # turn 2: the 2nd + run.trace.nodes.append(MessageNode(parent=1, message=UserMessage(content="go"))) + run.trace.nodes.append( + MessageNode(parent=2, message=AssistantMessage(content="ok"), sampled=True) + ) + assert await run._checkpoint(TURN) is None # turn 3 + assert backend.taken == 1 + + +async def test_the_restored_turn_is_not_snapshotted_again(): + run, backend = make_rollout(interval=2) + run.restored_turns = run.trace.num_turns + assert await run._checkpoint(TURN) is None + assert backend.taken == 0 + + +async def test_checkpoint_becomes_durable_in_the_background(): + run, backend = make_rollout(interval=1) + checkpoint = await run._checkpoint(TURN) + assert checkpoint is not None and checkpoint.captured + assert checkpoint.time.end == 0 and checkpoint.bytes is None + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert checkpoint.time.end and checkpoint.bytes == 1024 + assert run.trace.checkpoints == [checkpoint] + + +async def test_a_checkpoint_that_never_becomes_durable_is_dropped(): + run, backend = make_rollout(interval=1) + await run._checkpoint(TURN) + backend.fail = True + backend.durable.set() + await run._settle_checkpoints(wait=True) + assert run.trace.checkpoints == [] + + +async def test_settling_without_a_retry_stops_waiting(): + run, _ = make_rollout(interval=1) + checkpoint = await run._checkpoint(TURN) + await asyncio.wait_for(run._settle_checkpoints(wait=False), timeout=1) + assert run._durable == {} + assert run.trace.checkpoints == [checkpoint] and checkpoint.time.end == 0 + + +async def test_a_closing_run_takes_no_checkpoint(): + run, backend = make_rollout(interval=1) + capture = asyncio.create_task(run._checkpoint(TURN)) + run._closed = True + await run._settle_checkpoints(wait=True) + assert await capture is None + assert backend.taken == 0 and run.trace.checkpoints == [] + + +def durable(*ids: str, deleted: tuple[str, ...] = ()) -> list[Checkpoint]: + return [ + Checkpoint( + id=i, + snapshot=vf.Snapshot(id=i), + time=TimeSpan(start=1.0, end=2.0), + deleted=i in deleted, + ) + for i in ids + ] + + +def test_keep_policies(): + checkpoints = durable("a", "b", "c", "d", "e") + assert kept(checkpoints, "last") == {"e"} + assert kept(checkpoints, 1) == {"a", "b", "c", "d", "e"} + assert kept(checkpoints, 2) == {"a", "c", "e"} + assert kept(checkpoints, 3) == {"a", "d", "e"} + # Deleted and not yet durable snapshots are never kept. + pending = [ + *durable("a", "b", deleted=("a",)), + Checkpoint(id="c", snapshot=vf.Snapshot(id="c")), + ] + assert kept(pending, 1) == {"b"} + + +async def test_a_durable_snapshot_prunes_the_older_ones(): + run, backend = make_rollout(keep="last") + run.trace.checkpoints = durable("old1", "old2") + run.rewound_snapshots = [vf.Snapshot(id="rewound")] + backend.durable.set() + newest = await run._checkpoint(TURN) + await run._settle_checkpoints(wait=True) + assert sorted(backend.deleted) == ["old1", "old2", "rewound"] + assert [c.id for c in run.trace.checkpoints if not c.deleted] == [newest.id] + assert run.deleted_snapshots == {"old1", "old2", "rewound"} + + +def gated(harness: str, runtime: vf.RuntimeConfig, **checkpoint) -> vf.Agent: + return vf.Agent( + vf.AgentConfig( + model="test-model", + harness={"id": harness}, + runtime=runtime, + checkpoint=RolloutCheckpointConfig(**checkpoint), + ) + ) + + +def test_checkpointing_is_on_by_default(): + assert vf.AgentConfig().checkpoint == RolloutCheckpointConfig() + assert gated("bash", vf.PrimeConfig())._checkpointing() is not None + + +def test_unsupported_checkpointing_warns_once(caplog, monkeypatch): + monkeypatch.setattr(agent_module, "_checkpointing_warned", set()) + for _ in range(2): + assert gated("bash", vf.SubprocessConfig())._checkpointing() is None + assert gated("terminus-2", vf.PrimeConfig())._checkpointing() is None + warnings = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert warnings == [ + ( + "rollout checkpointing is off: the 'subprocess' runtime takes no " + "snapshots; a failed rollout retries from the start" + ), + ( + "rollout checkpointing is off: harness 'terminus-2' does not support it; " + "a failed rollout retries from the start" + ), + ] diff --git a/tests/unit/test_retry_ids.py b/tests/unit/test_retry_ids.py index 80cb35978e..3a046037bc 100644 --- a/tests/unit/test_retry_ids.py +++ b/tests/unit/test_retry_ids.py @@ -6,6 +6,7 @@ import verifiers.v1 as vf import verifiers.v1.agent as agent_module import verifiers.v1.utils.retries as retries_module +from verifiers.v1.configs.agent import RolloutCheckpointConfig from verifiers.v1.env import Env, RunSlot RETRIES = vf.RetryConfig(max_retries=2) @@ -33,7 +34,11 @@ async def test_agent_retries_keep_the_trace_id(monkeypatch): minted: list[vf.Trace] = [] class FakeRollout: + runtime = None + def __init__(self, *, on_trace, trace_id, attempt, **_): + self.rewound_snapshots: list[str] = [] + self.deleted_snapshots: set[str] = set() self.trace = make_trace(id=trace_id, attempt=attempt) on_trace(self.trace) @@ -61,6 +66,57 @@ async def close(self) -> vf.Trace: assert [e.type for e in trace.errors] == ["SandboxError", "SandboxError"] +@pytest.mark.asyncio +async def test_a_restore_that_leads_nowhere_is_not_picked_again(monkeypatch): + """Attempt 1 takes a snapshot and fails. Attempt 2 restores it and fails before its + first new turn, as a harness that cannot load its session would every time. The + next retry starts over instead of restoring the same snapshot again.""" + restores: list[str | None] = [] + + class FakeRollout: + runtime = None + + def __init__(self, *, on_trace, trace_id, attempt, restore=None, **_): + restores.append(None if restore is None else restore[1].id) + self.rewound_snapshots: list[vf.Snapshot] = [] + self.deleted_snapshots: set[str] = set() + self.restored_turns = None + if restore is None: + self.trace = make_trace(id=trace_id, attempt=attempt) + else: + self.trace = restore[0] + self.trace.attempt = attempt + self.restored_turns = self.trace.num_turns + + async def open(self) -> bool: + if self.trace.attempt == 1: + self.trace.checkpoints.append( + vf.Checkpoint( + id="c1", + snapshot=vf.Snapshot(id="c1"), + time=vf.TimeSpan(start=1.0, end=2.0), + ) + ) + if self.trace.attempt < 3: + self.trace.errors.append(FAILURE) + else: + self.trace.ok = True + return False + + async def close(self) -> vf.Trace: + return self.trace + + monkeypatch.setattr(agent_module, "Rollout", FakeRollout) + agent = vf.Agent(vf.AgentConfig(model="test-model", retries=RETRIES)) + monkeypatch.setattr(agent, "_rollout_params", lambda *args: {}) + monkeypatch.setattr(agent, "_checkpointing", lambda: RolloutCheckpointConfig()) + + trace = await agent.run(vf.Task(vf.TaskData(idx=0, prompt=None))) + + assert restores == [None, "c1", None] + assert trace.ok and trace.attempt == 3 + + @pytest.mark.asyncio async def test_episode_retries_keep_the_episode_id(): calls: list[tuple[str, int]] = [] diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index ecaf8b4d6e..b4d9844c0a 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -95,6 +95,7 @@ Runtime, RuntimeInfo, RuntimeProcess, + Snapshot, ) from verifiers.v1.runtimes.compose import ComposeSpec from verifiers.v1.runtimes.deployment import Deployment, ServicesSpec @@ -119,6 +120,7 @@ AgentInfo, AgentSpan, Branch, + Checkpoint, Error, EvalRunInfo, EvalWorkInfo, @@ -209,6 +211,7 @@ "BaseConfig": "pydantic_config", "BindMount": "verifiers.v1.configs.runtime", "Branch": "verifiers.v1.trace", + "Checkpoint": "verifiers.v1.trace", "Client": "verifiers.v1.clients", "ClientConfig": "verifiers.v1.configs.client", "ComposeSpec": "verifiers.v1.runtimes.compose", @@ -277,6 +280,7 @@ "RuntimeConfig": "verifiers.v1.configs.runtime", "RuntimeInfo": "verifiers.v1.runtimes", "RuntimeProcess": "verifiers.v1.runtimes", + "Snapshot": "verifiers.v1.runtimes", "Sampling": "verifiers.v1.configs.sampling", "SamplingConfig": "verifiers.v1.configs.sampling", "SamplingMask": "verifiers.v1.types", @@ -430,6 +434,7 @@ "StateT", "MessageNode", "Branch", + "Checkpoint", "TurnTokens", "SamplingMask", "TopLogprobs", @@ -486,6 +491,7 @@ "RuntimeProcess", "RuntimeConfig", "RuntimeInfo", + "Snapshot", "ProgramResult", "SubprocessConfig", "DockerConfig", diff --git a/verifiers/v1/acp/__init__.py b/verifiers/v1/acp/__init__.py index 4bed677ef5..22169fbbf9 100644 --- a/verifiers/v1/acp/__init__.py +++ b/verifiers/v1/acp/__init__.py @@ -7,14 +7,14 @@ from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path -from typing import Any, TypeAlias, TypeVar, cast +from typing import Any, ClassVar, TypeAlias, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig from verifiers.v1.errors import HarnessError, InterceptionError -from verifiers.v1.harness import Harness, HarnessSession +from verifiers.v1.harness import RESTORE_PROMPT, Harness, HarnessSession from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess from verifiers.v1.semantic import ( ACP_SEMANTIC_EDGES_METADATA_KEY, @@ -22,7 +22,7 @@ ) from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace -from verifiers.v1.types import Messages +from verifiers.v1.types import Messages, UserMessage from verifiers.v1.utils.aio import run_shielded ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text() @@ -70,6 +70,15 @@ class ACPConfig: class ACPHarness(Harness[ConfigT]): """Harness backed by one live ACP process and native session per rollout.""" + LOADS_SESSIONS: ClassVar[bool] = False + """Whether the pinned agent implements ACP `session/load`, so a new process can + continue a session from its files in the box. An agent that does not advertise + `loadSession` at runtime takes no snapshots either way.""" + + @classmethod + def can_restore(cls) -> bool: + return cls.LOADS_SESSIONS + async def setup(self, runtime: Runtime) -> None: await runtime.prepare_uv_script( ACP_SOURCE, {**self.config.resolved_env, "UV_FROZEN": "false"} @@ -237,6 +246,10 @@ def __init__( self._responses: dict[int, asyncio.Future] = {} self._request_id = 0 self._response_task: asyncio.Task | None = None + self._session_id: str | None = None + """The agent's ACP session id, once a checkpoint asked for it.""" + self._restore_id: str | None = None + """The session the next process loads from the restored box (`restore`).""" async def _start(self) -> None: self._stderr_tail.clear() @@ -284,6 +297,28 @@ async def _request(self, operation: str, **payload: Any) -> JsonObject: if not future.done(): future.cancel() + async def checkpoint_state(self) -> dict[str, Any] | None: + # The agent keeps its session in files of the box, so a restored box can load + # it in a new process — if the agent advertises `loadSession`. + if self._session_id is None: + if self._process is None: + return None + response = await self._request("session") + result = response.get("result") + if not response.get("ok") or not isinstance(result, dict): + raise RuntimeError(response.get("error") or "ACP session query failed") + if not result.get("load_session") or result.get("session_id") is None: + return None + self._session_id = result["session_id"] + return {"acp_session_id": self._session_id} + + def restore(self, state: dict[str, Any], tail: Messages) -> Messages: + # The agent's own session files hold the conversation, the tail included as + # far as it flushed them; the restored process takes only the notice. + self._restore_id = state["acp_session_id"] + self._session_id = self._restore_id + return [UserMessage(content=RESTORE_PROMPT)] + async def steer(self, message: str, *, message_id: str | None = None) -> dict: if self._closed: raise RuntimeError("ACP session is closed") @@ -326,6 +361,7 @@ async def _run(self, messages: Messages | None) -> ProgramResult: ) config = { "command": self.config.command, + "load_session_id": self._restore_id, "user_contents": user_contents, "mcp_urls": self.mcp_urls, "mcp_headers": self.harness.config.resolve_mcp_headers(self.mcp_urls), @@ -354,6 +390,10 @@ async def _run(self, messages: Messages | None) -> ProgramResult: except BaseException: await run_shielded(self._stop(graceful=False)) raise + finally: + # Only the restored box's first process loads the session; later + # turns continue the live one. + self._restore_id = None turn = _turn_result(response) self.trace.root_reply = turn.reply.strip() self.trace.info["acp_stop_reason"] = turn.stop_reason diff --git a/verifiers/v1/acp/runner.py b/verifiers/v1/acp/runner.py index b19942b5b9..5a7ec185af 100644 --- a/verifiers/v1/acp/runner.py +++ b/verifiers/v1/acp/runner.py @@ -87,6 +87,8 @@ def __init__(self) -> None: self.stop_reason: str | None = None self.response_metadata: dict[str, Any] = {} self.update_metadata: list[dict[str, Any]] = [] + self.loading = False + """Set while `session/load` replays the history, which no turn reports.""" def reset(self) -> None: self.visible_reply = "" @@ -104,6 +106,8 @@ def turn_result(self) -> ACPTurn: ) async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None: + if self.loading: + return metadata = dict(kwargs) if isinstance(field_meta := getattr(update, "field_meta", None), dict): metadata.update(field_meta) @@ -267,18 +271,37 @@ async def start(self, config: dict) -> None: self.supports_steering = (initialized.field_meta or {}).get( "steering", {} ).get("supported") is True - session = await self.connection.new_session( - cwd=os.getcwd(), - mcp_servers=mcp_servers(config), - **config["session_meta"], - ) + load_id = config.get("load_session_id") + if load_id is None: + session = await self.connection.new_session( + cwd=os.getcwd(), + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) + session_id = session.session_id + else: + # A box restored from a checkpoint: the agent's session files are + # here, its process is not. Load the session in this new process. + if not (self.capabilities and self.capabilities.load_session): + raise RuntimeError("ACP agent does not support session/load") + self.client.loading = True + try: + await self.connection.load_session( + cwd=os.getcwd(), + session_id=load_id, + mcp_servers=mcp_servers(config), + **config["session_meta"], + ) + finally: + self.client.loading = False + session_id = load_id except BaseException: with suppress(BaseException): await self.stack.aclose() self._reset() raise - self.session_id = session.session_id - self.is_new = True + self.session_id = session_id + self.is_new = load_id is None async def run(self, config: dict) -> ACPTurn: gate = config.get("tool_interception") @@ -381,6 +404,13 @@ async def handle(request: dict) -> None: result = await session.steer( request["message"], request.get("message_id") ) + elif operation == "session": + result = { + "session_id": session.session_id, + "load_session": bool( + session.capabilities and session.capabilities.load_session + ), + } elif operation == "shutdown": result = {"response_metadata": await session.close()} else: diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index 3efdf9d229..df4be8c0c9 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -8,12 +8,13 @@ server; un-entered, each run brings its own.""" import asyncio +import logging import uuid from collections.abc import AsyncIterator, Awaitable, Callable, Mapping from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Generic, Literal, Self, cast +from typing import Any, Generic, Literal, Self, cast from weakref import WeakValueDictionary from typing_extensions import TypeVar @@ -24,6 +25,7 @@ ) from verifiers.v1.configs.agent import ( AgentConfig, + RolloutCheckpointConfig, TimeoutConfig, agent_config_fields, resolve_agent, @@ -37,13 +39,14 @@ from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, backend_cls, provision_backend, runtime_is_local, ) from verifiers.v1.session import RolloutLimits from verifiers.v1.task import Task -from verifiers.v1.trace import Trace +from verifiers.v1.trace import Checkpoint, Error, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -69,6 +72,9 @@ logger = get_logger(__name__) +# (harness id, runtime type) pairs already warned that they take no checkpoints. +_checkpointing_warned: set[tuple[str, str]] = set() + # Only nested runs in this scoring attempt participate in its rollback. _scoring_runs: ContextVar[list[tuple["_EpisodeAgent", Trace]] | None] = ContextVar( "scoring_runs", default=None @@ -649,6 +655,31 @@ def _interception_for( return self._server return None + def _checkpointing(self) -> RolloutCheckpointConfig | None: + """The checkpoint config when this harness on this runtime can take and restore + checkpoints; otherwise None, with a warning once per harness and runtime.""" + config = self.config.checkpoint + if config is None: + return None + harness = self.harness + backend = backend_cls(self.config.runtime) + if harness.can_restore() and backend.SUPPORTS_FILESYSTEM_SNAPSHOTS: + return config + key = (harness.config.id, self.config.runtime.type) + if key not in _checkpointing_warned: + _checkpointing_warned.add(key) + reason = ( + f"harness {harness.config.id!r} does not support it" + if not harness.can_restore() + else f"the {self.config.runtime.type!r} runtime takes no snapshots" + ) + logger.warning( + "rollout checkpointing is off: %s; a failed rollout retries from the " + "start", + reason, + ) + return None + def _check_resume_support(self) -> None: # Multi-turn capability is derived: a harness needs transcript replay, a # native resume implementation, or a rollout-scoped session implementation. @@ -683,51 +714,138 @@ async def run( owner, counted in the pairing check; `on_trace` observes the trace the moment it's minted, before any I/O. `collect_artifacts` captures the task's declared artifacts after its finalizer while its container runtime is still - alive. Retries whole while the trace ends with a retryable error - (`config.retries`) — never into a borrowed box; every attempt's trace keeps - the run's id with its own `attempt`, and the final trace keeps earlier - attempts' errors. `on_score(trace, runtime)` runs after task scoring in + alive. Retries while the trace ends with a retryable error (`config.retries`) + — never into a borrowed box. A retry restores the failed attempt's newest + restorable checkpoint when it has one (`config.checkpoint`, on a runtime and a + harness that support it), keeping that trace and its turns up to there; a + checkpoint whose restore failed is skipped, and without one the retry starts + over. Every attempt's trace keeps the run's id with its own `attempt`, and the + final trace keeps every attempt's errors. `on_score(trace, runtime)` runs after task scoring in the selected verifier workspace, inside its timeout and retry boundary.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") retry = self.config.retries retry_state = RetryState(retry) - history: list = [] - trace_id = uuid.uuid4().hex + checkpointing = self._checkpointing() if runtime is None else None + trace_id = uuid.uuid4().hex # one id for the run across its attempts + history: list[Error] = [] # errors of the attempts started over from + trace: Trace | None = None + target: Checkpoint | None = None # the checkpoint the next attempt restores + snapshots: dict[str, Snapshot] = {} # every box snapshot any attempt took + deleted: set[str] = set() + failed_restores: set[str] = set() + run: Rollout | None = None + # The `finally` also covers a cancellation between attempts, when no rollout is + # open to delete them on abort. + budget = retry_state.max_retries + # Built once: every attempt runs the same harness instance, so per-run harness + # state (RLM's skill home) is the one a restored box holds. + params = self._rollout_params(task, runtime, dict(tools or {})) + attempt = 0 with scope(trace=short_id(trace_id), agent=self._log_name): - for attempt in range(retry_state.max_retries + 1): - trace = await self._run_once( - task, - runtime, - tools, - on_trace, - collect_artifacts, - on_score, - trace_id=trace_id, - attempt=attempt + 1, - ) - if attempt == retry_state.max_retries or trace.ok: - break - if runtime is not None: + try: + while True: + restoring = ( + (trace, target) + if trace is not None and target is not None + else None + ) + # This attempt's own errors decide — a restored trace still carries + # the earlier attempts'. + seen = len(restoring[0].errors) if restoring is not None else 0 + run = await self._run_once( + task, + runtime, + params, + on_trace, + collect_artifacts, + on_score, + trace_id=trace_id, + attempt=attempt + 1, + restore=restoring, + checkpoint=checkpointing, + ) + trace = run.trace + snapshots.update((s.id, s) for s in run.rewound_snapshots) + snapshots.update( + (c.snapshot.id, c.snapshot) for c in trace.checkpoints + ) + deleted.update(run.deleted_snapshots) + if target is not None and ( + any(error.stage == "setup" for error in trace.errors[seen:]) + or ( + not trace.ok + and run.restored_turns is not None + and trace.num_turns <= run.restored_turns + ) + ): + # The restore led nowhere: the box never booted from it, or the + # attempt failed before its first new turn (a harness that + # cannot load its session fails the same way every time). The + # next retry steps back to an older snapshot, or starts over. + failed_restores.add(target.id) + if trace.ok: + break + if attempt >= budget: + break + if runtime is not None: + logger.warning( + "Not retrying rollout on a borrowed box - its state is no " + "longer the task's start state" + ) + break + cause = retry_state.next_error(trace.errors[seen:]) + if cause is None: + break + target = next( + ( + c + for c in reversed(trace.checkpoints) + if c.time.end + and not c.deleted + and c.id not in failed_restores + ), + None, + ) + if target is None: + history.extend(trace.errors) + delay = backoff(attempt) logger.warning( - "Not retrying rollout on a borrowed box - its state is no " - "longer the task's start state" + "Retrying rollout in %s (attempt %d/%d) from %s - %s: %s", + format_time(delay), + attempt + 2, + budget + 1, + f"checkpoint {target.id}" + if target is not None + else "the start", + cause.type, + clip(cause.message), ) - break - cause = retry_state.next_error(trace.errors) - if cause is None: - break - history.extend(trace.errors) - delay = backoff(attempt) - logger.warning( - "Retrying rollout in %s (attempt %d/%d) - %s: %s", - format_time(delay), - attempt + 2, - retry_state.max_retries + 1, - cause.type, - clip(cause.message), - ) - await asyncio.sleep(delay) + await asyncio.sleep(delay) + attempt += 1 + finally: + # The run is final (or cancelled): delete every snapshot left. Every box + # is gone by now, so the platform refuses none of them. + drop = [ + snapshot + for snapshot_id, snapshot in sorted(snapshots.items()) + if snapshot_id not in deleted + ] + if drop and run is not None and run.runtime is not None: + try: + run.record_deleted( + await run_shielded( + run.runtime.backend.discard_snapshots(drop) + ) + ) + except Exception as e: + logger.warning( + "Failed to discard snapshots - %s: %s", + type(e).__name__, + e, + exc_info=logger.isEnabledFor(logging.DEBUG), + ) + assert trace is not None and run is not None if history: # The full history rides the final trace either way; success is the # `ok` stamp, never errors-emptiness. @@ -738,15 +856,16 @@ async def _run_once( self, task: Task, runtime: Runtime | None, - shared_tools: Mapping[str, SharedToolServer] | None, + params: dict[str, Any], on_trace: Callable[[Trace], None] | None, collect_artifacts: bool, on_score: Callable[[Trace, Runtime], Awaitable[None]] | None, *, trace_id: str, attempt: int, - ) -> Trace: - params = self._rollout_params(task, runtime, dict(shared_tools or {})) + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, + ) -> Rollout: if ( collect_artifacts and not backend_cls(params["runtime_config"]).SUPPORTS_TASK_IMAGES @@ -761,12 +880,17 @@ async def _run_once( collect_artifacts=collect_artifacts, on_score=on_score, trace_id=trace_id, + restore=restore, + checkpoint=checkpoint, attempt=attempt, **params, ) try: if await run.open(): - await run.step() + if restore is not None: + await run.restore() + else: + await run.step() if run.ok: run.trace.stop("agent_completed") trace = await run.close() @@ -776,7 +900,7 @@ async def _run_once( raise if trace.agent.runtime is not None: trace.agent.runtime.borrowed = runtime is not None - return trace + return run @asynccontextmanager async def interaction( diff --git a/verifiers/v1/configs/agent.py b/verifiers/v1/configs/agent.py index 19d484563d..7a920b279c 100644 --- a/verifiers/v1/configs/agent.py +++ b/verifiers/v1/configs/agent.py @@ -1,10 +1,11 @@ """One env agent's config: who plays the seat, and its per-run caps.""" -from typing import Any +from typing import Any, Literal from pydantic import ( BaseModel, Field, + PositiveInt, SerializationInfo, SerializeAsAny, field_serializer, @@ -38,6 +39,23 @@ class TimeoutConfig(BaseConfig): isolated verifier setup and retries. Finalization has its own budget.""" +class RolloutCheckpointConfig(BaseConfig): + """Rollout checkpointing: snapshot the box every n-th model turn, so a retry + continues from the newest snapshot instead of starting over. The runtime takes the + best snapshot it supports, and each checkpoint records its type. Needs a runtime + that takes snapshots (`SUPPORTS_FILESYSTEM_SNAPSHOTS`) and a harness whose new + process can continue the rollout (`Harness.can_restore`); elsewhere a retry starts + over. Every snapshot is deleted once the run is final.""" + + interval: PositiveInt = 10 + """Snapshot at every n-th model turn (the n-th, the 2n-th, ...): a rollout shorter + than n turns takes none, and a restore loses fewer than n turns.""" + keep: Literal["last"] | PositiveInt = "last" + """Snapshots kept while the rollout runs: the newest (`"last"`), or every k-th plus + the newest (`k`; 1 keeps every one). A failed restore falls back to an older kept + one.""" + + class AgentConfig(BaseConfig): harness: SerializeAsAny[HarnessConfig] | None = None """The agent's program (None = the taskset's default harness).""" @@ -75,6 +93,12 @@ class AgentConfig(BaseConfig): timeout: TimeoutConfig = TimeoutConfig() retries: RetryConfig = RetryConfig() + checkpoint: RolloutCheckpointConfig | None = Field( + default_factory=RolloutCheckpointConfig + ) + """Rollout checkpointing, on by default: snapshot the box as turns go upstream, so + a retry restores the newest snapshot instead of starting over (see + `RolloutCheckpointConfig`). None turns it off.""" network_notice: bool = True """Include the runtime's network restrictions in the agent's prompt.""" diff --git a/verifiers/v1/harness.py b/verifiers/v1/harness.py index 907089a388..84b7c10f46 100644 --- a/verifiers/v1/harness.py +++ b/verifiers/v1/harness.py @@ -4,7 +4,7 @@ import os from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import DialectName, HarnessConfig, skill_destination @@ -17,7 +17,7 @@ ) from verifiers.v1.runtimes import ProgramResult, Runtime from verifiers.v1.task import TaskData -from verifiers.v1.types import Message, Messages +from verifiers.v1.types import Message, Messages, UserMessage from verifiers.v1.utils.decorators import ( discover_decorated, invoke_all, @@ -40,6 +40,16 @@ ConfigT = TypeVar("ConfigT", bound=HarnessConfig) +RESTORE_PROMPT = ( + "Your workspace was restored from a filesystem checkpoint taken before your last " + "model turn, and you are continuing in a new process. Files are as they were at " + "the checkpoint. Background processes, running shells, sub-agents, and in-memory " + "state such as REPL variables from before the restore are gone; start again " + "whatever you still need. Your last action may not have taken effect, so check " + "before you rely on it. Continue the task." +) +"""The user turn a harness gets after a restore (`HarnessSession.restore`).""" + class Harness(ABC, Generic[ConfigT]): APPENDS_SYSTEM_PROMPT: ClassVar[bool] = False @@ -68,6 +78,14 @@ class Harness(ABC, Generic[ConfigT]): leaks host state (auth, config, processes) both ways. Only the minimal in-house loops (`bash`, `null`) override to False.""" + @classmethod + def can_restore(cls) -> bool: + """Whether a new process of this harness can continue a rollout in a box + restored from a snapshot (`AgentConfig.checkpoint`): the default relaunches on + the trace's transcript, so a harness that resumes that way (`SUPPORTS_RESUME`) + can.""" + return cls.SUPPORTS_RESUME + def __init__(self, config: ConfigT) -> None: self.config = config @@ -410,6 +428,21 @@ async def _run(self, messages: Messages | None) -> ProgramResult: self.tool_interception_url, ) + async def checkpoint_state(self) -> dict[str, Any] | None: + """What `restore` needs to continue in a box restored from a checkpoint taken + now, or None when this session cannot be restored (no checkpoint is taken). + The default relaunches on the trace's transcript and needs nothing.""" + return {} + + def restore(self, state: dict[str, Any], tail: Messages) -> Messages: + """Prepare this new session to continue a rollout in a box restored from a + checkpoint, and return the messages of its first segment. `state` is what + `checkpoint_state` returned when the checkpoint was taken; `tail` is the + checkpoint's uncommitted prompt after the trace's branch. The default + relaunches on the branch plus `tail`, then tells the program about the + restore (`RESTORE_PROMPT`).""" + return [*tail, UserMessage(content=RESTORE_PROMPT)] + async def close(self) -> None: """Close session-owned resources. Idempotent.""" self._closed = True diff --git a/verifiers/v1/harnesses/claude_code/harness.py b/verifiers/v1/harnesses/claude_code/harness.py index 3287be9994..f4b7462916 100644 --- a/verifiers/v1/harnesses/claude_code/harness.py +++ b/verifiers/v1/harnesses/claude_code/harness.py @@ -19,7 +19,7 @@ ACP_VERSION = "0.99.1" CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp" -CLAUDE_CONFIG_ROOT = ".vf-claude" +STATE_ROOT = "/tmp/vf-claude-code-runs" ACP_INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -42,6 +42,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True + LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True # Claude Code drops whitespace-only text blocks on replay. Closed source: observed @@ -104,6 +105,10 @@ async def prepare_acp( "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"), "ANTHROPIC_API_KEY": secret, "ANTHROPIC_MODEL": ctx.model, + # Claude Code validates a model it has not seen in this process with a + # one-token "Hi" request (e.g. when a restored session re-selects its + # model); a custom model option is trusted without that request. + "ANTHROPIC_CUSTOM_MODEL_OPTION": ctx.model, "CLAUDE_CODE_EXECUTABLE": CLAUDE_BIN.format(**versions), "CLAUDE_CONFIG_DIR": config_dir, "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", @@ -135,4 +140,4 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def config_dir(trace: Trace) -> str: - return f"{CLAUDE_CONFIG_ROOT}/{trace.id}" + return f"{STATE_ROOT}/{trace.id}" diff --git a/verifiers/v1/harnesses/codex/harness.py b/verifiers/v1/harnesses/codex/harness.py index 8fc16e543b..3b28c64fcd 100644 --- a/verifiers/v1/harnesses/codex/harness.py +++ b/verifiers/v1/harnesses/codex/harness.py @@ -22,6 +22,7 @@ CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}" PACKAGES_DIR = f"{CODEX_DIR}/acp" +STATE_ROOT = "/tmp/vf-codex-runs" ACP_VERSION = "2.5.0" CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex" ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp" @@ -62,6 +63,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]): APPENDS_SYSTEM_PROMPT = False # TODO SUPPORTS_MCP = True SUPPORTS_SKILLS = True + LOADS_SESSIONS = True SUPPORTS_TOOL_INTERCEPTION = True def acp_turn_result(self, trace: Trace, result: ACPTurn) -> None: @@ -174,7 +176,7 @@ async def cleanup(self, trace: Trace, runtime: Runtime) -> None: @staticmethod def trace_home(trace: Trace) -> str: - return f"/tmp/vf-codex-home-{trace.id}" + return f"{STATE_ROOT}/{trace.id}" async def build_env( self, diff --git a/verifiers/v1/harnesses/rlm/harness.py b/verifiers/v1/harnesses/rlm/harness.py index 4c6f6fd593..61f49228ea 100644 --- a/verifiers/v1/harnesses/rlm/harness.py +++ b/verifiers/v1/harnesses/rlm/harness.py @@ -31,8 +31,8 @@ BuiltinTool = Literal["bash", "edit", "fetch", "ipython"] RLM_REPO = "github.com/PrimeIntellect-ai/nano-rlm.git" -RLM_CACHE_DIR = "/tmp/vf-rlm" -RLM_STATE_DIR = "/tmp/vf-rlm-state" +RLM_CACHE_DIR = "/var/tmp/vf-rlm" +STATE_ROOT = "/tmp/vf-rlm-runs" RLM_RUNTIME_METADATA_KEY = "ai.prime.rlm/runtime-v1" RLM_SESSION_METADATA_KEY = "ai.prime.rlm/session-v1" @@ -71,7 +71,7 @@ class CompactionConfig(BaseConfig): class RLMHarnessConfig(HarnessConfig): dialect: Literal["chat_completions"] = "chat_completions" version: str = Field( - default="ae1cc01f8293987d18b09286b34d93c0dfbf8dac", min_length=1 + default="a057f58d144384fed6ecfed1dabcf350fe540910", min_length=1 ) """Git ref (branch, tag, or commit) of nano-rlm to install. Must know every field this harness puts on the wire, i.e. be at least the default ref.""" @@ -142,12 +142,18 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]): APPENDS_SYSTEM_PROMPT = True SUPPORTS_MCP = True SUPPORTS_SKILLS = True + LOADS_SESSIONS = True _skills_install_dir: str | None = None async def setup(self, runtime: Runtime) -> None: if self.config.skills: # Editable skill packages and uv's tool environment must belong to this run. - self._skills_install_dir = f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" + # A retry runs the same harness and keeps the directory: it holds + # `RLM_HOME`, whose session a box restored from a checkpoint loads. + if self._skills_install_dir is None: + self._skills_install_dir = ( + f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}" + ) await self.install_skills(runtime, f"{self._skills_home()}/skills") directory = self._install_dir() binary = f"{directory}/bin/rlm" @@ -306,14 +312,14 @@ def acp_close_result(self, trace: Trace, response_metadata: dict[str, Any]) -> N self._consume_snapshot(trace, response_metadata) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: - await runtime.run(["rm", "-rf", f"{RLM_STATE_DIR}/{trace.id}"], {}) + await runtime.run(["rm", "-rf", f"{STATE_ROOT}/{trace.id}"], {}) if self._skills_install_dir is not None: await remove_dir( runtime, self._skills_install_dir, "RLM skill installation" ) def _home(self, trace: Trace) -> str: - return self._skills_home() or f"{RLM_STATE_DIR}/{trace.id}/home" + return self._skills_home() or f"{STATE_ROOT}/{trace.id}/home" def _skills_home(self) -> str | None: """rlm's home for a run that carries skills: they sit in its `skills/`, where diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index 72116bf838..a25cbbf71d 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -32,6 +32,7 @@ from contextlib import asynccontextmanager from dataclasses import dataclass, field from functools import partial +from typing import NoReturn import httpx from aiohttp import web @@ -618,16 +619,30 @@ async def start(self) -> None: ) def _fail( - self, session: RolloutSession, dialect: Dialect, error: RolloutError + self, + session: RolloutSession, + dialect: Dialect, + error: RolloutError, ) -> web.Response: - """Stash a model-turn-adjacent failure (such as a hook raising) so the rollout - re-raises it as the real cause, and report it to the harness as an HTTP error.""" + """Stash a model-turn-adjacent failure so the rollout re-raises it as the real + cause, and report it to the harness as an HTTP error.""" session.error = error logger.warning("Failed turn - %s: %s", type(error).__name__, clip(str(error))) return _error_response( dialect, str(error), status=getattr(error, "status_code", 502) ) + async def _abort(self, session: RolloutSession, error: RolloutError) -> NoReturn: + """End the rollout's segment with a request hook's failure. The harness gets no + reply: the rollout cancels its turn, and releasing the session cancels this + handler.""" + logger.warning( + "rollout %s failed: %s: %s", session.trace.id, type(error).__name__, error + ) + session.abort(error) + await asyncio.get_running_loop().create_future() + raise AssertionError("unreachable") + def mediate_capabilities( self, session: RolloutSession, dialect: Dialect, body: dict ) -> tuple[dict, list[str]]: @@ -871,11 +886,10 @@ async def coalesced( if stopped is None and model_request != original_request: dialect.rewrite_request(body, original_request, model_request) except RolloutError as error: - return self._fail(session, dialect, error) + return await self._abort(session, error) except Exception as error: # noqa: BLE001 - surface task hook failures - return self._fail( + return await self._abort( session, - dialect, TaskError( f"model boundary hook failed: {type(error).__name__}: {error}" ), @@ -933,6 +947,14 @@ async def sample() -> web.Response: node: int | None = None error: Exception | None = None started = time.time() + # The box is idle while the harness awaits this turn, so a snapshot captured + # before the reply returns matches the prompt exactly (tool results in, this + # reply not yet). It runs alongside the model call; the reply waits for it. + capture = ( + asyncio.create_task(session.checkpointer(turn)) + if session.checkpointer is not None + else None + ) try: try: # What actually goes upstream: the native body with the rollout's model + @@ -1024,24 +1046,42 @@ async def sample() -> web.Response: error = e raise finally: - if node is None: - turn.abandon() - # The turn's one per-exchange record: settings, timing, outcome, and - # the error that ended it (if any). - self.record_call( - session, - dialect, - body, - started, - node=node, - finish_reason=call_response.finish_reason - if call_response - else None, - usage=call_response.usage if call_response else None, - error=error, - policy_paths=policy_paths, - acp=acp, - ) + try: + if capture is not None: + held = time.monotonic() + try: + # Shielded: a cancelled exchange leaves the snapshot to + # finish in the background, where the rollout awaits it. + checkpoint = await asyncio.shield(capture) + # A lost snapshot never fails the turn. + except Exception: + logger.warning( + "checkpoint failed: id=%s", + session.trace.id, + exc_info=True, + ) + else: + if checkpoint is not None: + checkpoint.held = time.monotonic() - held + finally: + if node is None: + turn.abandon() + # The turn's one per-exchange record: settings, timing, outcome, + # and the error that ended it (if any). + self.record_call( + session, + dialect, + body, + started, + node=node, + finish_reason=call_response.finish_reason + if call_response + else None, + usage=call_response.usage if call_response else None, + error=error, + policy_paths=policy_paths, + acp=acp, + ) return serve(call_response, events) if streaming: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 9d7da6a56f..9d507ef30e 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -2,15 +2,19 @@ import asyncio import contextlib +import copy import logging import time +import uuid from collections.abc import Awaitable, Callable from contextlib import AsyncExitStack from dataclasses import dataclass +from typing import Literal from urllib.parse import urlsplit +from verifiers.v1 import graph from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.agent import AgentConfig +from verifiers.v1.configs.agent import AgentConfig, RolloutCheckpointConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.configs.scoring import resolve_scoring from verifiers.v1.errors import ( @@ -34,14 +38,17 @@ from verifiers.v1.runtimes import ( Runtime, RuntimeConfig, + Snapshot, + backend_cls, make_backend, ) from verifiers.v1.runtimes.private_network import tool_vms from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary from verifiers.v1.state import state_cls -from verifiers.v1.task import Task -from verifiers.v1.trace import AgentInfo, Trace, TraceTask +from verifiers.v1.task import Task, TaskData +from verifiers.v1.trace import AgentInfo, Checkpoint, TimeSpan, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage +from verifiers.v1.utils.aio import run_shielded from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke from verifiers.v1.utils.format import format_time @@ -106,6 +113,23 @@ async def resolve_agent_runtime( return view +def kept(checkpoints: list[Checkpoint], keep: Literal["last"] | int) -> set[str]: + """The ids of the restorable `checkpoints` (durable, not deleted) that the `keep` + policy holds on to (`RolloutCheckpointConfig.keep`): the newest always, plus every + k-th (by position, the 1st, the 1+k-th, ...) for `k`.""" + live = [ + (index, c) + for index, c in enumerate(checkpoints) + if c.time.end and not c.deleted + ] + if not live: + return set() + held = {live[-1][1].id} + if keep != "last": + held.update(c.id for index, c in live if index % keep == 0) + return held + + class Rollout: """Manages one rollout's lifecycle (open, step, close).""" @@ -128,7 +152,17 @@ def __init__( on_score: Callable[[Trace, Runtime], Awaitable[None]] | None = None, trace_id: str | None = None, attempt: int = 1, + restore: tuple[Trace, Checkpoint] | None = None, + checkpoint: RolloutCheckpointConfig | None = None, ) -> None: + """`attempt` numbers this run among the trace's retries, from 1 (stamped on + the trace and the errors it captures). `restore` continues a failed attempt's trace from one of + its checkpoints: the trace rewinds to it and this run boots its box from the + snapshot instead of setting the task up again. `checkpoint` snapshots the box as + model turns go upstream (on a runtime and a harness that support it) for a + retry to restore, and deletes the ones its `keep` policy drops as newer ones + become restorable; the owner of the retries deletes the rest once the run is + final — this rollout deletes them only when it aborts.""" self._on_score = on_score self.task = task self.harness = harness @@ -148,23 +182,48 @@ def __init__( self.runtime = runtime self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts - self.trace: Trace = Trace( - **({"id": trace_id} if trace_id is not None else {}), - attempt=attempt, - task=TraceTask( - type=type(task).__name__, - data=task.data, - key=task.key, - hash=task.hash, - ), - state=state_cls(type(task))(), - # The seat's resolved config, role overrides included — the agent - # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), - ) + self._checkpointing = checkpoint + self._durable: dict[str, asyncio.Task[None]] = {} + self._captures: set[asyncio.Task] = set() + """Snapshots being taken (`_checkpoint`), for `_settle_checkpoints` to wait for.""" + """The snapshots still on their way to restorable, by checkpoint id.""" + self._attempt = attempt + self._restore: Checkpoint | None = None + self.restored_turns: int | None = None + """The model turns the restored trace held when this run began (None for a run + that did not restore). The next turn is the one the snapshot was taken at, which + this run does not snapshot again.""" + self._harness_data: TaskData | None = None + self.deleted_snapshots: set[str] = set() + """The snapshots this run deleted.""" + self._pruning = asyncio.Lock() + self.rewound_snapshots: list[Snapshot] = [] + """The snapshots of the checkpoints the restored trace dropped (taken after the + one restored).""" + if restore is not None: + self.trace, self._restore = restore + self.trace.attempt = attempt + self.rewound_snapshots = self.trace.rewind(self._restore) + self.restored_turns = self.trace.num_turns + self._restore.restored = True + else: + self.trace = Trace( + **({"id": trace_id} if trace_id is not None else {}), + attempt=attempt, + task=TraceTask( + type=type(task).__name__, + data=task.data, + key=task.key, + hash=task.hash, + ), + state=state_cls(type(task))(), + # The seat's resolved config, role overrides included — the agent + # this trace can be reproduced with. + agent=AgentInfo(config=agent_config), + ) + if on_trace is not None: + on_trace(self.trace) self.trace._replay_view = harness.replay_view - if on_trace is not None: - on_trace(self.trace) interceptors = [ (hook_boundary(fn, allow_trace=False), fn) for fn in discover_decorated(task, "intercept") @@ -226,10 +285,11 @@ def failure(self) -> Exception | None: """The original exception most recently captured onto the trace.""" return self._failure - def fail(self, error: Exception) -> None: + def fail(self, error: Exception, stage: str | None = None) -> None: """Record `error` as this rollout's outcome (captured onto the trace, the remaining stages skipped) — the run's owner reporting a failure the run - itself couldn't see, e.g. its user raising between segments.""" + itself couldn't see, e.g. its user raising between segments. `stage` names + the rollout stage it broke in, when known.""" if ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -246,7 +306,7 @@ def fail(self, error: Exception) -> None: logger.exception("Failed rollout with an unexpected error") self._failed = True self._failure = error - self.trace.record_error(error) + self.trace.record_error(error, attempt=self._attempt, stage=stage) def timeout(self, stage: str) -> None: """Record `stage`'s expired deadline as this rollout's outcome: a stop, not @@ -263,7 +323,16 @@ async def open(self) -> bool: self.trace.timing.boot.start = time.time() self.trace.notify() if self._borrowed_runtime is None: - self.runtime = make_backend(self.runtime_config, name=self.trace.id).bind() + config = self.runtime_config + name = self.trace.id + if self._restore is not None: + # A restored trace keeps its id; its new box must not collide with the + # box an earlier attempt or process left behind. + name = f"{self.trace.id}-{uuid.uuid4().hex[:8]}" + config = backend_cls(config).restore_config( + config, self._restore.snapshot + ) + self.runtime = make_backend(config, name=name).bind() elif ( self._borrowed_runtime is not None and self._borrowed_runtime.backend.stopped @@ -278,11 +347,20 @@ async def open(self) -> bool: runtime = self.runtime assert self.trace.agent is not None # minted with the trace self.trace.agent.runtime = runtime.info.model_copy() - logger.debug( - "Starting rollout harness=%s runtime=%s", - self.harness.config.name, - self.runtime_config.type, - ) + if self._restore is None: + logger.debug( + "Starting rollout harness=%s runtime=%s", + self.harness.config.name, + self.runtime_config.type, + ) + else: + logger.info( + "Restoring rollout from checkpoint %s at turn %d harness=%s runtime=%s", + self._restore.id, + self.trace.num_turns, + self.harness.config.name, + self.runtime_config.type, + ) loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None setup_deadline = ( @@ -327,6 +405,12 @@ async def open(self) -> bool: finally: # Providers may assign an ID before startup fails; retain it for diagnostics. self.trace.agent.runtime = runtime.info.model_copy() + if ( + self._checkpointing is not None + and self._borrowed_runtime is None + and runtime.backend.SUPPORTS_FILESYSTEM_SNAPSHOTS + ): + self._session.checkpointer = self._checkpoint if not self.task.scoring_deferred: isolated = self._scoring_policy.mode == "isolated" self._scoring_task = self.task.grading_task( @@ -350,21 +434,22 @@ async def open(self) -> bool: self.trace.timing.boot.end = now self.trace.timing.setup.start = now self.trace.notify() - started = time.perf_counter() - async with ( - boundary(TaskError, "task setup"), - ): - await invoke( - self.task.setup, - { - "trace": self.trace, - "runtime": runtime, - "deployment": self.deployment, - }, + # A restored box already holds the task's setup; only the harness's + # (idempotent) install runs again. + if self._restore is None: + started = time.perf_counter() + async with boundary(TaskError, "task setup"): + await invoke( + self.task.setup, + { + "trace": self.trace, + "runtime": runtime, + "deployment": self.deployment, + }, + ) + logger.debug( + "Set up task in %s", format_time(time.perf_counter() - started) ) - logger.debug( - "Set up task in %s", format_time(time.perf_counter() - started) - ) started = time.perf_counter() async with ( boundary(HarnessError, "harness setup"), @@ -489,9 +574,11 @@ async def open(self) -> bool: harness_data = harness_data.model_copy( update={"system_prompt": network_prompt} ) + # A restored trace already carries its prompt's rewrites. if ( self._session.request_interceptors and harness_data.prompt is not None + and self._restore is None ): prompt = harness_data.prompt system_prompt = harness_data.system_prompt @@ -542,6 +629,7 @@ async def open(self) -> bool: ) else {} ) + self._harness_data = harness_data self._harness_session = await self.harness.session( self.ctx, self.trace, @@ -559,13 +647,14 @@ async def open(self) -> bool: self.timeout("setup") if not self._deployment_started: self.fail( - SandboxError("deployment provisioning exceeded setup timeout") + SandboxError("deployment provisioning exceeded setup timeout"), + stage="setup", ) else: - self.fail(e) + self.fail(e, stage="setup") return False except Exception as e: # noqa: BLE001 - setup boundary records every rollout failure - self.fail(e) + self.fail(e, stage="setup") return False except BaseException: # A cancellation mid-setup kills the driver's await with it, so no @@ -591,15 +680,18 @@ async def step( self, messages: Messages | None = None, *, + intercepted: int = 0, on_input: Callable[[], None] | None = None, ) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - Returns whether the exchange can continue — a refused turn (limit, @stop), - a failure (an expired agent timeout included), or a segment that made no - progress all end it.""" + The first `intercepted` messages pass to the harness as they are — they were + intercepted when the trace first saw them (a restored checkpoint's tail). + Returns whether the exchange can continue — a refused turn (limit, + @stop), a failure (an expired agent timeout included), or a segment that + made no progress all end it.""" if not self._opened or self._closed or not self.ok: return False trace = self.trace @@ -618,25 +710,32 @@ async def step( try: async with deadline: assert self._harness_session is not None - if messages is not None and self._session.request_interceptors: + if ( + messages is not None + and len(messages) > intercepted + and self._session.request_interceptors + ): prepared, rewrites = await self._session.prepare_users( - Request(messages=messages) + Request(messages=messages[intercepted:]) ) - messages = prepared.messages + messages = [*messages[:intercepted], *prepared.messages] self.trace.request_rewrites.extend(rewrites) if self._session.stopped: return False self._session.on_turn_input = on_input assert self.deployment is not None - await self._harness_session.turn(messages) + if await self._turn(messages): + assert self._session.aborted is not None + self.fail(self._session.aborted, stage="agent") + return False except TimeoutError as e: # An expired rollout deadline is the agent breaking its time budget: a - # timeout stop. A TimeoutError from the harness's own I/O with no - # expired deadline stays the raw failure. + # timeout stop. A TimeoutError from the harness's own I/O with no expired + # deadline stays the raw failure. if self.deadline_at is not None and (loop.time() >= self.deadline_at): self.timeout("agent") else: - self.fail(e) + self.fail(e, stage="agent") return False except Exception as e: # noqa: BLE001 - harness boundary records every rollout failure if self._session.stopped: @@ -650,9 +749,9 @@ async def step( real = self._session.error if real is not None and isinstance(e, RolloutError): real.__cause__ = e - self.fail(real) + self.fail(real, stage="agent") else: - self.fail(e) + self.fail(e, stage="agent") return False finally: self._session.on_turn_input = None @@ -672,6 +771,186 @@ async def step( # never moved, forever. return self.ok and trace.num_turns > turns_before + async def _turn(self, messages: Messages | None) -> bool: + """Run one harness segment. A request hook that fails ends it at once (the + session aborts): the harness's turn is cancelled, so the harness never handles + the failure. Returns whether the segment was aborted.""" + assert self._harness_session is not None + turn = asyncio.ensure_future(self._harness_session.turn(messages)) + aborted = asyncio.ensure_future(self._session.abort_event.wait()) + try: + await asyncio.wait({turn, aborted}, return_when=asyncio.FIRST_COMPLETED) + finally: + aborted.cancel() + if not turn.done(): + turn.cancel() + # Joined even through another cancellation: a turn left running + # could still write the trace or use the box during teardown. + await run_shielded(asyncio.wait({turn})) + if self._session.aborted is not None: + if not turn.cancelled(): + turn.exception() # the abort decides the outcome, whatever the turn did + return True + turn.result() + return False + + async def restore(self) -> bool: + """Run the restored attempt's first segment: a new harness process in the box + restored from the checkpoint continues the conversation the checkpoint left + off at (`HarnessSession.restore`), told that the box was restored.""" + checkpoint = self._restore + assert checkpoint is not None and self._harness_data is not None + assert self._harness_session is not None + # The relaunch re-emits the session's system prompt (the task's, plus a + # network notice) through the program's own system message; the tail's copy + # would double it. + system = self._harness_data.system_prompt + tail = [m for m in checkpoint.tail if m.role != "system" or system is None] + self._session.pinned_tool_results.update(checkpoint.pins) + self._session.mark_intercepted(self.trace.messages, tail) + messages = self._harness_session.restore(checkpoint.harness, tail) + # Only the restore notice is new; user hooks prepare it like any user turn. + return await self.step(messages, intercepted=len(messages) - 1) + + async def _checkpoint(self, turn: graph.PendingTurn) -> Checkpoint | None: + if self._closed: + # Settling may have drained `_captures` already: a snapshot taken now + # would start its upload after the box is gone. + return None + task = asyncio.current_task() + if task is not None: + self._captures.add(task) + try: + return await self._capture(turn) + finally: + self._captures.discard(task) + + async def _capture(self, turn: graph.PendingTurn) -> Checkpoint | None: + assert self.runtime is not None + assert self._checkpointing is not None + turn_number = self.trace.num_turns + 1 + if turn_number % self._checkpointing.interval or ( + self.restored_turns is not None and turn_number == self.restored_turns + 1 + ): + return None + if len(self.trace._pending) > 1: + # Another request is in flight (parallel sub-agents share the trace): the + # box is not idle, so no snapshot would match this turn's prefix. + return + assert self._harness_session is not None + # Read before the first await: the reply may commit and run its hooks while + # the harness answers, and the checkpoint must describe the idle box. + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + tail = list(turn.tail) + state = copy.deepcopy(self.trace.state) + info = copy.deepcopy(self.trace.info) + pins = dict(self._session.pinned_tool_results) + try: + harness = await self._harness_session.checkpoint_state() + except Exception: + logger.warning( + "harness checkpoint state failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + return + if harness is None: + return + started = time.time() + try: + snapshot = await self.runtime.backend.snapshot() + except Exception: + logger.warning("snapshot failed (rollout %s)", self.trace.id, exc_info=True) + return None + checkpoint = Checkpoint( + id=snapshot.id, + node=node, + tail=tail, + state=state, + info=info, + pins=pins, + harness=harness, + snapshot=snapshot, + time=TimeSpan(start=started), + captured=time.time(), + ) + self.trace.checkpoints.append(checkpoint) + self._durable[checkpoint.id] = asyncio.create_task( + self._make_durable(checkpoint) + ) + logger.debug( + "checkpoint %s taken: id=%s node=%s", checkpoint.id, self.trace.id, node + ) + return checkpoint + + async def _make_durable(self, checkpoint: Checkpoint) -> None: + assert self.runtime is not None + try: + checkpoint.bytes = await self.runtime.backend.wait_snapshot( + checkpoint.snapshot + ) + except Exception: + # Not restorable: no retry may pick it. The backend still holds it as + # stray and frees it with the box. + logger.warning( + "checkpoint %s never became durable (rollout %s)", + checkpoint.id, + self.trace.id, + exc_info=True, + ) + with contextlib.suppress(ValueError): + self.trace.checkpoints.remove(checkpoint) + else: + checkpoint.time.end = time.time() + self.trace.notify() + await self._prune() + finally: + self._durable.pop(checkpoint.id, None) + + async def _prune(self) -> None: + """Delete the snapshots the `keep` policy drops now that a newer one is + restorable, and the ones a restore rewound past.""" + assert self.runtime is not None and self._checkpointing is not None + async with self._pruning: + keep = kept(self.trace.checkpoints, self._checkpointing.keep) + drop = [ + *( + c.snapshot + for c in self.trace.checkpoints + if c.time.end and not c.deleted and c.id not in keep + ), + *( + s + for s in self.rewound_snapshots + if s.id not in self.deleted_snapshots + ), + ] + if drop: + self.record_deleted(await self.runtime.backend.discard_snapshots(drop)) + + def record_deleted(self, ids: list[str]) -> None: + """Mark the snapshots with these ids deleted.""" + self.deleted_snapshots.update(ids) + for checkpoint in self.trace.checkpoints: + if checkpoint.id in self.deleted_snapshots: + checkpoint.deleted = True + + async def _settle_checkpoints(self, *, wait: bool) -> None: + """Before the box goes: wait for the snapshots still uploading, so a retry + can restore them (its teardown would fail them), or, with nothing to retry, + stop waiting and leave them to be freed.""" + # A snapshot still being taken would start its upload after this returned. + await asyncio.gather(*self._captures, return_exceptions=True) + tasks = list(self._durable.values()) + if not wait: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + # A task cancelled before it first ran never reached its own cleanup. + for checkpoint_id, task in list(self._durable.items()): + if task.done(): + del self._durable[checkpoint_id] + async def _cleanup_harness(self, runtime: Runtime) -> None: if self._harness_cleaned: return @@ -692,6 +971,7 @@ async def abort(self) -> None: (a cancellation mid-setup, a lifetime bug raised to the caller) means the driver will never reach `close()`. Safe after a partial `close()`.""" self._closed = True + await self._settle_checkpoints(wait=False) if self._harness_session is not None: with contextlib.suppress(Exception): await self._harness_session.close() @@ -710,6 +990,22 @@ async def abort(self) -> None: ): with contextlib.suppress(Exception): await self.runtime.backend.stop() + if self.runtime is not None: + # Strays were never restorable, so nobody else knows of them. + with contextlib.suppress(Exception): + self.record_deleted( + await self.runtime.backend.discard_snapshots( + [ + *self.rewound_snapshots, + *( + c.snapshot + for c in self.trace.checkpoints + if not c.deleted + ), + *self.runtime.backend.stray_snapshots, + ] + ) + ) async def _score_harness(self, runtime: Runtime, deadline: float | None) -> None: """Collect best-effort metrics without replacing a task failure.""" @@ -983,7 +1279,7 @@ async def close(self) -> Trace: "Scored rollout in %s", format_time(trace.timing.scoring.duration) ) except Exception as e: # noqa: BLE001 - finalize boundary records every rollout failure - self.fail(e) + self.fail(e, stage="scoring" if trace.timing.scoring.start else "finalize") finally: if self._harness_session is not None: with contextlib.suppress(Exception): @@ -1005,6 +1301,9 @@ async def close(self) -> Trace: trace.split_agent_time() if runtime is not None: await self._cleanup_harness(runtime) + # A failed attempt's retry restores the newest snapshot, which must be + # durable before the box goes. + await self._settle_checkpoints(wait=not trace.ok) # Tear down here — the env's `score()` (later) needs only the traces, # not a live runtime. A borrowed runtime is its creator's to tear down, # not this rollout's. @@ -1019,6 +1318,10 @@ async def close(self) -> Trace: ): # Setup can fail before a deployment takes ownership. await runtime.backend.stop() + if runtime is not None and runtime.backend.stray_snapshots: + await runtime.backend.discard_snapshots( + runtime.backend.stray_snapshots + ) except Exception as e: logger.warning( "Failed to tear down runtime - %s: %s", diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 46f82c44f6..1acfc71a09 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -28,6 +28,7 @@ ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, register, ) from verifiers.v1.runtimes.runtime import Runtime @@ -108,6 +109,7 @@ def runtime_is_local(config: RuntimeConfig) -> bool: "RuntimeConfig", "RuntimeInfo", "RuntimeProcess", + "Snapshot", "SubprocessBackend", "SubprocessConfig", "SubprocessRuntimeInfo", diff --git a/verifiers/v1/runtimes/backends/prime.py b/verifiers/v1/runtimes/backends/prime.py index e6b3eae99d..587c678ef7 100644 --- a/verifiers/v1/runtimes/backends/prime.py +++ b/verifiers/v1/runtimes/backends/prime.py @@ -27,6 +27,7 @@ ProgramResult, RuntimeBackend, RuntimeProcess, + Snapshot, parse_gpu, ) from verifiers.v1.runtimes.limiters import creation_limiter @@ -53,6 +54,9 @@ _OUTPUT_RETRIES = 10 """Re-reads of a finished job's output that the SDK still failed to fetch, before the exec is reported as failed.""" +_CHECKPOINT_DURABLE_SECONDS = 300 +"""How long a checkpoint gets to become restorable (`DURABLE`): the platform uploads it +after the request returns, and only a durable one restores.""" _PUBLIC_DOMAIN = "sandbox-v2-prod.pinfra.io" """Domain of the public URLs `https://-.` that a LAN network's `internet` rule publishes ports at.""" @@ -152,6 +156,7 @@ class PrimeBackend(RuntimeBackend): SUPPORTS_USER_SWITCHING: ClassVar[bool] = True SUPPORTS_COMPOSE: ClassVar[bool] = True SUPPORTS_PORT_EXPOSURE: ClassVar[bool] = True + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -245,8 +250,8 @@ async def create_and_capture_id(): CreateSandboxRequest( name=self.name, labels=list(dict.fromkeys(labels)), - **source, environment_vars=self.env, + **source, **{k: v for k, v in options.items() if v is not None}, ) ) @@ -269,10 +274,13 @@ async def create_and_capture_id(): ) await self._client.wait_for_creation(self.info.id, max_attempts=900) logger.debug( - "Started prime runtime %s in %s image=%s", + "Started prime runtime %s in %s image=%s%s", self.info.id, format_time(time.perf_counter() - started), self.config.image, + f" checkpoint={self.config.checkpoint}" + if self.config.checkpoint + else "", ) await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" @@ -289,6 +297,59 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e + async def snapshot(self) -> Snapshot: + # The platform fixes the snapshot's content before the request returns, then + # uploads it. Until it is durable the snapshot is stray: the platform refuses + # to delete it while this box lives, so teardown frees it after. + async def create() -> Snapshot: + checkpoint = await self._client.checkpoint(self.info.id) + snapshot = Snapshot(id=checkpoint.id) + self.stray_snapshots.append(snapshot) + return snapshot + + try: + return await run_shielded(create()) + except Exception as e: + raise SandboxError(f"prime checkpoint failed: {e}") from e + + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: + started = time.monotonic() + try: + durable = await self._client.wait_for_checkpoint( + snapshot.id, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + ) + except Exception as e: + raise SandboxError(f"prime checkpoint {snapshot.id}: {e}") from e + self.stray_snapshots.remove(snapshot) + logger.debug( + "prime: checkpoint %s durable after %.1fs on sandbox %s", + snapshot.id, + time.monotonic() - started, + self.info.id, + ) + return durable.stored_bytes + + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: + # The platform refuses only the newest checkpoint of a running box (the parent + # of its live state); older ones go while it runs, their data kept as long as a + # newer one needs it. Usually called after teardown released `_client`. + deleted = [] + async with leased_client() as client: + for snapshot in snapshots: + try: + await client.delete_checkpoint(snapshot.id) + except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + logger.warning( + "prime: failed to delete checkpoint %s: %s", snapshot.id, e + ) + else: + deleted.append(snapshot.id) + return deleted + + @classmethod + def restore_config(cls, config: PrimeConfig, snapshot: Snapshot) -> PrimeConfig: + return config.model_copy(update={"checkpoint": snapshot.id}) + async def prepare_execution( self, routes: list[str] | None, network: NetworkPolicyConfig | None = None ) -> None: diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 903fae63e5..71c354fba6 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -13,8 +13,9 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Mapping from dataclasses import dataclass, field -from typing import TYPE_CHECKING, ClassVar +from typing import TYPE_CHECKING, Any, ClassVar, Literal +from pydantic import BaseModel from pydantic_config import BaseConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig @@ -142,6 +143,17 @@ class BaseRuntimeInfo(BaseConfig): (`Agent.run(runtime=...)`) rather than provisioning its own.""" +class Snapshot(BaseModel): + """A snapshot of a box that new boxes can boot from (`RuntimeBackend.snapshot`, + `restore_config`), as many times as needed.""" + + id: str + """The provider's id for the snapshot.""" + type: Literal["filesystem"] = "filesystem" + """What the snapshot captures, as the runtime took it: `filesystem` holds the box's + files, not its processes.""" + + @dataclass class RuntimeCache: """Framework state for one provisioned resource, kept off the backend contract: @@ -186,6 +198,11 @@ class RuntimeBackend(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" + SUPPORTS_FILESYSTEM_SNAPSHOTS: ClassVar[bool] = False + """Whether `snapshot` captures the box's filesystem for new boxes to boot from + (`restore_config`). Rollout checkpointing (`AgentConfig.checkpoint`) takes one at + model turns, so a retry continues from there instead of starting over.""" + info: BaseRuntimeInfo def __init__(self, name: str | None = None) -> None: @@ -194,6 +211,9 @@ def __init__(self, name: str | None = None) -> None: # Explicit process values (model credentials, proxy settings, etc.) override these. self.env: dict[str, str] = {} self.cache = RuntimeCache() + self.stray_snapshots: list[Snapshot] = [] + """Snapshots this box took that never became restorable (a failed or cancelled + durability wait), so no trace records them; freed after the box is gone.""" self._setup_claimed = False self._background: list[RuntimeProcess] = [] self.stopped = False @@ -220,6 +240,30 @@ def type(self) -> str: async def start(self) -> None: pass + async def snapshot(self) -> Snapshot: + """Snapshot the box and return it once its content is fixed: writes after this + returns are not in it. It becomes restorable only later (`wait_snapshot`); the + box keeps running.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") + + async def wait_snapshot(self, snapshot: Snapshot) -> int | None: + """Wait until `snapshot` is restorable and return its stored size in bytes, + when the provider reports it. Raises if it never becomes restorable. The box + must stay alive until then: its teardown fails a pending snapshot.""" + raise NotImplementedError(f"{type(self).__name__} does not support snapshots") + + async def discard_snapshots(self, snapshots: list[Snapshot]) -> list[str]: + """Free snapshots nothing will restore from and return the ids it freed. + Best-effort: a provider may refuse one a live box still needs (prime: the + newest one of a running box).""" + return [] + + @classmethod + def restore_config(cls, config: Any, snapshot: Snapshot) -> Any: + """The runtime config for a new box that boots from `snapshot` instead of the + task's image.""" + raise NotImplementedError(f"{cls.__name__} does not support snapshots") + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index c98d081b8e..e7f62db47d 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -33,7 +33,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig from verifiers.v1.errors import HarnessError, RolloutError, TaskError from verifiers.v1.harnesses.utils.compaction import bound_tool_message -from verifiers.v1.trace import InterceptRecord, Trace +from verifiers.v1.trace import Checkpoint, InterceptRecord, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -191,6 +191,10 @@ class RolloutSession: when its program dies on it the rollout records this original error instead of a secondary `HarnessError`. A harness that completes cleanly after the failure handled it. Reset before each model turn, so a successful retry clears it.""" + aborted: "RolloutError | None" = None + """A request hook's failure: the task, not the provider, refused a turn. The rollout + ends the running segment with it (`Rollout.step`); the harness never sees it.""" + abort_event: asyncio.Event = field(default_factory=asyncio.Event) idempotent_requests: dict[str, IdempotentRequest] = field(default_factory=dict) """Explicit keys or marked SDK retries mapped to their replay state.""" released: bool = False @@ -216,6 +220,12 @@ class RolloutSession: gated_tools: set[tuple[str, str]] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" prepared_users: Counter[str] = field(default_factory=Counter) + checkpointer: Callable[[graph.PendingTurn], Awaitable[Checkpoint | None]] | None = ( + None + ) + """Snapshots the rollout's box as a turn goes upstream, alongside the model call; + the reply waits for the capture (`Rollout` sets it when its runtime supports + checkpoints and a retry may restore one). Returns the checkpoint it took, if any.""" @property def stopped(self) -> bool: @@ -392,6 +402,21 @@ async def rewrite_request( self.pinned_tool_results[tool_keys[position]] = after return current, records, None + def mark_intercepted(self, branch: Messages, tail: Messages) -> None: + """Treat `tail`, which follows `branch` and was intercepted in an earlier + attempt (a restored checkpoint's), as prepared: the harness re-sends it, and + a non-idempotent hook must not rewrite it twice.""" + request = Request(messages=[*branch, *tail]) + for position, key in self.tool_result_keys(request).items(): + message = request.messages[position] + if position >= len(branch) and isinstance(message, ToolMessage): + self.pinned_tool_results.setdefault(key, message) + self.prepared_users.update( + graph.message_hash(message) + for message in tail + if isinstance(message, UserMessage) + ) + def consume_prepared(self, messages: Messages) -> None: """Forget pre-harness user rewrites only after their model request commits.""" for message in messages: @@ -614,6 +639,12 @@ def adopt(self, task: "asyncio.Task | None") -> None: self.tasks.add(task) task.add_done_callback(self.tasks.discard) + def abort(self, error: "RolloutError") -> None: + """End the running segment with `error`. The first abort wins.""" + if self.aborted is None: + self.aborted = error + self.abort_event.set() + def release(self) -> None: """Seal the session: no further trace mutation, and in-flight handlers cancel.""" self.released = True diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index c41cccf355..0ae847959c 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -30,7 +30,7 @@ from verifiers.v1.configs.sampling import Sampling from verifiers.v1.errors import ProviderError, stop_condition from verifiers.v1.graph import RECORD_FLOAT_DECIMALS, MessageNode -from verifiers.v1.runtimes import RuntimeInfo +from verifiers.v1.runtimes import RuntimeInfo, Snapshot from verifiers.v1.semantic import ACPInfo, ParentLink, SemanticEdgeSet from verifiers.v1.state import State, StateT from verifiers.v1.task import DataT, WireTaskData @@ -252,6 +252,8 @@ class TimeSplit(BaseModel): class AgentSpan(TimeSpan): model: TimeSplit = Field(default_factory=TimeSplit) harness: TimeSplit = Field(default_factory=TimeSplit) + checkpoint: TimeSplit = Field(default_factory=TimeSplit) + """Time the model replies waited for checkpoint captures (`Checkpoint.held`).""" class Timing(BaseModel): @@ -268,6 +270,10 @@ class Error(BaseModel): message: str status_code: int | None = None traceback: str | None = None + attempt: int = 0 + """The run attempt that captured it: 0 for the first, then one per retry.""" + stage: str | None = None + """The rollout stage that failed (`setup`, `agent`, `finalize`, `scoring`), when known.""" class VersionInfo(BaseModel): @@ -371,6 +377,51 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall prev_total = call.usage.total_tokens +class Checkpoint(BaseModel): + """A snapshot of the rollout's box taken as a model turn went upstream: the box holds + the effects of every message in `node`'s conversation plus `tail`, and nothing of the + turn that followed. A retry restores it (`Trace.rewind`).""" + + id: str + """The snapshot's id.""" + snapshot: Snapshot + """The box's snapshot (`RuntimeBackend.snapshot`) a retry boots a new box from.""" + node: int | None = None + """The last committed node whose conversation the box's state follows; None when + the snapshot preceded the first turn.""" + tail: Messages = Field(default_factory=list, exclude=True) + """The turn's prompt messages after `node` that had no node yet (tool results, user + turns): with `node`'s conversation, the prompt a restored harness continues from. + Live-only — a retry restores it in the process that took the snapshot.""" + state: Any = Field(None, exclude=True) + """The trace's typed state as the snapshot was taken (tool servers write it over + `/state`), restored with the box so a retry sees one consistent world. Live-only.""" + info: dict[str, Any] | None = Field(None, exclude=True) + """`Trace.info` as the snapshot was taken, restored with `state`. Live-only.""" + pins: dict[tuple[str, str], ToolMessage] = Field(default_factory=dict, exclude=True) + """The rollout's pinned tool results as the snapshot was taken + (`RolloutSession.pinned_tool_results`): a restored harness re-sends its own + copies, which must be pinned again to match the trace. Live-only.""" + time: TimeSpan = Field(default_factory=TimeSpan) + """From the snapshot request until it was durable (restorable); `end` stays 0 while + it is not.""" + captured: float = 0.0 + """When the request returned: the snapshot's content was fixed from then on.""" + held: float = 0.0 + """Seconds the turn's reply waited for the capture; 0 when the model call took + longer, since the snapshot runs alongside it.""" + bytes: int | None = None + """The snapshot's stored size, which the provider bills, once it is durable.""" + harness: dict[str, Any] = Field(default_factory=dict) + """What the harness session needs to continue in a box restored from the snapshot + (`HarnessSession.checkpoint_state`), e.g. an ACP agent's session id.""" + restored: bool = False + """Whether a retry restored the run from this snapshot.""" + deleted: bool = False + """Whether the snapshot was deleted (by `RolloutCheckpointConfig.keep`, or once the + run is final): it can no longer be restored.""" + + def _concat(parts: list[list | np.ndarray], dtype: type[np.generic]) -> np.ndarray: """Node streams (lists or arrays) joined into one array of `dtype`.""" if not parts: @@ -605,6 +656,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) """Response changes made by `@intercept`, in execution order.""" + checkpoints: list[Checkpoint] = Field(default_factory=list) + """Snapshots of the run's box, oldest first — one per model turn on a runtime that + supports them; a retry restores the newest.""" rewards: dict[str, Reward | None] = Field(default_factory=dict) """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a @@ -645,6 +699,14 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): _pending: dict[int, list[Message]] = PrivateAttr(default_factory=dict) """The messages of the request in flight that no node holds yet (the harness's tool results and user turns): a preview for live watchers until the turn commits.""" + _generation: int = PrivateAttr(default=0) + """Bumped by `rewind`: the graph is no longer an extension of what it was, so a + consumer streaming it incrementally starts over.""" + + @property + def generation(self) -> int: + """How many times the trace was rewound (`rewind`).""" + return self._generation def watch(self, on_change: Callable[[Trace], None]) -> None: """Have `on_change` called at each of this trace's phase changes and turns.""" @@ -893,6 +955,11 @@ def last_error(self) -> Error | None: """The last error captured across attempts.""" return self.errors[-1] if self.errors else None + @property + def has_restored(self) -> bool: + """Whether a retry restored this run from a checkpoint rather than starting over.""" + return any(checkpoint.restored for checkpoint in self.checkpoints) + @property def transcript(self) -> str: """Text transcript of the last branch's messages; tool outputs are omitted.""" @@ -967,13 +1034,22 @@ def drop(self, reason: str) -> None: self.dropped = reason def split_agent_time(self) -> None: - """Split the agent span into model and harness time.""" + """Split the agent span into model, checkpoint and harness time.""" span = self.timing.agent if not span.end: return - model = sum(call.time.duration for call in self.calls) - span.model.duration = min(model, span.duration) - span.harness.duration = span.duration - span.model.duration + # A restored trace keeps the calls and checkpoints of the attempts before + # this span. + calls = sum( + call.time.duration for call in self.calls if call.time.start >= span.start + ) + # A call's span includes the time its reply waited for the capture. + held = sum(c.held for c in self.checkpoints if c.captured >= span.start) + span.checkpoint.duration = min(held, calls, span.duration) + span.model.duration = min(calls, span.duration) - span.checkpoint.duration + span.harness.duration = ( + span.duration - span.model.duration - span.checkpoint.duration + ) def record_timeout(self, stage: str) -> None: """Record a stage deadline's expiry, and stop the trace as `_timeout`. @@ -982,7 +1058,58 @@ def record_timeout(self, stage: str) -> None: self.is_timeout = True self.stop(f"{stage}_timeout", override=True) - def record_error(self, error: Exception) -> None: + def rewind_state(self, checkpoint: Checkpoint) -> None: + """Set the typed state and `info` back to `checkpoint`'s.""" + if checkpoint.state is not None: + self.state = copy.deepcopy(checkpoint.state) + if checkpoint.info is not None: + self.info = copy.deepcopy(checkpoint.info) + + def rewind(self, checkpoint: Checkpoint) -> list[Snapshot]: + """Roll the graph back to `checkpoint` — the nodes committed after it go, its own + conversation stays sampled — and reopen the trace for another attempt: the outcome + stamps reset, while `errors` and every call stay as the attempts' history (the + calls of the dropped nodes unlinked, so usage still counts them). Returns the + later box snapshots this drops.""" + keep = 0 if checkpoint.node is None else checkpoint.node + 1 + if keep > len(self.nodes): + raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + ids = [c.id for c in self.checkpoints] + if checkpoint.id not in ids: + raise ValueError(f"checkpoint {checkpoint.id!r} is not on this trace") + del self.nodes[keep:] + for node in self.nodes: + node.semantic_parents = [ + link for link in node.semantic_parents if link.node < keep + ] + self._head_index = {} + self._pending = {} + self._generation += 1 + for call in self.calls: + if call.node is not None and call.node >= keep: + call.node = None + self.rewind_state(checkpoint) + # The restored rollout re-stamps every stage span; the run's start stays. + self.timing = Timing(start=self.timing.start) + # Checkpoints are in the order they were taken: the ones after `checkpoint` hold + # the future it rewinds past, including those at its own node. + position = ids.index(checkpoint.id) + 1 + rewound = self.checkpoints[position:] + del self.checkpoints[position:] + dropped = [c.snapshot for c in rewound] + self.rewards = {} + self.metrics = {} + self.dropped = None + self.root_reply = None + self.is_completed = False + self.ok = False + self.stop_condition = None + self.is_timeout = False + return dropped + + def record_error( + self, error: Exception, *, attempt: int = 0, stage: str | None = None + ) -> None: """Record an error, and stop the trace as failed: `_error` for a typed rollout error (`errors.stop_condition`), `error` for any other.""" self.errors.append( @@ -990,6 +1117,8 @@ def record_error(self, error: Exception) -> None: type=type(error).__name__, message=str(error), status_code=getattr(error, "status_code", None), + attempt=attempt, + stage=stage, # Provider errors already carry the actionable upstream diagnostic. # Keep full tracebacks for every other failure. traceback=None