diff --git a/pyproject.toml b/pyproject.toml index 8598cc3086..65690cf3ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,8 @@ url = "https://pypi.org/simple" default = true [tool.uv.sources] +# Filesystem checkpoints ship in prime-cli PR #971 (unreleased): pin the SDK to that branch. +prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime.git", subdirectory = "packages/prime-sandboxes", rev = "13c8688" } compact = { path = "environments/compact", editable = true } glossary = { path = "environments/glossary", editable = true } deepwiki = { path = "environments/deepwiki", editable = true } diff --git a/tests/v1/fixtures/checkpoint_resume_v1.py b/tests/v1/fixtures/checkpoint_resume_v1.py new file mode 100644 index 0000000000..0ebc8e5684 --- /dev/null +++ b/tests/v1/fixtures/checkpoint_resume_v1.py @@ -0,0 +1,72 @@ +"""checkpoint-resume: an agentic task whose first attempt dies mid-rollout, so the retry +resumes from the box's newest filesystem checkpoint instead of starting over. + +`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model +for two file steps that build on each other, and the reward reads the second file back +out of the box. A request `@intercept` raises a `ProviderError` once the first attempt has +made two turns — after the checkpoint of turn 2 exists, whose box holds step one's file. +The resumed attempt continues from that checkpoint: it never sees the injected failure +again (the trace already carries the error), inherits step one, and finishes. Needs an +agentic (shell) harness that can continue a conversation, e.g. `bash`, on a runtime with +checkpoints (docker, prime). +""" + +import uuid + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +SYSTEM = "You complete the task by running shell commands with the bash tool." +MARKER = "marker.txt" +STEP_ONE = "step1.txt" +DONE = "done.txt" +FAIL_AFTER_TURNS = 2 + + +class CheckpointResumeTask(vf.Task[vf.TaskData]): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.write(MARKER, nonce.encode()) + + @vf.intercept + def fail_once(self, request: vf.Request, trace: vf.Trace) -> None: + # Request hooks also judge each proposed tool call after a turn commits, over + # the branch plus an empty probe result; fail only as a real turn goes upstream + # (every step's command echoes, so a real result is never empty). + last = request.messages[-1] + probe = isinstance(last, vf.ToolMessage) and not last.content + if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not trace.errors: + raise vf.ProviderError("injected mid-rollout failure", status_code=503) + + @vf.reward(weight=1.0) + async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float: + try: + content = (await runtime.read(DONE)).decode(errors="replace") + except (SandboxError, OSError, ValueError): + return 0.0 + return float(content.strip() == trace.info["nonce"]) + + +class CheckpointResumeTaskset(vf.Taskset[CheckpointResumeTask, vf.TasksetConfig]): + def load(self) -> list[CheckpointResumeTask]: + return [ + CheckpointResumeTask( + vf.TaskData( + idx=0, + prompt=( + "Complete these steps in order, one bash command per turn, " + "and wait for each result before the next.\n" + f"1. Run `cp {MARKER} {STEP_ONE} && echo copied`.\n" + f"2. Run `cat {STEP_ONE} > {DONE} && echo written`.\n" + "3. Reply with the single word: done." + ), + system_prompt=SYSTEM, + ), + self.config.task, + ) + ] + + +__all__ = ["CheckpointResumeTaskset"] diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 4982cd1fab..d5111d3f08 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -576,6 +576,73 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path): assert trace.reward == 1.0 +# Runtimes whose box can be checkpointed and restored: docker commits the container to +# an image, prime snapshots the sandbox filesystem (needs the platform's checkpoint API). +CHECKPOINT_RUNTIMES = [ + pytest.param("docker", marks=[mark.bash, mark.docker], id="bash-harness-in-docker"), + pytest.param("prime", marks=[mark.bash, mark.prime], id="bash-harness-in-prime"), +] + + +@pytest.mark.e2e +@pytest.mark.parametrize("harness_runtime", CHECKPOINT_RUNTIMES, indirect=True) +async def test_retry_resumes_from_checkpoint(run_v1, harness_runtime, tmp_path): + """A retry resumes the failed attempt's trace from its newest checkpoint: the box + restores to the state before the turn that failed, the turns up to there stay, + and the task's `setup` (a nonce the prompt never reveals) is not rerun.""" + (trace,) = await run_v1( + "checkpoint-resume-v1", + harness="bash", + runtime={"type": harness_runtime}, + env={"agent": {"retries": {"max_retries": 1, "include": ["ProviderError"]}}}, + output_dir=tmp_path, + max_turns=8, + max_tokens=4096, + ) + assert_resumed_from_checkpoint(trace) + + +@pytest.mark.e2e +@pytest.mark.bash +@pytest.mark.docker +async def test_retry_resumes_from_checkpoint_server(run_v1_server, tmp_path): + """The same resume through the env-server pool: the worker's retry rewinds the + trace it is streaming, and the client still assembles the trace the worker + finished with.""" + (trace,) = await run_v1_server( + "checkpoint-resume-v1", + harness="bash", + runtime={"type": "docker"}, + env={"agent": {"retries": {"max_retries": 1, "include": ["ProviderError"]}}}, + output_dir=tmp_path, + max_turns=8, + max_tokens=4096, + ) + assert_resumed_from_checkpoint(trace) + + +def assert_resumed_from_checkpoint(trace) -> None: + from checkpoint_resume_v1 import FAIL_AFTER_TURNS + + assert trace.ok + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + # The failure hit as turn FAIL_AFTER_TURNS + 1 went upstream, so the newest + # checkpoint is the one taken as turn FAIL_AFTER_TURNS was asked for: the box then + # held the earlier turns' effects, and the trace kept those turns. + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.restored_from + assert restored_from is not None + assert restored_from in [checkpoint.id for checkpoint in trace.checkpoints] + assert trace.num_turns >= FAIL_AFTER_TURNS + assert len(trace.checkpoints) >= trace.num_turns + # Every turn kept has its call; the failure never went upstream, so no call + # records it. + assert [call.node for call in trace.calls] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + + @pytest.mark.e2e async def test_multi_agent_env(run_v1, tmp_path): """An `Env` subclass shipped with its taskset (duet-v1): two roles run the diff --git a/tests/v1/test_serve_delta.py b/tests/v1/test_serve_delta.py index cb2b9ef5a2..51e7232183 100644 --- a/tests/v1/test_serve_delta.py +++ b/tests/v1/test_serve_delta.py @@ -109,6 +109,45 @@ async def send(data: dict) -> None: assert len(assembly.traces[trace.id]["calls"]) == 2 +@pytest.mark.asyncio +async def test_rewound_trace_streams_again_from_scratch(): + """A retry resumed from a checkpoint rewinds its trace below what the client + holds; the stream drops the client's copy and resends the trace as it now is.""" + traces: list[vf.Trace] = [] + frames: list[bytes] = [] + + async def send(data: dict) -> None: + frames.append(pack(data)) + + async with DeltaStreamer(lambda: traces, send) as streamer: + trace = make_trace() + traces.append(trace) + streamer.watch(trace) + add_turn(trace, "a1") + add_turn(trace, "a2") + add_turn(trace, "a3") + await settle() + assert len(assembled(frames)[trace.id]["nodes"]) == 4 + trace.record_error(RuntimeError("boom")) + trace.rewind(vf.Checkpoint(id="cp", node=1)) + trace.notify() # the resumed rollout's boot + await settle() + add_turn(trace, "a2 again") + await settle() + traces = [trace] + record = assembled(frames)[trace.id] + assert [n["message"]["content"] for n in record["nodes"]] == ["q", "a1", "a2 again"] + assert len(record["calls"]) == 2 + assert [e["type"] for e in record["errors"]] == ["RuntimeError"] + + +def assembled(frames: list[bytes]) -> dict[str, dict]: + assembly = EpisodeAssembly() + for frame in frames: + assembly.apply(unpack(frame)) + return assembly.traces + + @pytest.mark.asyncio async def test_pending_preview_streams_and_clears_on_commit(): traces: list[vf.Trace] = [] diff --git a/uv.lock b/uv.lock index 4b36b7f431..34758cd6b2 100644 --- a/uv.lock +++ b/uv.lock @@ -3485,8 +3485,8 @@ wheels = [ [[package]] name = "prime-sandboxes" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } +version = "0.3.2" +source = { git = "https://github.com/PrimeIntellect-ai/prime.git?subdirectory=packages%2Fprime-sandboxes&rev=13c8688#13c86889a6c277ae18e3dcaaaccfc2051b293db1" } dependencies = [ { name = "aiofiles" }, { name = "certifi" }, @@ -3497,10 +3497,6 @@ dependencies = [ { name = "pyqwest" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b6/46/2d118d8bbb901b729f3bc0678725eb28088d757a872db04cffa1492d0208/prime_sandboxes-0.3.0.tar.gz", hash = "sha256:c99075ed82a9bbfe5a453cee84b9f09565a7f6b07fd5ace072401cce59a1fe08", size = 132759, upload-time = "2026-09-18T01:32:31.263Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/04/c77c7f260babf93978f912b85511e76f63a3cfa5bb11a8c34108177d5825/prime_sandboxes-0.3.0-py3-none-any.whl", hash = "sha256:19078ff508fa72de8f565422918faf367e2831ed8a2f26a5a6a98110e4062941", size = 71441, upload-time = "2026-09-18T01:32:30.062Z" }, -] [[package]] name = "prime-traces" diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index b0cd5e1802..ae78beefd5 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -103,6 +103,7 @@ AgentInfo, AgentSpan, Branch, + Checkpoint, Error, InterceptRecord, ModelCall, @@ -226,6 +227,7 @@ "WireTaskData", "TaskResources", "TaskTimeout", + "Checkpoint", "Trace", "TraceTask", "WireTrace", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index a8dd4f4541..f996ebb83e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -36,7 +36,7 @@ ) from verifiers.v1.session import RolloutLimits from verifiers.v1.task import Task -from verifiers.v1.trace import Trace +from verifiers.v1.trace import Checkpoint, Error, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -50,7 +50,7 @@ resolve_runtime_config, validate_pairing, ) -from verifiers.v1.utils.retries import backoff, trace_should_retry +from verifiers.v1.utils.retries import backoff, errors_should_retry __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"] @@ -361,15 +361,19 @@ def _interception_for( return self._server return None - def _check_resume_support(self) -> None: - # Multi-turn capability is derived: a harness needs transcript replay, a - # native resume implementation, or a rollout-scoped session implementation. + def _supports_resume(self) -> bool: + """Whether the harness can continue a conversation it is handed: transcript + replay, a native resume implementation, or a rollout-scoped session.""" harness = self.harness - if ( - type(harness).session is Harness.session - and type(harness).resume is Harness.resume - and not harness.SUPPORTS_RESUME - ): + return ( + type(harness).session is not Harness.session + or type(harness).resume is not Harness.resume + or harness.SUPPORTS_RESUME + ) + + def _check_resume_support(self) -> None: + if not self._supports_resume(): + harness = self.harness raise ValueError( f"Harness {harness.config.id!r} cannot host a user: resuming an " "exchange takes transcript-backed resume (SUPPORTS_RESUME) for the " @@ -394,18 +398,42 @@ async def run( owner, counted in the pairing check; `on_trace` observes the trace the moment it's minted, before any I/O. `collect_artifacts` captures the task's declared artifacts after its finalizer while its container runtime is still - alive. Retries whole while the trace ends with a retryable error - (`config.retries`) — never into a borrowed box; the final trace keeps earlier - attempts' errors.""" + alive. Retries while the trace ends with a retryable error (`config.retries`) + — never into a borrowed box. A retry resumes from the failed attempt's newest + checkpoint when it has one (`retries.checkpoint`, a runtime that supports + them, a harness that can continue a conversation), keeping that trace and its + turns up to there; otherwise it starts over. The final trace keeps every + attempt's errors.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") retry = self.config.retries - history: list = [] + can_resume = retry.checkpoint and runtime is None and self._supports_resume() + history: list[Error] = [] # errors of the attempts started over from + trace: Trace | None = None + resume: Checkpoint | None = None for attempt in range(retry.max_retries + 1): + resumed = trace if resume is not None else None + seen = len(resumed.errors) if resumed is not None else 0 + + def should_retry( + current: Trace, *, attempt: int = attempt, seen: int = seen + ) -> bool: + # This attempt's own errors decide — a resumed trace still carries + # the earlier attempts'. + return attempt < retry.max_retries and errors_should_retry( + current.errors[seen:], retry + ) + trace = await self._run_once( - task, runtime, tools, on_trace, collect_artifacts + task, + runtime, + tools, + on_trace, + collect_artifacts, + resume=(resumed, resume) if resumed is not None else None, + retry=should_retry if can_resume and retry.max_retries else None, ) - if attempt == retry.max_retries or not trace_should_retry(trace, retry): + if not should_retry(trace): break if runtime is not None: logger.warning( @@ -413,16 +441,22 @@ async def run( "longer the task's start state); the error stands" ) break - history.extend(trace.errors) + resume = trace.checkpoints[-1] if can_resume and trace.checkpoints else None + if resume is None: + history.extend(trace.errors) delay = backoff(attempt) logger.warning( - "retrying agent rollout (retry %d/%d) in %.1fs after error: %s", + "retrying agent rollout (retry %d/%d) in %.1fs after error: %s (%s)", attempt + 1, retry.max_retries, delay, trace.last_error.type if trace.last_error else "?", + f"resuming from checkpoint {resume.id}" + if resume is not None + else "from scratch", ) await asyncio.sleep(delay) + assert trace is not None if history: # The full history rides the final trace either way; success is the # `ok` stamp, never errors-emptiness. @@ -436,6 +470,9 @@ async def _run_once( shared_tools: Mapping[str, SharedToolServer] | None, on_trace: Callable[[Trace], None] | None, collect_artifacts: bool, + *, + resume: tuple[Trace, Checkpoint] | None = None, + retry: Callable[[Trace], bool] | None = None, ) -> Trace: params = self._rollout_params(task, runtime, dict(shared_tools or {})) if collect_artifacts and isinstance(params["runtime_config"], SubprocessConfig): @@ -447,11 +484,16 @@ async def _run_once( task=task, on_trace=on_trace, collect_artifacts=collect_artifacts, + resume=resume, + retry=retry, **params, ) try: if await run.open(): - await run.step() + if resume is not None: + await run.resume() + else: + await run.step() if run.ok: run.trace.stop("agent_completed") trace = await run.close() diff --git a/verifiers/v1/configs/retries.py b/verifiers/v1/configs/retries.py index 488a4e4860..3a24999dcb 100644 --- a/verifiers/v1/configs/retries.py +++ b/verifiers/v1/configs/retries.py @@ -15,3 +15,10 @@ class RetryConfig(BaseConfig): """Only retry errors whose type is listed. Empty = retry anything not excluded.""" exclude: list[str] = Field(default_factory=list) """Never retry errors whose type is listed (wins over `include`).""" + checkpoint: bool = True + """Checkpoint the runtime's filesystem before every model turn (on runtimes that + support it, e.g. prime and docker) and resume a retry from the newest checkpoint: + the trace keeps its turns up to there and the harness continues the conversation + in a box restored to that state. False retries from scratch. Only the box is + restored — state kept outside it (a tool server's typed rollout state) stays as + the failed attempt left it.""" diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index d59118f877..07934c7427 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -779,6 +779,10 @@ async def sample() -> web.Response: started = time.time() try: try: + # The box is idle while the harness awaits this turn: the snapshot + # matches the prompt exactly (tool results in, this reply not yet). + if session.checkpointer is not None: + await session.checkpointer(turn) # What actually goes upstream: the native body with the rollout's model + # sampling imposed — recorded raw on the trace, per call. if relay: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 0e3613e1a3..10d9f72bff 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -8,6 +8,7 @@ from contextlib import AsyncExitStack from dataclasses import dataclass +from verifiers.v1 import graph from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig @@ -30,7 +31,7 @@ from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary from verifiers.v1.state import state_cls from verifiers.v1.task import Task -from verifiers.v1.trace import AgentInfo, Trace, TraceTask +from verifiers.v1.trace import AgentInfo, Checkpoint, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke @@ -71,7 +72,15 @@ def __init__( runtime: Runtime | None = None, on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, + resume: tuple[Trace, Checkpoint] | None = None, + retry: Callable[[Trace], bool] | None = None, ) -> None: + """`resume` continues a failed attempt's trace from one of its checkpoints: + the trace rewinds to it and this run boots its box from the snapshot instead + of setting the task up again. `retry` says, at close, whether a retry will + resume this run: set, the run checkpoints its box before every turn (on a + runtime that supports it), and a failure bound for a retry keeps them past + teardown.""" self.task = task self.harness = harness self.ctx = ctx @@ -84,20 +93,27 @@ def __init__( self.runtime = runtime self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts - self.trace: Trace = Trace( - task=TraceTask( - type=type(task).__name__, - data=task.data, - key=task.key, - hash=task.hash, - ), - state=state_cls(type(task))(), - # The seat's resolved config, role overrides included — the agent - # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), - ) - if on_trace is not None: - on_trace(self.trace) + self._retry = retry + self._resume: Checkpoint | None = None + self._stale_checkpoints: list[str] = [] + if resume is not None: + self.trace, self._resume = resume + self._stale_checkpoints = self.trace.rewind(self._resume) + else: + self.trace = Trace( + task=TraceTask( + type=type(task).__name__, + data=task.data, + key=task.key, + hash=task.hash, + ), + state=state_cls(type(task))(), + # The seat's resolved config, role overrides included — the agent + # this trace can be reproduced with. + agent=AgentInfo(config=agent_config), + ) + if on_trace is not None: + on_trace(self.trace) interceptors = [ (hook_boundary(fn, allow_trace=False), fn) for fn in discover_decorated(task, "intercept") @@ -201,13 +217,30 @@ async def open(self) -> bool: runtime = self.runtime assert self.trace.agent is not None # minted with the trace self.trace.agent.runtime = runtime.info - logger.info( - "rollout start: id=%s task=%s harness=%s runtime=%s", - self.trace.id, - self.task.data.idx, - self.harness.config.name, - self.runtime_config.type, - ) + if ( + self._retry is not None + and self._borrowed_runtime is None + and runtime.supports_checkpoints + ): + self._session.checkpointer = self._checkpoint + if self._resume is None: + logger.info( + "rollout start: id=%s task=%s harness=%s runtime=%s", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + ) + else: + logger.info( + "rollout resume: id=%s task=%s harness=%s runtime=%s checkpoint=%s turns=%d", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + self._resume.id, + self.trace.num_turns, + ) loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None try: @@ -223,7 +256,9 @@ async def open(self) -> bool: "task.prompt, or drive the run through agent.interaction() and open " "it with the first turn(message)" ) - if self._borrowed_runtime is None: + if self._resume is not None: + await runtime.restore(self._resume.id) + elif self._borrowed_runtime is None: await runtime.start() await runtime.prepare_setup() now = time.time() @@ -236,11 +271,16 @@ async def open(self) -> bool: if self._timeouts.setup is None else loop.time() + self._timeouts.setup ) - async with ( - asyncio.timeout_at(setup_deadline) as setup_timeout, - boundary(TaskError, "task setup"), - ): - await invoke(self.task.setup, {"trace": self.trace, "runtime": runtime}) + # A restored box already holds the task's setup; only the harness's + # (idempotent) install runs again. + if self._resume is None: + async with ( + asyncio.timeout_at(setup_deadline) as setup_timeout, + boundary(TaskError, "task setup"), + ): + await invoke( + self.task.setup, {"trace": self.trace, "runtime": runtime} + ) async with ( asyncio.timeout_at(setup_deadline) as setup_timeout, boundary(HarnessError, "harness setup"), @@ -371,14 +411,18 @@ async def open(self) -> bool: self.trace.notify() return not self._session.stopped - async def step(self, messages: Messages | None = None) -> bool: + async def step( + self, messages: Messages | None = None, *, intercept: bool = True + ) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - Returns whether the exchange can continue — a refused turn (limit, @stop), - a failure (an expired agent timeout included), or a segment that made no - progress all end it.""" + `intercept=False` passes `messages` to the harness as they are — they were + already intercepted when the trace first saw them (a resumed checkpoint's + tail). Returns whether the exchange can continue — a refused turn (limit, + @stop), a failure (an expired agent timeout included), or a segment that + made no progress all end it.""" if not self._opened or self._closed or not self.ok: return False trace = self.trace @@ -396,7 +440,11 @@ async def step(self, messages: Messages | None = None) -> bool: try: async with asyncio.timeout_at(self.deadline_at): assert self._harness_session is not None - if messages is not None and self._session.request_interceptors: + if ( + messages is not None + and intercept + and self._session.request_interceptors + ): prepared, rewrites = await self._session.prepare_users( Request(messages=messages) ) @@ -441,6 +489,67 @@ async def step(self, messages: Messages | None = None) -> bool: # never moved, forever. return self.ok and trace.num_turns > turns_before + async def resume(self) -> bool: + """Run the resumed attempt's first segment: the harness continues the + conversation the checkpoint left off at — its recorded tail (tool results, + user turns) after the trace's rewound branch — in the restored box.""" + assert self._resume is not None + data = self.trace.task.data + # `Harness.resume` re-emits `data.system_prompt` through the program's own + # system message; the tail's copy would double it. + tail = [ + m + for m in self._resume.tail + if m.role != "system" or data.system_prompt is None + ] + return await self.step(tail, intercept=False) + + async def _checkpoint(self, turn: graph.PendingTurn) -> None: + assert self.runtime is not None + try: + checkpoint = await self.runtime.checkpoint() + except Exception: + logger.warning( + "checkpoint failed (rollout %s)", self.trace.id, exc_info=True + ) + return + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + self.trace.checkpoints.append( + Checkpoint(id=checkpoint, node=node, tail=list(turn.tail)) + ) + logger.debug( + "checkpoint %s taken: id=%s node=%s", checkpoint, self.trace.id, node + ) + + async def _settle_checkpoints(self) -> None: + """Decide the fate of the run's snapshots before its box goes: a retry that + will resume from them keeps every restorable one, anything else is freed.""" + runtime = self.runtime + if runtime is None or self._borrowed_runtime is not None: + return + ids = [c.id for c in self.trace.checkpoints] + kept: list[str] = [] + if ids and self._failed and self._retry is not None and self._retry(self.trace): + try: + kept = await runtime.preserve_checkpoints(ids) + except Exception: + logger.warning( + "preserving checkpoints failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + self.trace.checkpoints = [c for c in self.trace.checkpoints if c.id in kept] + stale = [*self._stale_checkpoints, *(i for i in ids if i not in kept)] + if stale: + try: + await runtime.discard_checkpoints(stale) + except Exception: + logger.warning( + "discarding checkpoints failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + async def abort(self) -> None: """Free everything this run holds — the entered servers and an owned runtime — without finalizing or scoring: the escape path when an exception @@ -456,6 +565,10 @@ async def abort(self) -> None: with contextlib.suppress(Exception): await self.harness.cleanup(self.trace, self.runtime) if self._borrowed_runtime is None and self.runtime is not None: + with contextlib.suppress(Exception): + await self.runtime.discard_checkpoints( + [*self._stale_checkpoints, *(c.id for c in self.trace.checkpoints)] + ) with contextlib.suppress(Exception): await self.runtime.stop() @@ -556,6 +669,7 @@ async def close(self) -> Trace: # not a live runtime. A borrowed runtime is its creator's to tear down, # not this rollout's. if self._borrowed_runtime is None and runtime is not None: + await self._settle_checkpoints() try: await runtime.stop() except Exception: @@ -568,6 +682,8 @@ async def close(self) -> Trace: self.task.data.idx, trace.reward, trace.num_turns, - trace.last_error.type if trace.last_error else trace.stop_condition, + trace.last_error.type + if trace.last_error and not trace.ok + else trace.stop_condition, ) return trace diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index c2f806c8b3..b551476250 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -133,6 +133,8 @@ class BaseRuntimeInfo(BaseConfig): borrowed: bool = False """Whether the run was placed into a live box owned by someone else (`Agent.run(runtime=...)`) rather than provisioning its own.""" + restored_from: str | None = None + """The checkpoint this box booted from (`Runtime.restore`); None for a fresh start.""" class Runtime(ABC): @@ -147,6 +149,11 @@ class Runtime(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" + supports_checkpoints: ClassVar[bool] = False + """Whether `checkpoint` snapshots the box's filesystem for `restore` to boot a new box + from. A rollout checkpoints before each model turn, so a retry resumes at the newest + turn instead of replaying the run (see `RetryConfig.checkpoint`).""" + info: BaseRuntimeInfo @property @@ -177,6 +184,25 @@ def type(self) -> str: async def start(self) -> None: pass + async def restore(self, checkpoint: str) -> None: + """Boot this box from `checkpoint` — a snapshot another box of this config took + with `checkpoint()` — instead of `start()`.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def checkpoint(self) -> str: + """Snapshot the filesystem and return the checkpoint's id; the box keeps running. + A provider may store the snapshot asynchronously — `preserve_checkpoints` waits + for that before the box goes away.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def preserve_checkpoints(self, ids: list[str]) -> list[str]: + """Make the box's checkpoints (`ids`, oldest first) outlive its teardown, and return + the ones that are restorable. A retry resumes from the newest returned id.""" + return list(ids) + + async def discard_checkpoints(self, ids: list[str]) -> None: + """Free checkpoints nothing will restore from. Best-effort and idempotent.""" + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/runtimes/docker/__init__.py b/verifiers/v1/runtimes/docker/__init__.py index f6661d0f39..26abe39372 100644 --- a/verifiers/v1/runtimes/docker/__init__.py +++ b/verifiers/v1/runtimes/docker/__init__.py @@ -73,6 +73,9 @@ class DockerRuntime(ContainerRuntime): engine: ClassVar[str] = "docker" """The CLI binary for the shared OCI container operations.""" info_cls: ClassVar[type[BaseRuntimeInfo]] = DockerRuntimeInfo + supports_checkpoints: ClassVar[bool] = True + """A checkpoint is the container committed to a local image (` commit`), so + a restore is a run from that image.""" def __init__( self, @@ -152,6 +155,43 @@ def published_port(self) -> int: return SERVICE_PORT async def start(self) -> None: + await self._start(self.config.image) + + async def restore(self, checkpoint: str) -> None: + self.info.restored_from = checkpoint + await self._start(checkpoint) + + async def checkpoint(self) -> str: + assert self._container is not None + image = f"localhost/vf-checkpoint:{uuid.uuid4().hex}" + # `commit` pauses the container while it captures, and the image keeps the + # container's config (env, user, workdir) for the run that restores it. + committed = await self._run_host( + self.engine, + "commit", + "--change", + f"LABEL verifiers.run={run_scope()}", + self._container, + image, + ) + if committed.exit_code != 0: + raise SandboxError( + f"{self.engine} commit failed: {committed.stderr.strip()}" + ) + return image + + async def discard_checkpoints(self, ids: list[str]) -> None: + if not ids: + return + removed = await self._run_host(self.engine, "rmi", "--force", *ids) + if removed.exit_code != 0: + logger.warning( + "%s: failed to remove checkpoint images: %s", + self.engine, + removed.stderr.strip(), + ) + + async def _start(self, image: str) -> None: try: version = await cli(self.engine, "version") except FileNotFoundError as e: @@ -239,7 +279,7 @@ async def start(self) -> None: "sleep", "--name", self._container, - self.config.image, + image, "infinity", ) if run.exit_code != 0: @@ -323,7 +363,7 @@ async def start(self) -> None: "%s: started container %s (image=%s)", self.engine, self._container, - self.config.image, + image, ) def host_url(self, url: str) -> str: diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index 1a1d46e80d..58bb2a7014 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -7,6 +7,7 @@ import math import shlex import tempfile +import time from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -45,6 +46,10 @@ _OUTPUT_RETRIES = 10 """Re-reads of a finished job's output that the SDK still failed to fetch, before the exec is reported as failed.""" +_CHECKPOINT_DURABLE_SECONDS = 300 +"""How long a checkpoint gets to become restorable (`DURABLE`): the platform captures +and uploads it after the request returns, and only a durable one restores. A turn +waits this long at most before it goes upstream.""" BASE_LABELS: list[str] = [] @@ -145,6 +150,7 @@ async def kill(self) -> None: class PrimeRuntime(Runtime): is_local: ClassVar[bool] = False + supports_checkpoints: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -158,7 +164,19 @@ def supports_live_processes(self) -> bool: return True async def start(self) -> None: - from prime_sandboxes import AsyncSandboxClient, CreateSandboxRequest + await self._boot(docker_image=self.config.image, disk_size_gb=self.config.disk) + + async def restore(self, checkpoint: str) -> None: + # The image and disk come with the checkpoint; the platform refuses them here. + self.info.restored_from = checkpoint + self._lease_client() + await self._wait_durable(checkpoint) + await self._boot(checkpoint_id=checkpoint) + + def _lease_client(self) -> None: + if self._client is not None: + return + from prime_sandboxes import AsyncSandboxClient loop = asyncio.get_running_loop() shared = _shared_clients.get(loop) @@ -168,6 +186,11 @@ async def start(self) -> None: ) shared.leases += 1 self._client = shared.client + + async def _boot(self, **source: Any) -> None: + from prime_sandboxes import CreateSandboxRequest + + self._lease_client() # Map the resources onto prime's API (minutes, split GPU; memory/disk are already # GB). gpu_type/region are only sent when set (else provider-chosen). gpu_type, gpu_count = parse_gpu(self.config.gpu) @@ -181,7 +204,6 @@ async def start(self) -> None: options = { "cpu_cores": self.config.cpu, "memory_gb": self.config.memory, - "disk_size_gb": self.config.disk, "gpu_count": gpu_count, "timeout_minutes": -1, "idle_timeout_minutes": idle_minutes, @@ -207,8 +229,8 @@ async def create_and_capture_id(): CreateSandboxRequest( name=self.name, labels=list(dict.fromkeys(labels)), - docker_image=self.config.image, environment_vars=self.env, + **source, **{k: v for k, v in options.items() if v is not None}, ) ) @@ -231,7 +253,12 @@ async def create_and_capture_id(): ) await self._client.wait_for_creation(self.info.id, max_attempts=180) logger.info( - "prime: sandbox %s up (image=%s)", self.info.id, self.config.image + "prime: sandbox %s up (image=%s%s)", + self.info.id, + self.config.image, + f", restored from {self.info.restored_from}" + if self.info.restored_from + else "", ) await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" @@ -241,6 +268,45 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e + async def checkpoint(self) -> str: + # Blocking: the turn goes upstream only once the snapshot is durable, so it + # cannot include that turn's writes, and teardown (which fails pending + # checkpoints) never loses one. + try: + checkpoint = await self._client.checkpoint(self.info.id) + except Exception as e: + raise SandboxError(f"prime checkpoint failed: {e}") from e + started = time.monotonic() + await self._wait_durable(checkpoint.id) + logger.debug( + "prime: checkpoint %s durable after %.1fs on sandbox %s", + checkpoint.id, + time.monotonic() - started, + self.info.id, + ) + return checkpoint.id + + async def _wait_durable(self, checkpoint: str) -> None: + try: + await self._client.wait_for_checkpoint( + checkpoint, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + ) + except Exception as e: + raise SandboxError(f"prime checkpoint {checkpoint}: {e}") from e + + async def discard_checkpoints(self, ids: list[str]) -> None: + if self._client is None: + return + for checkpoint in ids: + try: + await self._client.client.request( + "DELETE", f"/sandbox/checkpoints/{checkpoint}" + ) + except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + logger.warning( + "prime: failed to delete checkpoint %s: %s", checkpoint, e + ) + async def prepare_execution(self, routes: list[str] | None) -> None: """Apply the host policy after setup and wait until the platform enforces it.""" if not self.network_restricted: diff --git a/verifiers/v1/serve/delta.py b/verifiers/v1/serve/delta.py index e41bad72dd..2900d2ab55 100644 --- a/verifiers/v1/serve/delta.py +++ b/verifiers/v1/serve/delta.py @@ -50,6 +50,7 @@ "extra_usage", "request_rewrites", "response_rewrites", + "checkpoints", ) """Append-only on the worker: each delta carries the items past the sent count.""" @@ -91,7 +92,9 @@ def dump(model: BaseModel, **kwargs: Any) -> dict: class TraceCursor: """What of one trace has been sent.""" - def __init__(self) -> None: + def __init__(self, generation: int = 0) -> None: + self.generation = generation + """The `Trace.generation` the sent items belong to.""" self.sent = dict.fromkeys(LIST_FIELDS, 0) self.links: list[int] = [] """Per sent node, how many of its semantic links went out with or after it.""" @@ -206,7 +209,16 @@ def diff(self) -> list[tuple[str, dict, TraceCursor | None]]: for trace in traces: delta: dict[str, Any] = {"trace": trace.id} sent = self.cursors.get(trace.id) - cursor = copy.deepcopy(sent) if sent is not None else TraceCursor() + if sent is not None and sent.generation != trace.generation: + # A rewound trace (a retry resumed from a checkpoint) no longer extends + # what went out: the client drops its copy and receives it anew. + deltas.append((trace.id, {"trace": trace.id, "discard": True}, None)) + sent = None + cursor = ( + copy.deepcopy(sent) + if sent is not None + else TraceCursor(generation=trace.generation) + ) if sent is None: delta["open"] = dump(trace, include=set(HEADER_FIELDS)) links: dict[int, list[dict]] = {} diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index ae2e52b530..45ccad41ce 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -186,6 +186,9 @@ class RolloutSession: gated_tools: set[tuple[str, str]] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" prepared_users: Counter[str] = field(default_factory=Counter) + checkpointer: Callable[[graph.PendingTurn], Awaitable[None]] | None = None + """Snapshots the rollout's box right before a turn goes upstream (`Rollout` sets it + when its runtime supports checkpoints and a retry may resume from one).""" @property def stopped(self) -> bool: diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index be14b1d425..ce4c7a9ee9 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -189,6 +189,22 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall prev_total = call.usage.total_tokens +class Checkpoint(BaseModel): + """A snapshot of the rollout's box taken as a model turn went upstream: the box holds + the effects of every message in `node`'s conversation plus `tail`, and nothing of the + turn that followed. A retry resumes there (`Trace.rewind`).""" + + id: str + """The runtime's checkpoint id (`Runtime.checkpoint`).""" + node: int | None = None + """The last committed node whose conversation the box's state follows; None when + the snapshot preceded the first turn.""" + tail: Messages = Field(default_factory=list, exclude=True) + """The turn's prompt messages after `node` that had no node yet (tool results, user + turns): with `node`'s conversation, the prompt a resumed harness continues from. + Live-only — a retry resumes in the process that took the snapshot.""" + + class Branch(BaseModel): """A root-to-leaf message-graph path; each branch becomes one training sample.""" @@ -403,6 +419,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) """Response changes made by `@intercept`, in execution order.""" + checkpoints: list[Checkpoint] = Field(default_factory=list) + """Snapshots of the run's box, oldest first — one per model turn on a runtime that + supports them; a retry resumes from the newest.""" rewards: dict[str, Reward | None] = Field(default_factory=dict) """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a @@ -437,6 +456,14 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): _pending: dict[int, list[Message]] = PrivateAttr(default_factory=dict) """The messages of the request in flight that no node holds yet (the harness's tool results and user turns): a preview for live watchers until the turn commits.""" + _generation: int = PrivateAttr(default=0) + """Bumped by `rewind`: the graph is no longer an extension of what it was, so a + consumer streaming it incrementally starts over.""" + + @property + def generation(self) -> int: + """How many times the trace was rewound (`rewind`).""" + return self._generation def watch(self, on_change: Callable[[Trace], None]) -> None: """Have `on_change` called at each of this trace's phase changes and turns.""" @@ -761,6 +788,32 @@ def record_timeout(self, stage: str) -> None: self.is_timeout = True self.stop(f"{stage}_timeout", override=True) + def rewind(self, checkpoint: Checkpoint) -> list[str]: + """Roll the graph back to `checkpoint` — the nodes committed after it go, its own + conversation stays sampled — and reopen the trace for another attempt: the outcome + stamps reset, while `errors` and the calls that committed no node stay as the + attempts' history. Returns the ids of the later checkpoints this drops.""" + keep = 0 if checkpoint.node is None else checkpoint.node + 1 + if keep > len(self.nodes): + raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + del self.nodes[keep:] + self._head_index = {} + self._pending = {} + self._generation += 1 + self.calls = [c for c in self.calls if c.node is None or c.node < keep] + dropped = [ + c.id for c in self.checkpoints if c.node is not None and c.node >= keep + ] + self.checkpoints = [c for c in self.checkpoints if c.id not in dropped] + self.rewards = {} + self.metrics = {} + self.root_reply = None + self.is_completed = False + self.ok = False + self.stop_condition = None + self.is_timeout = False + return dropped + def record_error(self, error: Exception) -> None: """Record an error, and stop the trace as failed: `_error` for a typed rollout error (`errors.stop_condition`), `error` for any other.""" diff --git a/verifiers/v1/utils/retries.py b/verifiers/v1/utils/retries.py index 11dc474956..8b8d2b3835 100644 --- a/verifiers/v1/utils/retries.py +++ b/verifiers/v1/utils/retries.py @@ -15,7 +15,7 @@ import asyncio import logging import random -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Iterable from typing import TYPE_CHECKING from tenacity import ( @@ -85,10 +85,10 @@ def _retryable(error: Error | None, retry: RetryConfig) -> bool: return True -def trace_should_retry(trace, retry: RetryConfig) -> bool: - """Whether a finished agent rollout should be retried: any captured error on - its trace is retryable (all captures count, not just the most recent).""" - return any(_retryable(e, retry) for e in trace.errors) +def errors_should_retry(errors: Iterable[Error], retry: RetryConfig) -> bool: + """Whether a finished agent rollout should be retried: any error its attempt + captured is retryable (all captures count, not just the most recent).""" + return any(_retryable(e, retry) for e in errors) def episode_should_retry(episode: Episode, retry: RetryConfig) -> bool: