From cf766fa0b9d6eed721dd27b5dea5f49582073383 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Wed, 30 Sep 2026 19:04:43 +0000 Subject: [PATCH 1/2] feat: resume retries from runtime checkpoints Runtimes that can snapshot their filesystem (prime through the checkpoint SDK, docker/podman through `commit`) checkpoint the box right before each model turn goes upstream, while the box is idle. The trace records them, and an agent retry resumes from the newest one instead of starting over: the trace rewinds to the checkpoint (earlier turns stay sampled), the new box boots from the snapshot, task setup is skipped, and the harness continues the conversation. A failed attempt bound for a retry keeps its checkpoints past teardown; every other run discards them. Rewound traces stream again from scratch over the env-server delta protocol, so a client assembling deltas ends with the trace the worker finished with. `prime-sandboxes` is pinned to the branch of PrimeIntellect-ai/prime#971 until the checkpoint SDK is published; the platform API it needs (PrimeIntellect-ai/platform#5914) is not deployed yet, so the prime rows of the e2e test are unverified live. Co-Authored-By: Claude Fable 5.1 --- pyproject.toml | 2 + tests/v1/fixtures/checkpoint_resume_v1.py | 72 +++++++++ tests/v1/test_e2e.py | 67 ++++++++ tests/v1/test_serve_delta.py | 39 +++++ uv.lock | 8 +- verifiers/v1/__init__.py | 2 + verifiers/v1/agent.py | 80 +++++++--- verifiers/v1/configs/retries.py | 7 + verifiers/v1/interception/server.py | 4 + verifiers/v1/rollout.py | 184 ++++++++++++++++++---- verifiers/v1/runtimes/base.py | 26 +++ verifiers/v1/runtimes/docker/__init__.py | 44 +++++- verifiers/v1/runtimes/prime.py | 98 +++++++++++- verifiers/v1/serve/delta.py | 16 +- verifiers/v1/session.py | 3 + verifiers/v1/trace.py | 53 +++++++ verifiers/v1/utils/retries.py | 10 +- 17 files changed, 643 insertions(+), 72 deletions(-) create mode 100644 tests/v1/fixtures/checkpoint_resume_v1.py diff --git a/pyproject.toml b/pyproject.toml index 8598cc3086..e1bf80a68a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -115,6 +115,8 @@ url = "https://pypi.org/simple" default = true [tool.uv.sources] +# Filesystem checkpoints ship in prime-cli PR #971 (unreleased): pin the SDK to that branch. +prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime.git", subdirectory = "packages/prime-sandboxes", rev = "c10a621" } compact = { path = "environments/compact", editable = true } glossary = { path = "environments/glossary", editable = true } deepwiki = { path = "environments/deepwiki", editable = true } diff --git a/tests/v1/fixtures/checkpoint_resume_v1.py b/tests/v1/fixtures/checkpoint_resume_v1.py new file mode 100644 index 0000000000..0ebc8e5684 --- /dev/null +++ b/tests/v1/fixtures/checkpoint_resume_v1.py @@ -0,0 +1,72 @@ +"""checkpoint-resume: an agentic task whose first attempt dies mid-rollout, so the retry +resumes from the box's newest filesystem checkpoint instead of starting over. + +`setup` stamps a nonce into the box that the prompt never reveals; the task asks the model +for two file steps that build on each other, and the reward reads the second file back +out of the box. A request `@intercept` raises a `ProviderError` once the first attempt has +made two turns — after the checkpoint of turn 2 exists, whose box holds step one's file. +The resumed attempt continues from that checkpoint: it never sees the injected failure +again (the trace already carries the error), inherits step one, and finishes. Needs an +agentic (shell) harness that can continue a conversation, e.g. `bash`, on a runtime with +checkpoints (docker, prime). +""" + +import uuid + +import verifiers.v1 as vf +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import Runtime + +SYSTEM = "You complete the task by running shell commands with the bash tool." +MARKER = "marker.txt" +STEP_ONE = "step1.txt" +DONE = "done.txt" +FAIL_AFTER_TURNS = 2 + + +class CheckpointResumeTask(vf.Task[vf.TaskData]): + async def setup(self, trace: vf.Trace, runtime: Runtime) -> None: + nonce = uuid.uuid4().hex + trace.info["nonce"] = nonce + await runtime.write(MARKER, nonce.encode()) + + @vf.intercept + def fail_once(self, request: vf.Request, trace: vf.Trace) -> None: + # Request hooks also judge each proposed tool call after a turn commits, over + # the branch plus an empty probe result; fail only as a real turn goes upstream + # (every step's command echoes, so a real result is never empty). + last = request.messages[-1] + probe = isinstance(last, vf.ToolMessage) and not last.content + if trace.num_turns >= FAIL_AFTER_TURNS and not probe and not trace.errors: + raise vf.ProviderError("injected mid-rollout failure", status_code=503) + + @vf.reward(weight=1.0) + async def copied_marker(self, trace: vf.Trace, runtime: Runtime) -> float: + try: + content = (await runtime.read(DONE)).decode(errors="replace") + except (SandboxError, OSError, ValueError): + return 0.0 + return float(content.strip() == trace.info["nonce"]) + + +class CheckpointResumeTaskset(vf.Taskset[CheckpointResumeTask, vf.TasksetConfig]): + def load(self) -> list[CheckpointResumeTask]: + return [ + CheckpointResumeTask( + vf.TaskData( + idx=0, + prompt=( + "Complete these steps in order, one bash command per turn, " + "and wait for each result before the next.\n" + f"1. Run `cp {MARKER} {STEP_ONE} && echo copied`.\n" + f"2. Run `cat {STEP_ONE} > {DONE} && echo written`.\n" + "3. Reply with the single word: done." + ), + system_prompt=SYSTEM, + ), + self.config.task, + ) + ] + + +__all__ = ["CheckpointResumeTaskset"] diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 4982cd1fab..d5111d3f08 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -576,6 +576,73 @@ async def test_agentic(run_v1, harness, harness_runtime, tmp_path): assert trace.reward == 1.0 +# Runtimes whose box can be checkpointed and restored: docker commits the container to +# an image, prime snapshots the sandbox filesystem (needs the platform's checkpoint API). +CHECKPOINT_RUNTIMES = [ + pytest.param("docker", marks=[mark.bash, mark.docker], id="bash-harness-in-docker"), + pytest.param("prime", marks=[mark.bash, mark.prime], id="bash-harness-in-prime"), +] + + +@pytest.mark.e2e +@pytest.mark.parametrize("harness_runtime", CHECKPOINT_RUNTIMES, indirect=True) +async def test_retry_resumes_from_checkpoint(run_v1, harness_runtime, tmp_path): + """A retry resumes the failed attempt's trace from its newest checkpoint: the box + restores to the state before the turn that failed, the turns up to there stay, + and the task's `setup` (a nonce the prompt never reveals) is not rerun.""" + (trace,) = await run_v1( + "checkpoint-resume-v1", + harness="bash", + runtime={"type": harness_runtime}, + env={"agent": {"retries": {"max_retries": 1, "include": ["ProviderError"]}}}, + output_dir=tmp_path, + max_turns=8, + max_tokens=4096, + ) + assert_resumed_from_checkpoint(trace) + + +@pytest.mark.e2e +@pytest.mark.bash +@pytest.mark.docker +async def test_retry_resumes_from_checkpoint_server(run_v1_server, tmp_path): + """The same resume through the env-server pool: the worker's retry rewinds the + trace it is streaming, and the client still assembles the trace the worker + finished with.""" + (trace,) = await run_v1_server( + "checkpoint-resume-v1", + harness="bash", + runtime={"type": "docker"}, + env={"agent": {"retries": {"max_retries": 1, "include": ["ProviderError"]}}}, + output_dir=tmp_path, + max_turns=8, + max_tokens=4096, + ) + assert_resumed_from_checkpoint(trace) + + +def assert_resumed_from_checkpoint(trace) -> None: + from checkpoint_resume_v1 import FAIL_AFTER_TURNS + + assert trace.ok + assert trace.reward == 1.0 + assert [error.type for error in trace.errors] == ["ProviderError"] + # The failure hit as turn FAIL_AFTER_TURNS + 1 went upstream, so the newest + # checkpoint is the one taken as turn FAIL_AFTER_TURNS was asked for: the box then + # held the earlier turns' effects, and the trace kept those turns. + assert trace.agent.runtime is not None + restored_from = trace.agent.runtime.restored_from + assert restored_from is not None + assert restored_from in [checkpoint.id for checkpoint in trace.checkpoints] + assert trace.num_turns >= FAIL_AFTER_TURNS + assert len(trace.checkpoints) >= trace.num_turns + # Every turn kept has its call; the failure never went upstream, so no call + # records it. + assert [call.node for call in trace.calls] == [ + i for i, node in enumerate(trace.nodes) if node.sampled + ] + + @pytest.mark.e2e async def test_multi_agent_env(run_v1, tmp_path): """An `Env` subclass shipped with its taskset (duet-v1): two roles run the diff --git a/tests/v1/test_serve_delta.py b/tests/v1/test_serve_delta.py index cb2b9ef5a2..51e7232183 100644 --- a/tests/v1/test_serve_delta.py +++ b/tests/v1/test_serve_delta.py @@ -109,6 +109,45 @@ async def send(data: dict) -> None: assert len(assembly.traces[trace.id]["calls"]) == 2 +@pytest.mark.asyncio +async def test_rewound_trace_streams_again_from_scratch(): + """A retry resumed from a checkpoint rewinds its trace below what the client + holds; the stream drops the client's copy and resends the trace as it now is.""" + traces: list[vf.Trace] = [] + frames: list[bytes] = [] + + async def send(data: dict) -> None: + frames.append(pack(data)) + + async with DeltaStreamer(lambda: traces, send) as streamer: + trace = make_trace() + traces.append(trace) + streamer.watch(trace) + add_turn(trace, "a1") + add_turn(trace, "a2") + add_turn(trace, "a3") + await settle() + assert len(assembled(frames)[trace.id]["nodes"]) == 4 + trace.record_error(RuntimeError("boom")) + trace.rewind(vf.Checkpoint(id="cp", node=1)) + trace.notify() # the resumed rollout's boot + await settle() + add_turn(trace, "a2 again") + await settle() + traces = [trace] + record = assembled(frames)[trace.id] + assert [n["message"]["content"] for n in record["nodes"]] == ["q", "a1", "a2 again"] + assert len(record["calls"]) == 2 + assert [e["type"] for e in record["errors"]] == ["RuntimeError"] + + +def assembled(frames: list[bytes]) -> dict[str, dict]: + assembly = EpisodeAssembly() + for frame in frames: + assembly.apply(unpack(frame)) + return assembly.traces + + @pytest.mark.asyncio async def test_pending_preview_streams_and_clears_on_commit(): traces: list[vf.Trace] = [] diff --git a/uv.lock b/uv.lock index 4b36b7f431..2e31e258f1 100644 --- a/uv.lock +++ b/uv.lock @@ -3485,8 +3485,8 @@ wheels = [ [[package]] name = "prime-sandboxes" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } +version = "0.3.2" +source = { git = "https://github.com/PrimeIntellect-ai/prime.git?subdirectory=packages%2Fprime-sandboxes&rev=c10a621#c10a621a82b7212432f15a3627265d662caa71a7" } dependencies = [ { name = "aiofiles" }, { name = "certifi" }, @@ -3497,10 +3497,6 @@ dependencies = [ { name = "pyqwest" }, { name = "tenacity" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b6/46/2d118d8bbb901b729f3bc0678725eb28088d757a872db04cffa1492d0208/prime_sandboxes-0.3.0.tar.gz", hash = "sha256:c99075ed82a9bbfe5a453cee84b9f09565a7f6b07fd5ace072401cce59a1fe08", size = 132759, upload-time = "2026-09-18T01:32:31.263Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/04/c77c7f260babf93978f912b85511e76f63a3cfa5bb11a8c34108177d5825/prime_sandboxes-0.3.0-py3-none-any.whl", hash = "sha256:19078ff508fa72de8f565422918faf367e2831ed8a2f26a5a6a98110e4062941", size = 71441, upload-time = "2026-09-18T01:32:30.062Z" }, -] [[package]] name = "prime-traces" diff --git a/verifiers/v1/__init__.py b/verifiers/v1/__init__.py index b0cd5e1802..ae78beefd5 100644 --- a/verifiers/v1/__init__.py +++ b/verifiers/v1/__init__.py @@ -103,6 +103,7 @@ AgentInfo, AgentSpan, Branch, + Checkpoint, Error, InterceptRecord, ModelCall, @@ -226,6 +227,7 @@ "WireTaskData", "TaskResources", "TaskTimeout", + "Checkpoint", "Trace", "TraceTask", "WireTrace", diff --git a/verifiers/v1/agent.py b/verifiers/v1/agent.py index a8dd4f4541..f996ebb83e 100644 --- a/verifiers/v1/agent.py +++ b/verifiers/v1/agent.py @@ -36,7 +36,7 @@ ) from verifiers.v1.session import RolloutLimits from verifiers.v1.task import Task -from verifiers.v1.trace import Trace +from verifiers.v1.trace import Checkpoint, Error, Trace from verifiers.v1.types import ( AssistantMessage, Messages, @@ -50,7 +50,7 @@ resolve_runtime_config, validate_pairing, ) -from verifiers.v1.utils.retries import backoff, trace_should_retry +from verifiers.v1.utils.retries import backoff, errors_should_retry __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"] @@ -361,15 +361,19 @@ def _interception_for( return self._server return None - def _check_resume_support(self) -> None: - # Multi-turn capability is derived: a harness needs transcript replay, a - # native resume implementation, or a rollout-scoped session implementation. + def _supports_resume(self) -> bool: + """Whether the harness can continue a conversation it is handed: transcript + replay, a native resume implementation, or a rollout-scoped session.""" harness = self.harness - if ( - type(harness).session is Harness.session - and type(harness).resume is Harness.resume - and not harness.SUPPORTS_RESUME - ): + return ( + type(harness).session is not Harness.session + or type(harness).resume is not Harness.resume + or harness.SUPPORTS_RESUME + ) + + def _check_resume_support(self) -> None: + if not self._supports_resume(): + harness = self.harness raise ValueError( f"Harness {harness.config.id!r} cannot host a user: resuming an " "exchange takes transcript-backed resume (SUPPORTS_RESUME) for the " @@ -394,18 +398,42 @@ async def run( owner, counted in the pairing check; `on_trace` observes the trace the moment it's minted, before any I/O. `collect_artifacts` captures the task's declared artifacts after its finalizer while its container runtime is still - alive. Retries whole while the trace ends with a retryable error - (`config.retries`) — never into a borrowed box; the final trace keeps earlier - attempts' errors.""" + alive. Retries while the trace ends with a retryable error (`config.retries`) + — never into a borrowed box. A retry resumes from the failed attempt's newest + checkpoint when it has one (`retries.checkpoint`, a runtime that supports + them, a harness that can continue a conversation), keeping that trace and its + turns up to there; otherwise it starts over. The final trace keeps every + attempt's errors.""" if self._closed: raise RuntimeError("Agent is closed; create a new agent") retry = self.config.retries - history: list = [] + can_resume = retry.checkpoint and runtime is None and self._supports_resume() + history: list[Error] = [] # errors of the attempts started over from + trace: Trace | None = None + resume: Checkpoint | None = None for attempt in range(retry.max_retries + 1): + resumed = trace if resume is not None else None + seen = len(resumed.errors) if resumed is not None else 0 + + def should_retry( + current: Trace, *, attempt: int = attempt, seen: int = seen + ) -> bool: + # This attempt's own errors decide — a resumed trace still carries + # the earlier attempts'. + return attempt < retry.max_retries and errors_should_retry( + current.errors[seen:], retry + ) + trace = await self._run_once( - task, runtime, tools, on_trace, collect_artifacts + task, + runtime, + tools, + on_trace, + collect_artifacts, + resume=(resumed, resume) if resumed is not None else None, + retry=should_retry if can_resume and retry.max_retries else None, ) - if attempt == retry.max_retries or not trace_should_retry(trace, retry): + if not should_retry(trace): break if runtime is not None: logger.warning( @@ -413,16 +441,22 @@ async def run( "longer the task's start state); the error stands" ) break - history.extend(trace.errors) + resume = trace.checkpoints[-1] if can_resume and trace.checkpoints else None + if resume is None: + history.extend(trace.errors) delay = backoff(attempt) logger.warning( - "retrying agent rollout (retry %d/%d) in %.1fs after error: %s", + "retrying agent rollout (retry %d/%d) in %.1fs after error: %s (%s)", attempt + 1, retry.max_retries, delay, trace.last_error.type if trace.last_error else "?", + f"resuming from checkpoint {resume.id}" + if resume is not None + else "from scratch", ) await asyncio.sleep(delay) + assert trace is not None if history: # The full history rides the final trace either way; success is the # `ok` stamp, never errors-emptiness. @@ -436,6 +470,9 @@ async def _run_once( shared_tools: Mapping[str, SharedToolServer] | None, on_trace: Callable[[Trace], None] | None, collect_artifacts: bool, + *, + resume: tuple[Trace, Checkpoint] | None = None, + retry: Callable[[Trace], bool] | None = None, ) -> Trace: params = self._rollout_params(task, runtime, dict(shared_tools or {})) if collect_artifacts and isinstance(params["runtime_config"], SubprocessConfig): @@ -447,11 +484,16 @@ async def _run_once( task=task, on_trace=on_trace, collect_artifacts=collect_artifacts, + resume=resume, + retry=retry, **params, ) try: if await run.open(): - await run.step() + if resume is not None: + await run.resume() + else: + await run.step() if run.ok: run.trace.stop("agent_completed") trace = await run.close() diff --git a/verifiers/v1/configs/retries.py b/verifiers/v1/configs/retries.py index 488a4e4860..3a24999dcb 100644 --- a/verifiers/v1/configs/retries.py +++ b/verifiers/v1/configs/retries.py @@ -15,3 +15,10 @@ class RetryConfig(BaseConfig): """Only retry errors whose type is listed. Empty = retry anything not excluded.""" exclude: list[str] = Field(default_factory=list) """Never retry errors whose type is listed (wins over `include`).""" + checkpoint: bool = True + """Checkpoint the runtime's filesystem before every model turn (on runtimes that + support it, e.g. prime and docker) and resume a retry from the newest checkpoint: + the trace keeps its turns up to there and the harness continues the conversation + in a box restored to that state. False retries from scratch. Only the box is + restored — state kept outside it (a tool server's typed rollout state) stays as + the failed attempt left it.""" diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index d59118f877..07934c7427 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -779,6 +779,10 @@ async def sample() -> web.Response: started = time.time() try: try: + # The box is idle while the harness awaits this turn: the snapshot + # matches the prompt exactly (tool results in, this reply not yet). + if session.checkpointer is not None: + await session.checkpointer(turn) # What actually goes upstream: the native body with the rollout's model + # sampling imposed — recorded raw on the trace, per call. if relay: diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 0e3613e1a3..10d9f72bff 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -8,6 +8,7 @@ from contextlib import AsyncExitStack from dataclasses import dataclass +from verifiers.v1 import graph from verifiers.v1.clients import ModelContext from verifiers.v1.configs.agent import AgentConfig from verifiers.v1.configs.runtime import NetworkPolicyConfig @@ -30,7 +31,7 @@ from verifiers.v1.session import RolloutLimits, RolloutSession, hook_boundary from verifiers.v1.state import state_cls from verifiers.v1.task import Task -from verifiers.v1.trace import AgentInfo, Trace, TraceTask +from verifiers.v1.trace import AgentInfo, Checkpoint, Trace, TraceTask from verifiers.v1.types import Messages, Request, Response, SystemMessage, UserMessage from verifiers.v1.utils.artifacts import collect from verifiers.v1.utils.decorators import discover_decorated, invoke @@ -71,7 +72,15 @@ def __init__( runtime: Runtime | None = None, on_trace: Callable[[Trace], None] | None = None, collect_artifacts: bool = False, + resume: tuple[Trace, Checkpoint] | None = None, + retry: Callable[[Trace], bool] | None = None, ) -> None: + """`resume` continues a failed attempt's trace from one of its checkpoints: + the trace rewinds to it and this run boots its box from the snapshot instead + of setting the task up again. `retry` says, at close, whether a retry will + resume this run: set, the run checkpoints its box before every turn (on a + runtime that supports it), and a failure bound for a retry keeps them past + teardown.""" self.task = task self.harness = harness self.ctx = ctx @@ -84,20 +93,27 @@ def __init__( self.runtime = runtime self._borrowed_runtime = runtime self._collect_artifacts = collect_artifacts - self.trace: Trace = Trace( - task=TraceTask( - type=type(task).__name__, - data=task.data, - key=task.key, - hash=task.hash, - ), - state=state_cls(type(task))(), - # The seat's resolved config, role overrides included — the agent - # this trace can be reproduced with. - agent=AgentInfo(config=agent_config), - ) - if on_trace is not None: - on_trace(self.trace) + self._retry = retry + self._resume: Checkpoint | None = None + self._stale_checkpoints: list[str] = [] + if resume is not None: + self.trace, self._resume = resume + self._stale_checkpoints = self.trace.rewind(self._resume) + else: + self.trace = Trace( + task=TraceTask( + type=type(task).__name__, + data=task.data, + key=task.key, + hash=task.hash, + ), + state=state_cls(type(task))(), + # The seat's resolved config, role overrides included — the agent + # this trace can be reproduced with. + agent=AgentInfo(config=agent_config), + ) + if on_trace is not None: + on_trace(self.trace) interceptors = [ (hook_boundary(fn, allow_trace=False), fn) for fn in discover_decorated(task, "intercept") @@ -201,13 +217,30 @@ async def open(self) -> bool: runtime = self.runtime assert self.trace.agent is not None # minted with the trace self.trace.agent.runtime = runtime.info - logger.info( - "rollout start: id=%s task=%s harness=%s runtime=%s", - self.trace.id, - self.task.data.idx, - self.harness.config.name, - self.runtime_config.type, - ) + if ( + self._retry is not None + and self._borrowed_runtime is None + and runtime.supports_checkpoints + ): + self._session.checkpointer = self._checkpoint + if self._resume is None: + logger.info( + "rollout start: id=%s task=%s harness=%s runtime=%s", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + ) + else: + logger.info( + "rollout resume: id=%s task=%s harness=%s runtime=%s checkpoint=%s turns=%d", + self.trace.id, + self.task.data.idx, + self.harness.config.name, + self.runtime_config.type, + self._resume.id, + self.trace.num_turns, + ) loop = asyncio.get_running_loop() setup_timeout: asyncio.Timeout | None = None try: @@ -223,7 +256,9 @@ async def open(self) -> bool: "task.prompt, or drive the run through agent.interaction() and open " "it with the first turn(message)" ) - if self._borrowed_runtime is None: + if self._resume is not None: + await runtime.restore(self._resume.id) + elif self._borrowed_runtime is None: await runtime.start() await runtime.prepare_setup() now = time.time() @@ -236,11 +271,16 @@ async def open(self) -> bool: if self._timeouts.setup is None else loop.time() + self._timeouts.setup ) - async with ( - asyncio.timeout_at(setup_deadline) as setup_timeout, - boundary(TaskError, "task setup"), - ): - await invoke(self.task.setup, {"trace": self.trace, "runtime": runtime}) + # A restored box already holds the task's setup; only the harness's + # (idempotent) install runs again. + if self._resume is None: + async with ( + asyncio.timeout_at(setup_deadline) as setup_timeout, + boundary(TaskError, "task setup"), + ): + await invoke( + self.task.setup, {"trace": self.trace, "runtime": runtime} + ) async with ( asyncio.timeout_at(setup_deadline) as setup_timeout, boundary(HarnessError, "harness setup"), @@ -371,14 +411,18 @@ async def open(self) -> bool: self.trace.notify() return not self._session.stopped - async def step(self, messages: Messages | None = None) -> bool: + async def step( + self, messages: Messages | None = None, *, intercept: bool = True + ) -> bool: """Run ONE segment: the harness program to its exit. With `messages`, the segment resumes the exchange with the user's turn(s) (`Harness.resume` — for an exchange the user opens, this is also the first segment, on an empty conversation); without, it launches on the task's own prompt. - Returns whether the exchange can continue — a refused turn (limit, @stop), - a failure (an expired agent timeout included), or a segment that made no - progress all end it.""" + `intercept=False` passes `messages` to the harness as they are — they were + already intercepted when the trace first saw them (a resumed checkpoint's + tail). Returns whether the exchange can continue — a refused turn (limit, + @stop), a failure (an expired agent timeout included), or a segment that + made no progress all end it.""" if not self._opened or self._closed or not self.ok: return False trace = self.trace @@ -396,7 +440,11 @@ async def step(self, messages: Messages | None = None) -> bool: try: async with asyncio.timeout_at(self.deadline_at): assert self._harness_session is not None - if messages is not None and self._session.request_interceptors: + if ( + messages is not None + and intercept + and self._session.request_interceptors + ): prepared, rewrites = await self._session.prepare_users( Request(messages=messages) ) @@ -441,6 +489,67 @@ async def step(self, messages: Messages | None = None) -> bool: # never moved, forever. return self.ok and trace.num_turns > turns_before + async def resume(self) -> bool: + """Run the resumed attempt's first segment: the harness continues the + conversation the checkpoint left off at — its recorded tail (tool results, + user turns) after the trace's rewound branch — in the restored box.""" + assert self._resume is not None + data = self.trace.task.data + # `Harness.resume` re-emits `data.system_prompt` through the program's own + # system message; the tail's copy would double it. + tail = [ + m + for m in self._resume.tail + if m.role != "system" or data.system_prompt is None + ] + return await self.step(tail, intercept=False) + + async def _checkpoint(self, turn: graph.PendingTurn) -> None: + assert self.runtime is not None + try: + checkpoint = await self.runtime.checkpoint() + except Exception: + logger.warning( + "checkpoint failed (rollout %s)", self.trace.id, exc_info=True + ) + return + node = turn.prefix_node_ids[-1] if turn.prefix_node_ids else None + self.trace.checkpoints.append( + Checkpoint(id=checkpoint, node=node, tail=list(turn.tail)) + ) + logger.debug( + "checkpoint %s taken: id=%s node=%s", checkpoint, self.trace.id, node + ) + + async def _settle_checkpoints(self) -> None: + """Decide the fate of the run's snapshots before its box goes: a retry that + will resume from them keeps every restorable one, anything else is freed.""" + runtime = self.runtime + if runtime is None or self._borrowed_runtime is not None: + return + ids = [c.id for c in self.trace.checkpoints] + kept: list[str] = [] + if ids and self._failed and self._retry is not None and self._retry(self.trace): + try: + kept = await runtime.preserve_checkpoints(ids) + except Exception: + logger.warning( + "preserving checkpoints failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + self.trace.checkpoints = [c for c in self.trace.checkpoints if c.id in kept] + stale = [*self._stale_checkpoints, *(i for i in ids if i not in kept)] + if stale: + try: + await runtime.discard_checkpoints(stale) + except Exception: + logger.warning( + "discarding checkpoints failed (rollout %s)", + self.trace.id, + exc_info=True, + ) + async def abort(self) -> None: """Free everything this run holds — the entered servers and an owned runtime — without finalizing or scoring: the escape path when an exception @@ -456,6 +565,10 @@ async def abort(self) -> None: with contextlib.suppress(Exception): await self.harness.cleanup(self.trace, self.runtime) if self._borrowed_runtime is None and self.runtime is not None: + with contextlib.suppress(Exception): + await self.runtime.discard_checkpoints( + [*self._stale_checkpoints, *(c.id for c in self.trace.checkpoints)] + ) with contextlib.suppress(Exception): await self.runtime.stop() @@ -556,6 +669,7 @@ async def close(self) -> Trace: # not a live runtime. A borrowed runtime is its creator's to tear down, # not this rollout's. if self._borrowed_runtime is None and runtime is not None: + await self._settle_checkpoints() try: await runtime.stop() except Exception: @@ -568,6 +682,8 @@ async def close(self) -> Trace: self.task.data.idx, trace.reward, trace.num_turns, - trace.last_error.type if trace.last_error else trace.stop_condition, + trace.last_error.type + if trace.last_error and not trace.ok + else trace.stop_condition, ) return trace diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index c2f806c8b3..b551476250 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -133,6 +133,8 @@ class BaseRuntimeInfo(BaseConfig): borrowed: bool = False """Whether the run was placed into a live box owned by someone else (`Agent.run(runtime=...)`) rather than provisioning its own.""" + restored_from: str | None = None + """The checkpoint this box booted from (`Runtime.restore`); None for a fresh start.""" class Runtime(ABC): @@ -147,6 +149,11 @@ class Runtime(ABC): """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`; the host subprocess runtime overrides this with its per-user cache.""" + supports_checkpoints: ClassVar[bool] = False + """Whether `checkpoint` snapshots the box's filesystem for `restore` to boot a new box + from. A rollout checkpoints before each model turn, so a retry resumes at the newest + turn instead of replaying the run (see `RetryConfig.checkpoint`).""" + info: BaseRuntimeInfo @property @@ -177,6 +184,25 @@ def type(self) -> str: async def start(self) -> None: pass + async def restore(self, checkpoint: str) -> None: + """Boot this box from `checkpoint` — a snapshot another box of this config took + with `checkpoint()` — instead of `start()`.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def checkpoint(self) -> str: + """Snapshot the filesystem and return the checkpoint's id; the box keeps running. + A provider may store the snapshot asynchronously — `preserve_checkpoints` waits + for that before the box goes away.""" + raise NotImplementedError(f"{type(self).__name__} does not support checkpoints") + + async def preserve_checkpoints(self, ids: list[str]) -> list[str]: + """Make the box's checkpoints (`ids`, oldest first) outlive its teardown, and return + the ones that are restorable. A retry resumes from the newest returned id.""" + return list(ids) + + async def discard_checkpoints(self, ids: list[str]) -> None: + """Free checkpoints nothing will restore from. Best-effort and idempotent.""" + async def stop(self) -> None: """Free the provisioned resource on the normal path (the owner's `finally`), shielded from cancellation: a Ctrl-C / SIGTERM cancels that `finally` mid-await, diff --git a/verifiers/v1/runtimes/docker/__init__.py b/verifiers/v1/runtimes/docker/__init__.py index f6661d0f39..26abe39372 100644 --- a/verifiers/v1/runtimes/docker/__init__.py +++ b/verifiers/v1/runtimes/docker/__init__.py @@ -73,6 +73,9 @@ class DockerRuntime(ContainerRuntime): engine: ClassVar[str] = "docker" """The CLI binary for the shared OCI container operations.""" info_cls: ClassVar[type[BaseRuntimeInfo]] = DockerRuntimeInfo + supports_checkpoints: ClassVar[bool] = True + """A checkpoint is the container committed to a local image (` commit`), so + a restore is a run from that image.""" def __init__( self, @@ -152,6 +155,43 @@ def published_port(self) -> int: return SERVICE_PORT async def start(self) -> None: + await self._start(self.config.image) + + async def restore(self, checkpoint: str) -> None: + self.info.restored_from = checkpoint + await self._start(checkpoint) + + async def checkpoint(self) -> str: + assert self._container is not None + image = f"localhost/vf-checkpoint:{uuid.uuid4().hex}" + # `commit` pauses the container while it captures, and the image keeps the + # container's config (env, user, workdir) for the run that restores it. + committed = await self._run_host( + self.engine, + "commit", + "--change", + f"LABEL verifiers.run={run_scope()}", + self._container, + image, + ) + if committed.exit_code != 0: + raise SandboxError( + f"{self.engine} commit failed: {committed.stderr.strip()}" + ) + return image + + async def discard_checkpoints(self, ids: list[str]) -> None: + if not ids: + return + removed = await self._run_host(self.engine, "rmi", "--force", *ids) + if removed.exit_code != 0: + logger.warning( + "%s: failed to remove checkpoint images: %s", + self.engine, + removed.stderr.strip(), + ) + + async def _start(self, image: str) -> None: try: version = await cli(self.engine, "version") except FileNotFoundError as e: @@ -239,7 +279,7 @@ async def start(self) -> None: "sleep", "--name", self._container, - self.config.image, + image, "infinity", ) if run.exit_code != 0: @@ -323,7 +363,7 @@ async def start(self) -> None: "%s: started container %s (image=%s)", self.engine, self._container, - self.config.image, + image, ) def host_url(self, url: str) -> str: diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index 1a1d46e80d..d1a68c7f75 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -45,6 +45,9 @@ _OUTPUT_RETRIES = 10 """Re-reads of a finished job's output that the SDK still failed to fetch, before the exec is reported as failed.""" +_CHECKPOINT_DURABLE_SECONDS = 300 +"""How long a checkpoint gets to become restorable (`DURABLE`): the platform captures +and uploads it after the request returns, and only a durable one restores.""" BASE_LABELS: list[str] = [] @@ -145,6 +148,7 @@ async def kill(self) -> None: class PrimeRuntime(Runtime): is_local: ClassVar[bool] = False + supports_checkpoints: ClassVar[bool] = True def __init__(self, config: PrimeConfig, name: str | None = None) -> None: ensure_prime_auth() @@ -158,7 +162,19 @@ def supports_live_processes(self) -> bool: return True async def start(self) -> None: - from prime_sandboxes import AsyncSandboxClient, CreateSandboxRequest + await self._boot(docker_image=self.config.image, disk_size_gb=self.config.disk) + + async def restore(self, checkpoint: str) -> None: + # The image and disk come with the checkpoint; the platform refuses them here. + self.info.restored_from = checkpoint + self._lease_client() + await self._wait_durable(checkpoint) + await self._boot(checkpoint_id=checkpoint) + + def _lease_client(self) -> None: + if self._client is not None: + return + from prime_sandboxes import AsyncSandboxClient loop = asyncio.get_running_loop() shared = _shared_clients.get(loop) @@ -168,6 +184,11 @@ async def start(self) -> None: ) shared.leases += 1 self._client = shared.client + + async def _boot(self, **source: Any) -> None: + from prime_sandboxes import CreateSandboxRequest + + self._lease_client() # Map the resources onto prime's API (minutes, split GPU; memory/disk are already # GB). gpu_type/region are only sent when set (else provider-chosen). gpu_type, gpu_count = parse_gpu(self.config.gpu) @@ -181,7 +202,6 @@ async def start(self) -> None: options = { "cpu_cores": self.config.cpu, "memory_gb": self.config.memory, - "disk_size_gb": self.config.disk, "gpu_count": gpu_count, "timeout_minutes": -1, "idle_timeout_minutes": idle_minutes, @@ -207,8 +227,8 @@ async def create_and_capture_id(): CreateSandboxRequest( name=self.name, labels=list(dict.fromkeys(labels)), - docker_image=self.config.image, environment_vars=self.env, + **source, **{k: v for k, v in options.items() if v is not None}, ) ) @@ -231,7 +251,12 @@ async def create_and_capture_id(): ) await self._client.wait_for_creation(self.info.id, max_attempts=180) logger.info( - "prime: sandbox %s up (image=%s)", self.info.id, self.config.image + "prime: sandbox %s up (image=%s%s)", + self.info.id, + self.config.image, + f", restored from {self.info.restored_from}" + if self.info.restored_from + else "", ) await self._client.execute_command( self.info.id, f"mkdir -p {shlex.quote(self.config.workdir)}" @@ -241,6 +266,71 @@ async def create_and_capture_id(): ) as e: # provisioning failure is one rollout's problem, not the eval's raise SandboxError(f"prime sandbox provisioning failed: {e}") from e + async def checkpoint(self) -> str: + try: + checkpoint = await self._client.checkpoint(self.info.id) + except Exception as e: + raise SandboxError(f"prime checkpoint failed: {e}") from e + logger.debug( + "prime: checkpoint %s requested on sandbox %s", checkpoint.id, self.info.id + ) + return checkpoint.id + + async def _wait_durable(self, checkpoint: str) -> None: + """Wait until `checkpoint` is `DURABLE`; a failed or overdue one raises.""" + try: + async with asyncio.timeout(_CHECKPOINT_DURABLE_SECONDS): + delay = 0.5 + while True: + state = await self._client.get_checkpoint(checkpoint) + if state.state == "DURABLE": + return + if state.error or state.state == "FAILED": + raise SandboxError( + f"prime checkpoint {checkpoint} failed: " + f"{state.error or state.state}" + ) + await asyncio.sleep(delay) + delay = min(delay * 2, 5) + except SandboxError: + raise + except TimeoutError as e: + raise SandboxError( + f"prime checkpoint {checkpoint} was not durable within " + f"{_CHECKPOINT_DURABLE_SECONDS}s" + ) from e + except Exception as e: + raise SandboxError( + f"prime checkpoint {checkpoint} lookup failed: {e}" + ) from e + + async def preserve_checkpoints(self, ids: list[str]) -> list[str]: + # Deleting the sandbox fails its pending checkpoints, so wait here, before + # teardown. The platform marks a checkpoint durable only after its parent, so + # the newest durable one vouches for every older one. + remaining = list(ids) + while remaining: + try: + await self._wait_durable(remaining[-1]) + return remaining + except SandboxError as e: + logger.warning("prime: %s - falling back to an older checkpoint", e) + remaining.pop() + return remaining + + async def discard_checkpoints(self, ids: list[str]) -> None: + if self._client is None: + return + for checkpoint in ids: + try: + await self._client.client.request( + "DELETE", f"/sandbox/checkpoints/{checkpoint}" + ) + except Exception as e: # noqa: BLE001 - provider cleanup is best-effort + logger.warning( + "prime: failed to delete checkpoint %s: %s", checkpoint, e + ) + async def prepare_execution(self, routes: list[str] | None) -> None: """Apply the host policy after setup and wait until the platform enforces it.""" if not self.network_restricted: diff --git a/verifiers/v1/serve/delta.py b/verifiers/v1/serve/delta.py index e41bad72dd..2900d2ab55 100644 --- a/verifiers/v1/serve/delta.py +++ b/verifiers/v1/serve/delta.py @@ -50,6 +50,7 @@ "extra_usage", "request_rewrites", "response_rewrites", + "checkpoints", ) """Append-only on the worker: each delta carries the items past the sent count.""" @@ -91,7 +92,9 @@ def dump(model: BaseModel, **kwargs: Any) -> dict: class TraceCursor: """What of one trace has been sent.""" - def __init__(self) -> None: + def __init__(self, generation: int = 0) -> None: + self.generation = generation + """The `Trace.generation` the sent items belong to.""" self.sent = dict.fromkeys(LIST_FIELDS, 0) self.links: list[int] = [] """Per sent node, how many of its semantic links went out with or after it.""" @@ -206,7 +209,16 @@ def diff(self) -> list[tuple[str, dict, TraceCursor | None]]: for trace in traces: delta: dict[str, Any] = {"trace": trace.id} sent = self.cursors.get(trace.id) - cursor = copy.deepcopy(sent) if sent is not None else TraceCursor() + if sent is not None and sent.generation != trace.generation: + # A rewound trace (a retry resumed from a checkpoint) no longer extends + # what went out: the client drops its copy and receives it anew. + deltas.append((trace.id, {"trace": trace.id, "discard": True}, None)) + sent = None + cursor = ( + copy.deepcopy(sent) + if sent is not None + else TraceCursor(generation=trace.generation) + ) if sent is None: delta["open"] = dump(trace, include=set(HEADER_FIELDS)) links: dict[int, list[dict]] = {} diff --git a/verifiers/v1/session.py b/verifiers/v1/session.py index ae2e52b530..45ccad41ce 100644 --- a/verifiers/v1/session.py +++ b/verifiers/v1/session.py @@ -186,6 +186,9 @@ class RolloutSession: gated_tools: set[tuple[str, str]] = field(default_factory=set) """Calls the harness's gate asked about before executing.""" prepared_users: Counter[str] = field(default_factory=Counter) + checkpointer: Callable[[graph.PendingTurn], Awaitable[None]] | None = None + """Snapshots the rollout's box right before a turn goes upstream (`Rollout` sets it + when its runtime supports checkpoints and a retry may resume from one).""" @property def stopped(self) -> bool: diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index be14b1d425..ce4c7a9ee9 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -189,6 +189,22 @@ def min_new_input_tokens(calls: Iterable[ModelCall]) -> Iterator[tuple[ModelCall prev_total = call.usage.total_tokens +class Checkpoint(BaseModel): + """A snapshot of the rollout's box taken as a model turn went upstream: the box holds + the effects of every message in `node`'s conversation plus `tail`, and nothing of the + turn that followed. A retry resumes there (`Trace.rewind`).""" + + id: str + """The runtime's checkpoint id (`Runtime.checkpoint`).""" + node: int | None = None + """The last committed node whose conversation the box's state follows; None when + the snapshot preceded the first turn.""" + tail: Messages = Field(default_factory=list, exclude=True) + """The turn's prompt messages after `node` that had no node yet (tool results, user + turns): with `node`'s conversation, the prompt a resumed harness continues from. + Live-only — a retry resumes in the process that took the snapshot.""" + + class Branch(BaseModel): """A root-to-leaf message-graph path; each branch becomes one training sample.""" @@ -403,6 +419,9 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) """Response changes made by `@intercept`, in execution order.""" + checkpoints: list[Checkpoint] = Field(default_factory=list) + """Snapshots of the run's box, oldest first — one per model turn on a runtime that + supports them; a retry resumes from the newest.""" rewards: dict[str, Reward | None] = Field(default_factory=dict) """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a @@ -437,6 +456,14 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): _pending: dict[int, list[Message]] = PrivateAttr(default_factory=dict) """The messages of the request in flight that no node holds yet (the harness's tool results and user turns): a preview for live watchers until the turn commits.""" + _generation: int = PrivateAttr(default=0) + """Bumped by `rewind`: the graph is no longer an extension of what it was, so a + consumer streaming it incrementally starts over.""" + + @property + def generation(self) -> int: + """How many times the trace was rewound (`rewind`).""" + return self._generation def watch(self, on_change: Callable[[Trace], None]) -> None: """Have `on_change` called at each of this trace's phase changes and turns.""" @@ -761,6 +788,32 @@ def record_timeout(self, stage: str) -> None: self.is_timeout = True self.stop(f"{stage}_timeout", override=True) + def rewind(self, checkpoint: Checkpoint) -> list[str]: + """Roll the graph back to `checkpoint` — the nodes committed after it go, its own + conversation stays sampled — and reopen the trace for another attempt: the outcome + stamps reset, while `errors` and the calls that committed no node stay as the + attempts' history. Returns the ids of the later checkpoints this drops.""" + keep = 0 if checkpoint.node is None else checkpoint.node + 1 + if keep > len(self.nodes): + raise ValueError(f"checkpoint {checkpoint.id!r} is past this trace's graph") + del self.nodes[keep:] + self._head_index = {} + self._pending = {} + self._generation += 1 + self.calls = [c for c in self.calls if c.node is None or c.node < keep] + dropped = [ + c.id for c in self.checkpoints if c.node is not None and c.node >= keep + ] + self.checkpoints = [c for c in self.checkpoints if c.id not in dropped] + self.rewards = {} + self.metrics = {} + self.root_reply = None + self.is_completed = False + self.ok = False + self.stop_condition = None + self.is_timeout = False + return dropped + def record_error(self, error: Exception) -> None: """Record an error, and stop the trace as failed: `_error` for a typed rollout error (`errors.stop_condition`), `error` for any other.""" diff --git a/verifiers/v1/utils/retries.py b/verifiers/v1/utils/retries.py index 11dc474956..8b8d2b3835 100644 --- a/verifiers/v1/utils/retries.py +++ b/verifiers/v1/utils/retries.py @@ -15,7 +15,7 @@ import asyncio import logging import random -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Iterable from typing import TYPE_CHECKING from tenacity import ( @@ -85,10 +85,10 @@ def _retryable(error: Error | None, retry: RetryConfig) -> bool: return True -def trace_should_retry(trace, retry: RetryConfig) -> bool: - """Whether a finished agent rollout should be retried: any captured error on - its trace is retryable (all captures count, not just the most recent).""" - return any(_retryable(e, retry) for e in trace.errors) +def errors_should_retry(errors: Iterable[Error], retry: RetryConfig) -> bool: + """Whether a finished agent rollout should be retried: any error its attempt + captured is retryable (all captures count, not just the most recent).""" + return any(_retryable(e, retry) for e in errors) def episode_should_retry(episode: Episode, retry: RetryConfig) -> bool: From 687a4ffb84f247853e2d85ae580f0754635bf279 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 1 Oct 2026 20:57:43 +0000 Subject: [PATCH 2/2] feat: block prime checkpoints until durable Wait for the snapshot with the SDK's `wait_for_checkpoint` before the turn goes upstream, so it can never include that turn's writes and a teardown (which fails pending checkpoints) never loses one. Preserving checkpoints at close therefore needs no prime-specific wait anymore. Pin `prime-sandboxes` to the branch commit that adds the method. Co-Authored-By: Claude Fable 5.1 --- pyproject.toml | 2 +- uv.lock | 2 +- verifiers/v1/runtimes/prime.py | 56 ++++++++++------------------------ 3 files changed, 18 insertions(+), 42 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e1bf80a68a..65690cf3ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,7 +116,7 @@ default = true [tool.uv.sources] # Filesystem checkpoints ship in prime-cli PR #971 (unreleased): pin the SDK to that branch. -prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime.git", subdirectory = "packages/prime-sandboxes", rev = "c10a621" } +prime-sandboxes = { git = "https://github.com/PrimeIntellect-ai/prime.git", subdirectory = "packages/prime-sandboxes", rev = "13c8688" } compact = { path = "environments/compact", editable = true } glossary = { path = "environments/glossary", editable = true } deepwiki = { path = "environments/deepwiki", editable = true } diff --git a/uv.lock b/uv.lock index 2e31e258f1..34758cd6b2 100644 --- a/uv.lock +++ b/uv.lock @@ -3486,7 +3486,7 @@ wheels = [ [[package]] name = "prime-sandboxes" version = "0.3.2" -source = { git = "https://github.com/PrimeIntellect-ai/prime.git?subdirectory=packages%2Fprime-sandboxes&rev=c10a621#c10a621a82b7212432f15a3627265d662caa71a7" } +source = { git = "https://github.com/PrimeIntellect-ai/prime.git?subdirectory=packages%2Fprime-sandboxes&rev=13c8688#13c86889a6c277ae18e3dcaaaccfc2051b293db1" } dependencies = [ { name = "aiofiles" }, { name = "certifi" }, diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index d1a68c7f75..58bb2a7014 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -7,6 +7,7 @@ import math import shlex import tempfile +import time from collections.abc import AsyncIterator from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -47,7 +48,8 @@ exec is reported as failed.""" _CHECKPOINT_DURABLE_SECONDS = 300 """How long a checkpoint gets to become restorable (`DURABLE`): the platform captures -and uploads it after the request returns, and only a durable one restores.""" +and uploads it after the request returns, and only a durable one restores. A turn +waits this long at most before it goes upstream.""" BASE_LABELS: list[str] = [] @@ -267,56 +269,30 @@ async def create_and_capture_id(): raise SandboxError(f"prime sandbox provisioning failed: {e}") from e async def checkpoint(self) -> str: + # Blocking: the turn goes upstream only once the snapshot is durable, so it + # cannot include that turn's writes, and teardown (which fails pending + # checkpoints) never loses one. try: checkpoint = await self._client.checkpoint(self.info.id) except Exception as e: raise SandboxError(f"prime checkpoint failed: {e}") from e + started = time.monotonic() + await self._wait_durable(checkpoint.id) logger.debug( - "prime: checkpoint %s requested on sandbox %s", checkpoint.id, self.info.id + "prime: checkpoint %s durable after %.1fs on sandbox %s", + checkpoint.id, + time.monotonic() - started, + self.info.id, ) return checkpoint.id async def _wait_durable(self, checkpoint: str) -> None: - """Wait until `checkpoint` is `DURABLE`; a failed or overdue one raises.""" try: - async with asyncio.timeout(_CHECKPOINT_DURABLE_SECONDS): - delay = 0.5 - while True: - state = await self._client.get_checkpoint(checkpoint) - if state.state == "DURABLE": - return - if state.error or state.state == "FAILED": - raise SandboxError( - f"prime checkpoint {checkpoint} failed: " - f"{state.error or state.state}" - ) - await asyncio.sleep(delay) - delay = min(delay * 2, 5) - except SandboxError: - raise - except TimeoutError as e: - raise SandboxError( - f"prime checkpoint {checkpoint} was not durable within " - f"{_CHECKPOINT_DURABLE_SECONDS}s" - ) from e + await self._client.wait_for_checkpoint( + checkpoint, timeout_seconds=_CHECKPOINT_DURABLE_SECONDS + ) except Exception as e: - raise SandboxError( - f"prime checkpoint {checkpoint} lookup failed: {e}" - ) from e - - async def preserve_checkpoints(self, ids: list[str]) -> list[str]: - # Deleting the sandbox fails its pending checkpoints, so wait here, before - # teardown. The platform marks a checkpoint durable only after its parent, so - # the newest durable one vouches for every older one. - remaining = list(ids) - while remaining: - try: - await self._wait_durable(remaining[-1]) - return remaining - except SandboxError as e: - logger.warning("prime: %s - falling back to an older checkpoint", e) - remaining.pop() - return remaining + raise SandboxError(f"prime checkpoint {checkpoint}: {e}") from e async def discard_checkpoints(self, ids: list[str]) -> None: if self._client is None: